/*	Copyright  (c)	Günter Woigk 2018 - 2018
					mailto:kio@little-bat.de

	This program is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.




			*** TEST HEADER ***






	THE COPYRIGHT HOLDER DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
	INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
	EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY SPECIAL, INDIRECT OR
	CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
	DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
	TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
	PERFORMANCE OF THIS SOFTWARE.
*/

#define LOGLEVEL 1
#define SAFETY 3
#include "Libraries/kio/kio.h"
#include "Libraries/cstrings/utf8.h"
#include "Tokenizer.h"
#include "Error.h"
#include "Word.h"
#include "Value.h"
#include "Name.h"
#include "Type.h"
#include "utilities.h"
#include "globals.h"

;

#if 0
	tokenize_line(…) tokenisiert idR. eine Zeile bis Nullbyte oder Newline.
	tokenize() tokenisiert alle Zeilen bis zum Nullbyte.
	Der Text muss mit einem Nullbyte abgeschlossen sein. (c-style string)
	Als Zeilenende wird nur Newline '\n' erkannt.
	Der Text muss UTF8-kodiert sein oder reines 7-Bit Ascii.

	tokenize() überspringt '//…' Zeilenkommentare und '/*…*/' Blockkommentare.
	tokenize() tokenisiert u.U. mehr als eine Zeile, wenn ein Blockkommentar oder ein Langstring «…» gefunden wird
	oder wenn die Zeile mit einem Backslash endet. (Ausnahme: Zeilenkommentar)
	Langstrings und Blockkommentare könnengeschachtelt werden.

	Macros werden nicht expandiert.
	Preprozessor-Direktiven werden nicht ausgeführt.

	Die abgetrennten Token werden mit den Methoden des TokenizeInterface übergeben
		und von diesem z.B. in Worte gewandelt und an einen Array gehängt.
		Dabei werden Zeiger in den Originaltext übergeben, bei Strings inkl. Quotes und Escape-Zeichen.

	tokenize() wirft TokenizeError, wenn ein Blockkommentar oder ein String nicht terminiert ist.
		Bei Kurzstrings zeigt die enthaltene pos auf das unerwartete nl oder null,
		bei Blockkommentar und Langstring auf den Blockstarter.

	tokenize() erkennt
	• Strings: ".." '..' `..` und «..»
	• Bezeichner: ['_' <unicodeletter>] ['_' <unicodeletter> 0-9]*
	• Zahlen:
		[0-9] [0-9 a-z A-Z]*
		[0-9]+ '.' [0-9]* [a-z A-Z]*
		[0-9]+ '.' [0-9]* [eE] [+-]? [0-9]+ [a-z A-Z]*
	• Operatoren und Sonderzeichen:
		^!$%&/()=?+*#-.:,;<>@{}[]|
	• Mehrzeichenoperatoren:
		++ -- == != >= <= ≤ ≥ << >> && || -> /% := += -= *= /= %= >>= <<= &&= ||=

	Ketten von Sonderzeichen werden von links nach rechts gebrochen, "+++" wird zu "++" plus "+".

	Zahlen werden nicht ausgewertet. Es wird nicht garantiert, dass sie syntaktisch korrekt sind!
	Vorzeichen werden separat als Operator '+'  bzw. '-' gespeichert.
#endif


static cptr skip_linecomment (cptr q) noexcept
{
	// skip to end of line
	// stop at endofline or null
	// also works with DOS line ends

	while(*q && *q != '\n') q++;
	return q;
}

#ifndef NDEBUG
ON_INIT([]{
	static cstr s1 = "12 \\t4€ Af3'456";
	static cstr s2 = "/2 \\t4€ A+3\n//456'";
	static cstr s3 = "//\t\\t4€ A*3\\\n456'";
	static cstr s4 = "12 \\t4€ //3\r\n//456'";
	assert(skip_linecomment(s1) == s1+17);
	assert(skip_linecomment(s2) == s2+13);
	assert(skip_linecomment(s3) == s3+14);
	assert(skip_linecomment(s4) == s4+14);
});
#endif

static cptr skip_string (cptr q) throws // cstr
{
	// skip beyond next delim
	// throw at nl or null
	// note: '\' before line end works with "\r\n" DOS line ends:
	//		 caller must also handle this when unescaping the string!

	char delim = *q++;
	assert(delim=='"' || delim=='`' || delim=='\'');

	if(*q==delim && *(q+1)==delim) return q+2;	// accept ''' and """ as 1-char strings

	for(char c; (c=*q++) && c!='\n'; )
	{
		if(c == delim) return q;
		if(c == '\\') { if(*q=='\r' && *(q+1)=='\n') q++; if(*q) q++; }
	}
	throw "unterminated string literal";
}

#ifndef NDEBUG
ON_INIT([]{
	static cstr s1 = "'12 \\t4€ Af3'456";
	static cstr s2 = "'12 \t4€ A+3\n456'";
	static cstr s3 = "'12 \"t\\'€ A*3";
	assert(skip_string(s1) == s1+15);
	try{ skip_string(s2); IERR(); } catch(cstr) {}
	try{ skip_string(s3); IERR(); } catch(cstr) {}
});
#endif

static cptr skip_longstring (cptr q) throws // cstr
{
	// skip beyond next '»'
	// skip over nl
	// throw at null
	// skip recursively over '«' .. '»' (must be balanced)
	// note: "\r\n" DOS line ends are preserved
	//		 and caller must handle these when unescaping the string!

	// note: '«' = $C2,$AB
	//       '»' = $C2,$BB

	assert(*q==char(0xC2) && *(q+1)==char(0xAB));
	q += 2;

	while(char c = *q++)
	{
		if(c > 0)
		{
			if(c == '\\') { if(*q) q++; }
		}
		else if(c == char(0xC2))
		{
			c = *q++;
			if(c == char(0xBB)) return q;		// '»'
			if(c == char(0xAB)) { q = skip_longstring(q-2); continue; }
		}
	}
	throw "unterminated string literal";
}

#ifndef NDEBUG
ON_INIT([]{
	static cstr s1 = "«12 \\t4€ A¢3'456»xx";
	static cstr s2 = "«12 \t4€ A+3\n456'»»";
	static cstr s3 = "«12 \"t4«€» ««»»A*3»\n";
	static cstr s4 = "«12 \"t4«€» A*3\n";
	static cstr s5 = "«12 \"t4«» A*3\"";
	static cstr s6 = "«123\\";
	assert(skip_longstring(s1) == s1+22);
	assert(skip_longstring(s2) == s2+21);
	assert(skip_longstring(s3) == s3+29);
	try{skip_longstring(s4); IERR();}catch(cstr){}
	try{skip_longstring(s5); IERR();}catch(cstr){}
	try{skip_longstring(s6); IERR();}catch(cstr){}
});
#endif

static cptr skip_blockcomment (cptr q) throws // cstr
{
	// skip beyond next "*/"
	// skip over nl
	// throw at null

	// nested block comments are detected and skipped. "/*" .. "*/" must be balanced.
	// long strings are detected and skipped. '«' .. '»' must be balanced. spurious '»' are ignored.
	// line comments "//" are detected and skipped.

	// contained short strings should be delimited.
	// '*/' at the end of lines with unbalanced short string delimiters are recognized!
	//		-> natural text with apostrophs should work
	//		while commented-out source is expected to contain only balanced strings.

	assert(*q=='/' && *(q+1)=='*');

	q += 2;
	while(char c = *q++)
	{
		if(c > '/') continue;

		switch(c)
		{
		case '*':
			if(*q == '/') return q+1;	// closing '*/' found
			continue;

		case '/':
			if(*q == '*')      q = skip_blockcomment(q-1);
			else if(*q == '/') q = skip_linecomment(q);
			continue;

		case '\'':
		case '"':
		case '`':
			try { q = skip_string(q-1); }	// skip string (may contain '*/')
			catch(cstr) {}					// else just skip unbalanced quote
			continue;

		default:
			if(c == char(0xC2) && *q == char(0xAB)) q = skip_longstring(q-1);
			continue;
		}
	}

	throw "unterminated block comment";
}

#ifndef NDEBUG
ON_INIT([]{
	{static cstr s = "/*123 \t \nxx**//zz";		assert(skip_blockcomment(s) == s+14);}
	{static cstr s = "/*12'*/'; // xyz\n\t*/ ";	assert(skip_blockcomment(s) == s+20);}
	{static cstr s = "/*/*foo*/«/*\n»*/;";		assert(skip_blockcomment(s) == s+18);}
	{static cstr s = "/*\n//foo*/\n//«\n*/x";	assert(skip_blockcomment(s) == s+18);}
	{static cstr s = "/*it's a lie! */\n";		assert(skip_blockcomment(s) == s+16);}
	{static cstr s = "/*foo\n//*/";				try{skip_blockcomment(s);IERR();}catch(cstr){}}
	{static cstr s = "/*foo «*/";				try{skip_blockcomment(s);IERR();}catch(cstr){}}
	{static cstr s = "/*foo \\";				try{skip_blockcomment(s);IERR();}catch(cstr){}}
});
#endif

static cptr skip_spaces (cptr q) noexcept
{
	// skip spaces
	// skip backslash + endofline
	// stop at endofline, nonspace or null
	// works with "\r\n" DOS line ends

	for(char c; (c = *q++); )
	{
		if(uchar(c) <= ' ') { if(c != '\n') continue; else break; }
		if(c == '\\' && *q == '\n') { q++; continue; }
		if(c == '\\' && *q == '\r' && *(q+1) == '\n') { q+=2; continue; }
		if(c == '/'  && *q == '/') { return skip_linecomment(q+1); }
		else break;
	}
	return q-1;
}

#ifndef NDEBUG
ON_INIT([]{
	static cstr s1 = "12 \\t4€ Af3";
	static cstr s2 = "\t2";
	static cstr s3 = "  \t\\\n\t \\\r\n \t \\\r\n  \r\nx";
	static cstr s4 = "\\\t 12\nx";
	static cstr s5 = "\t";
	assert(skip_spaces(s1) == s1);
	assert(skip_spaces(s2) == s2+1);
	assert(skip_spaces(s3) == s3+19);
	assert(skip_spaces(s4) == s4);
	assert(skip_spaces(s5) == s5+1);
});
#endif

static cptr skip_identifier (cptr q) noexcept
{
	// Identifier: ['_' <unicodeletter>] ['_' <unicodeletter> 0-9]*

	assert(utf8::is_letter(q) || *q == '_');

	do
	{
		q = utf8::nextchar(q);
	}
	while( utf8::is_letter(q) || *q=='_' || utf8::is_dec_digit(q) );

	return q;
}

#ifndef NDEBUG
ON_INIT([]{
	{static cstr s = "L23L+";		assert(skip_identifier(s) == s+4);}
	{static cstr s = "_a65qh.f";	assert(skip_identifier(s) == s+6);}
	{static cstr s = "é1Ä€";		assert(skip_identifier(s) == s+5);}
	{static cstr s = "кирлица+";	assert(skip_identifier(s) == s+14);}
	{static cstr s = "an_0n«»";		assert(skip_identifier(s) == s+5);}
	{static cstr s = "L23L\n";		assert(skip_identifier(s) == s+4);}
	{static cstr s = "L23L";		assert(skip_identifier(s) == s+4);}
});
#endif

inline cptr skip_decimals (cptr p) noexcept
{
	while (is_dec_digit(*p)) { p++; }
	return p;
}

static cptr skip_number (cptr q) noexcept
{
	// Number:
	//	'0x' [0-9,a-f,A-F]+
	//	'0b' [01]+
	// 	[0-9]+
	// 	[0-9]+ '.' [0-9]+
	// 	[0-9]+ '.' [0-9]+ [eE] [+-]? [0-9]+
	// 	[0-9]+            [eE] [+-]? [0-9]+

	if (*q == '0')
	{
		if ((*(q+1)|0x20)=='x' && is_hex_digit(*(q+2)))	// hex number
		{
			q += 3;
			while(is_hex_digit(*q)) { q++; }
			return q;
		}
		if ((*(q+1)|0x20)=='b' && is_bin_digit(*(q+2)))	// bin number
		{
			q += 3;
			while(is_bin_digit(*q)) { q++; }
			return q;
		}
	}

	if (*q == '+' || *q == '-') q++;		// skip sign

	assert(is_dec_digit(*q));
	q = skip_decimals(q+1);					// skip manissa or integer number

	if (*q=='.' && is_dec_digit(*(q+1)))	// decimal dot: fractional part of floating point number
	{										// test is_dec_digit() wg. member functions, e.g. 123.lo()
		q = skip_decimals(q+2);
	}

	if ((*q|0x20)=='e')						// exponent
	{
		cptr q0 = q++;
		if (*q=='+'||*q=='-') q++;
		if (no_dec_digit(*q)) return q0;
		q = skip_decimals(q);
	}

	if (*q == 's' || *q == 'l') q++;		// size specifier

	return q;
}

#ifndef NDEBUG
ON_INIT([]{
	{static cstr s = "123L+";		assert(skip_number(s) == s+3);}
	{static cstr s = "0xA23L+";		assert(skip_number(s) == s+5);}
	{static cstr s = "0A23h0";		assert(skip_number(s) == s+1);}
	{static cstr s = "1e65qh.f";	assert(skip_number(s) == s+4);}
	{static cstr s = "12.34e+5s.";	assert(skip_number(s) == s+9);}
	{static cstr s = "123L+";		assert(skip_number(s) == s+3);}
	{static cstr s = "123L\n";		assert(skip_number(s) == s+3);}
	{static cstr s = "123L";		assert(skip_number(s) == s+3);}
	{static cstr s = "12.L";		assert(skip_number(s) == s+2);}
	{static cstr s = "12e34";		assert(skip_number(s) == s+5);}
	{static cstr s = "12.0e";		assert(skip_number(s) == s+4);}
});
#endif

static cptr skip_operator (cptr q) noexcept
{
	// • Operatoren und Sonderzeichen:
	// 	^!$%&/()=?+*#-.:,;<>@{}[]|
	// • Mehrzeichenoperatoren:
	// 	≤ ≥ ++ -- == != >= <= << >> && || -> /% := += -= *= /= %= >>= <<= &&= ||=

	char c1 = *q;
	if(strchr("+-*/%><:=!&|^",c1))		// potential 2- and 3-char operators
	{
		char c2 = *(q+1);
		static char o[] = "<<=>>=&&=||=++ -- == != >= <= !! -> /% := += -= *= /= %= &= |= ^= ";

		for(uint i=0; i<NELEM(o)-1; i+=3)
		{
			if(c1==o[i] && c2==o[i+1])
			{
				char c3 = *(q+2);
				return c3!=' ' && c3==o[i+2] ? q+3 : q+2;		// 2 or 3 char operator
			}
		}
	}

	// single unicode char, operator or special character, maybe ≥ or ≤
	return utf8::nextchar(q);
}

#ifndef NDEBUG
ON_INIT([]{
	{static cstr s = ">=";		assert(skip_operator(s) == s+2);}
	{static cstr s = ">> ";		assert(skip_operator(s) == s+2);}
	{static cstr s = ">>=";		assert(skip_operator(s) == s+3);}
	{static cstr s = "<>>";		assert(skip_operator(s) == s+1);}
	{static cstr s = "!a";		assert(skip_operator(s) == s+1);}
	{static cstr s = "+++";		assert(skip_operator(s) == s+2);}
	{static cstr s = "+--";		assert(skip_operator(s) == s+1);}
	{static cstr s = "%\n";		assert(skip_operator(s) == s+1);}
	{static cstr s = "%==\n";	assert(skip_operator(s) == s+2);}
	{static cstr s = "¢";		assert(skip_operator(s) == s+2);}
	{static cstr s = "≥";		assert(skip_operator(s) == s+3);}
	{static cstr s = "&&=||=";	assert(skip_operator(s) == s+3);}
	{static cstr s = "&&||=";	assert(skip_operator(s) == s+2);}
	{static cstr s = "&||=";	assert(skip_operator(s) == s+1);}
});
#endif

static IdfID parse_identifier (cptr a, cptr e)
{
	// identifier ID 'IdfID' is retrieved from names[]
	// for names, operators, separators, etc.

	char c = *e; *ptr(e) = 0;
	IdfID id = names.add(a);
	*ptr(e) = c;
	return id;
}

static Value* parse_number (cptr a, cptr e) throws // cstr
{
	// parse number literal and return Value object
	//
	//	'0x' [0-9,a-f,A-F]+ [sl]?
	//	'0b' [01]+ [sl]?
	// 	[+-]? [0-9]+ [sl]?
	// 	[+-]? [0-9]+ '.' [0-9]* [sl]?
	// 	[+-]? [0-9]+ '.' [0-9]* [eE] [+-]? [0-9]+ [sl]?

	bool s = *(e-1) == 's';		// short float if available, or short integer (not recommended)
	bool l = *(e-1) == 'l';		// long float if available,  or long integer  (rarely recommended)
	e -= s|l;

	char c = *e; *ptr(e) = 0;				// stopper, must be undone
	ptr z;
	float128 value;
	cBasicType* type = nullptr;
	errno = noerror;

	if (*a=='0' && (*(a+1)|0x20)=='x')		// 0x1234..
	{
		value = strtoull(a+2,&z,16);
		*ptr(e) = c;

		uint bits = (uint(e - a) -2) * 4;
		type = bits <= 8  ? tushort : bits <= 16 ? tuint16 : bits <= 32 ? tuint32 : tuint64;
		if (type->bits > tulong->bits) throw "literal size exceeds sizeof ulong";
	}
	else if (*a=='0' && (*(a+1)|0x20)=='b')	// 0b0101..
	{
		value = strtoull(a+2,&z,2);
		*ptr(e) = c;

		uint bits = uint(e - a) -2;
		type = bits <= 8  ? tushort : bits <= 16 ? tuint16 : bits <= 32 ? tuint32 : tuint64;
		if (type->bits > tulong->bits) throw "literal size exceeds sizeof ulong";
	}
	else if (skip_decimals(a) == e)			// 1234..
	{
		value = strtoull(a,&z,10);
		*ptr(e) = c;
		if(errno) goto x;

		type = value <= tushort->max ? tushort :
			   value <= tuint16->max ? tuint16 :
			   value <= tuint32->max ? tuint32 : tuint64;

		if (l && type->bits < tulong->bits)  type = tulong;
		if (s && type->bits > tushort->bits) throw "value exceeds sizeof ushort";
		if (type->bits > tulong->bits) throw "value exceeds sizeof ulong";
	}
	else if ((*a=='+' || *a=='-') && skip_decimals(a+1) == e)	// ±1234..
	{
		value = strtoll(a,&z,10);
		*ptr(e) = c;
		if(errno) goto x;

		type = value >= tshort->min && value <= tshort->max ? tshort :
			   value >= tint16->min && value <= tint16->max ? tint16 :
			   value >= tint32->min && value <= tint32->max ? tint32 : tint64;

		if (l && type->bits < tlong->bits)  type = tlong;
		if (s && type->bits > tshort->bits) throw "value exceeds sizeof short";
		if (type->bits > tlong->bits) throw "value exceeds sizeof long";
	}
	else									// float
	{
		value = std::strtold(a,&z);
		*ptr(e) = c;
		if(errno) goto x;

		type = s ? tsfloat : l ? tlfloat : tfloat;
		if (type == nullptr) throw "the target system does not support floating point numbers";
		if (!type->canRepresent(value))
		{
			cstr sfx = type->name;
			if (type->isInfinity(value)) throw usingstr("%s: value reached infinity", sfx);
			if (type->isRoundToZero(value)) throw usingstr("%s: value rounded to null", sfx);
			if (type->isDenormalized(value))
			{
				if (type == tlfloat || type->numLostBits(value) < 3)
					logline("%s %s: %i bits lost due to denormalization", sfx, substr(a,e), type->numLostBits(value));
				else throw usingstr("%s: %i bits lost due to denormalization", sfx, type->numLostBits(value));
			}
		}
	}

x:	if(errno) throw strerror(errno);		// EINVAL, ERANGE
	assert(z==e);
	assert(*e==c);
	return new Value(type,value);
}

static Value* parse_base256_number(cptr a, cptr e) throws // cstr
{
	assert (*a=='\'');	// base-256 number

	errno = noerror;
	cstr s = utf8::unescapedstr(substr(a+1,e-1));	// preserves escaped char(0) as 2 byte code
	//if(errno){}
	uint len = utf8::charcount(s);

	if (len==1)	// single character: may be any unicode value in range of target's char type
	{
		uint32 n;
		utf8::utf8_to_ucs4(s,&n);
		//if(errno){}
		if (n > tchar->basicType()->max) throw "character code exceeds char";
		return new Value(tchar,n);
	}
	else		// multi char: ucs1 only
	{
		if (len < 1) throw "base-256 literal: min. 1 character required";
		if (len > 8) throw "base-256 literal: max. 8 characters allowed";

		ucs1char us[8] = {0,0,0,0,0,0,0,0};
		utf8::utf8_to_ucs1(s,us+8-len);
		if(errno) throw "base-256 number: character code exceeds ucs1 (latin-1)";

		uint64 n = peek8Z(us);
		auto type = len <= 2 ? tuint16 : len <= 4 ? tuint32 : tuint64;
		if (type->bits > tulong->bits) throw "literal size exceeds ulong";
		return new Value(type,n);
	}
}

static Value* parse_string(cptr a, cptr e) throws // cstr
{
	// parse string literal and return Value object
	// cstring is unquoted, unescaped and utf8-encoded
	// DOS linebreaks in long strings are normalized to '\n'

	//errno = noerror;

	if (*a=='"')			// short string
	{
		cstr s = utf8::unescapedstr(substr(a+1,e-1));
		//if(errno){}
		return new Value(s);
	}
	else if(*a==char(0xC2))	// long string
	{
		str s = substr(a+2,e-2);
		normalizeLinebreaks(s);
		s = utf8::unescapedstr(s);
		//if(errno){}
		return new Value(s);
	}
	else IERR();
}

Words tokenize (cstr source, uint32 offs) throws	// AnyError
{
	// tokenize source[offs++] --> words[]
	// skips:  initial BOM and SHEBANG
	//         white space, line and block comments
	// stores: identifiers, operators etc., numbers and text values in words[]
	//         linebreaks (except if inside string or block comment)
	// stores: errors in errors[]
	// throws: if too many errors

	Words words;
	cptr a, q = source + offs;
	uint32 spos;

	// skip BOM:
	if (*q==char(0xEF) && *(q+1)==char(0xBB) && *(q+2)==char(0xBF)) q += 3;

	// skip SHEBANG:
	if (*q=='#' && *(q+1)=='!') { q = strchr(q,'\n'); if(!q) return words; }

	// DOIT:
	while (errors.count() <= 30)
	{
		try
		{
			for (a = q = skip_spaces(q); char c = *a; a = q = skip_spaces(q))
			{
				spos = uint32(a-source);

				if(utf8::is_letter(a) || c=='_')
				{
					q = skip_identifier(a);
					words.append(Word(spos,parse_identifier(a,q)));
				}
				else if(is_dec_digit(c))
				{
	num:			q = skip_number(a);
					if (*q=='s' || *q=='l') q++;
					words.append(Word(spos,parse_number(a,q)));	// throws
				}
				else if(c=='"')
				{
					q = skip_string(a);			// throws
					words.append(Word(spos,parse_string(a,q)));	// throws
				}
				else if(c=='\'')
				{
					q = skip_string(a);			// throws
					words.append(Word(spos,parse_base256_number(a,q)));	// throws
				}
				else if(c==char(0xC2) && *(a+1)==char(0xAB))	// '«'
				{
					q = skip_longstring(a);		// throws
					words.append(Word(spos,parse_string(a,q)));
				}
				else if(c == '/'  && *(a+1) == '*')
				{
					q = skip_blockcomment(a);	// throws
					continue;
				}
				else if((c=='+' || c=='-') && is_dec_digit(*(a+1)))
				{
					// try to use '+' and '-' as numeric sign:
					//	this is important to handle numeric signs before unsigned values
					//	which grow in size when they become signed
					//	e.g. 40000 = uint16  -->  +40000 = int32
					//	because Value::operator+(int) will not grow size beyond size of dflt. int
					//	but truncate the value instead, to mimic what compiled code does.

					//char d = *(q+1);
					//char e = *(q+2) | 0x20;
					//if(d=='0' && (e=='x' || e=='b')) goto op;

					// test for number sign:
					// pass-through bei nl
					// nicht nach ++ --
					// nach operator
					// nach ({[,;~!?:
					// nicht nach string, number or other idf
					// nicht nach )}]		note: ')' könnte cast sein

					uint i = words.count();
					IdfID idf; do { idf = words[--i].idf; } while(idf == tNL);

					if(idf==tIVAL) goto op;					// after literal number or string
					if(idf==tINCR || idf==tDECR) goto op;	// must be postfixes: ++ival is not possible
					if(idf<=tEKauf) goto num;				// after operator ( { [ , ;
					//if(idf==tRKauf || idf==tGKauf || idf==tEKauf) goto num;
					//if(idf==tKOMMA || idf==tSEMIK) goto num;
					goto op;
				}
				else
				{
	op:				q = skip_operator(a);
					words.append(Word(spos,parse_identifier(a,q)));
				}
			}
			return words;	// ok
		}
		catch (cstr msg)
		{
			errors.append(SyntaxError(spos, "%s", msg));
		}
	}

	// too many errors:
	throw AnyError(fatalerror,"too many errors");
}