// utf-8

/*	Copyright  (c)	Günter Woigk   2001-2003
  					mailto:kio@little-bat.de

	This file is free software

 	This program is distributed in the hope that it will be useful,
 	but WITHOUT ANY WARRANTY; without even the implied warranty of
 	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

 	Permission to use, copy, modify, distribute, and sell this software and
 	its documentation for any purpose is hereby granted without fee, provided
 	that the above copyright notice appear in all copies and that both that
 	copyright notice and this permission notice appear in supporting
 	documentation, and that the name of the copyright holder not be used
 	in advertising or publicity pertaining to distribution of the software
 	without specific, written prior permission. The copyright holder makes no
 	representations about the suitability of this software for any purpose.
 	It is provided "as is" without express or implied warranty.

 	THE COPYRIGHT HOLDER DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
 	INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
 	EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY SPECIAL, INDIRECT OR
 	CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
 	DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
 	TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 	PERFORMANCE OF THIS SOFTWARE.

	2003-07... kio	changes to use UCS4String library
*/


#define SAFE 3
#define LOG  2


#include	"config.h"
#include	<math.h>
#include	"Stream.h"

#define		TOKEN_PASS	1
#include	"token.h"
#define		TOKEN_PASS	2
#include	"token.h"
#define		TOKEN_PASS	3
#include	"token.h"
#define		TOKEN_PASS	4
#include	"token.h"

const double d_ln10 = log(10.0);





/* ----	skip token ------------------------------------------
		skip token and it's immediate arguments
*/
uchar* SkipToken ( uchar* ip )
{
	switch(*ip)
	{
	case tEOF:	return ip;		// don't skip over tEOF
	case tIF:
	case tELSE:
	case tWHILE:
	case tUNTIL:
	case tEXIT:
	case tNEXT:
	case tSHORT:
	case tLOOP:	return ip+1 +sizeof(short);
	case tNUM:	return ip+1 +sizeof(double);
	case tBYTE:	return ip+1 +sizeof(char);
	case tSTR2:	return ip+1 +1 /*csz*/ +2/*chars*/ +ip[1/*csz*/] * *(ushort*)(ip+2/*chars*/);
	case tSTR:
	case tIDF:	return ip+1 +1 /*csz*/ +1/*chars*/ +ip[1/*csz*/]*ip[2/*chars*/];

	default:	return ip+1;	// skip token
	}
}


/* ----	skip matching brackets and flow controls -------------------------------
		expects pointer to opening token
		returns pointer behind closing token.

		we assume that flow control structure is checked and set.

		then we do a shortcut after tUNTIL, tWHILE and tEXIT, because their branch offset
		points behind their corresponding tLOOP. same applies to tELSE / tTHEN.

		shortcut cannot be immediately after tDO, because tDO has no offset field (jump target only).
		shortcut cannot be immediately after tIF, because we do not know, whether tELSE exists or not.
*/
uchar* SkipMatching ( uchar* ip )
{
	if(Error()) return ip;

	uchar c = *ip;					// opening token
	c = c==tEKauf ? tEKzu			// matching closing token
	  : c==tRKauf ? tRKzu
	  : c==tGKauf ? tGKzu
	  : c==tDO    ? tLOOP
	  : c==tIF    ? tTHEN
	  :             0;

	XXTRAP(c==0);

	ip = SkipToken(ip);

	for (;;)
	{
		uchar d = *ip;

		if (d==c)					// closing token found
		{
			return SkipToken(ip);
		}

		switch(d)
		{
		default:
			ip = SkipToken(ip);
			continue;

		case tRKauf:
		case tEKauf:
		case tGKauf:
		case tDO:
		case tIF:
			ip = SkipMatching(ip);
			continue;

		case tELSE:
			XXTRAP(c!=tTHEN);
			ip += *(short*)(ip+1);
			XXTRAP(ip[-1]!=tTHEN);
			return ip;

		case tWHILE:
		case tUNTIL:
		case tEXIT:
//			XXTRAP(c!=tLOOP);			// ist ok: do..loop kann um if..then oder {..} liegen!
			ip += *(short*)(ip+1);
			XXTRAP(ip[-3]!=tLOOP);
			return ip;

/*		case tNEXT:
			XXTRAP(c!=tLOOP);			// ist ok: do..loop kann um if..then oder {..} liegen!
			continue;
*/
	#if XXSAFE
		case tTHEN:
		case tLOOP:
		case tRKzu:
		case tEKzu:
		case tGKzu:
		case tEOF:		IERR();
	#endif
		}
	}
}


static void SetDist ( uptr ip, cuptr dest )
{
	long n = dest-ip;
	if ((short)n!=n) SetError("branch too far");
	*(short*)(ip+1) = n;
}


/* ----	set flow controls in matching range -------------------------------
		if ip points to an opening token,
			then run up to matching closing token
			else run until tEOF
		sets errno on error
		noerror: returns pointer behind closing token or to tEOF.
		error:   returns pointer to error position:
				 if matching closing token is not found, then returns position of opening token.
				 if unexpected intermediate or closing token is found, then this position is returned.

		set branch offsets in flow control tokens
		back branches are set immediately
		forward branches are remembered by building a linked list starting in ip1

		do0==NULL: do0 and do1 are ignored
		else:      do0 and do1 are ip0 and ip1 of an outer do..loop
				   they are used to handle 'unexpected' next, until, while and exit for the outer loop
*/
static uchar* SetOffsets ( uchar* ip, uchar* do0, uchar*& do1 )
{
	if (Error()) return ip;

	uchar* ip0 =  ip;
	uchar* ip1 =  ip0;				// ptr -> tELSE  or  last tUNTIL/tWHILE/tEXIT
	uchar  c   = *ip;				// opening token
	c = c==tEKauf ? tEKzu			// matching closing token
	  : c==tRKauf ? tRKzu
	  : c==tGKauf ? tGKzu
	  : c==tDO    ? tLOOP
	  : c==tIF    ? tTHEN
	  :             tEOF;

	XXXLog("<<%s: %s ",CString(tokenname[c]),CString(tokenname[*ip]));

	ip = SkipToken(ip);

	while ( NoError() )
	{
		uchar d = *ip;

		XXXLog("%s ",CString(tokenname[d]));

		if (d==c)					// closing token found
		{
			XXXLog(" >>");

			if (d==tLOOP)
			{
				SetDist(ip,ip0+1); if (Error()) return ip;
				ip+=3;
				for ( uptr p; ip1!=ip0; ip1=p )
				{
					p = ip1 + *(short*)(ip1+1);	// next forward branch opcode position
					SetDist(ip1,ip);			// set forward branch distance
					if (Error()) return ip1;		// extrem selten: ein tUNTIL direkt nach tDO kann oorange sein, obwohl backbranch tLOOP->tDO gerade noch klappte.
				}
				return ip;
			}
			else if (d==tTHEN)
			{
				ip+=1;
				SetDist(ip1,ip);				// set offset in tIF or tELSE
				return Error() ? ip1 : ip;
			}

			return SkipToken(ip);
		}

		switch(d)
		{
		default:
			ip = SkipToken(ip);
			continue;

		case tEOF:
			XXXLog("[tEOF]");
			SetError( CatStr( "'", CString(tokenname[c]), "' missing") );
			return ip0;

		case tGKauf:
		case tIF:
			XXXLog("[tGKauf|tIF]");
			if(c==tLOOP) ip = SetOffsets(ip,ip0,ip1);
			else		 ip = SetOffsets(ip,do0,do1);
			continue;

		case tRKauf:
		case tEKauf:
		case tDO:
			XXXLog("[tRKauf|tEKauf|tDO]");
			ip = SetOffsets(ip,NULL,ip1/*dummy*/);
			continue;

		case tELSE:
			XXXLog("[tELSE]");
			if (c!=tTHEN) goto ue;
			if (ip1!=ip0) goto ue;		// unexpected 2nd tELSE
			SetDist(ip0,ip+3);
			if (Error()) return ip0;
			ip1 = ip; ip += 3;
			continue;

		case tNEXT:
			XXXLog("[tNEXT]");
			if (c!=tLOOP)
			{
				if(!do0) goto ue;
				SetDist(ip,do0+1);		// do..loop außerhalb von if..then oder {..}
			}
			else
			{
				SetDist(ip,ip0+1);
			}
			if (Error()) return ip;
			ip += 3;
			continue;

		case tWHILE:
		case tUNTIL:
		case tEXIT:
			XXXLog("[tWHILE|tUNTIL|tEXIT]");
			if (c!=tLOOP)
			{
				if(!do0) goto ue;
				SetDist(ip,do1);		// do..loop außerhalb von if..then oder {..}
				do1 = ip;
			}
			else
			{
				SetDist(ip,ip1);
				ip1 = ip;
			}
			if (Error()) return ip;
			ip += 3;
			continue;

		case tTHEN:
		case tLOOP:
		case tRKzu:
		case tEKzu:
		case tGKzu:
			XXXLog("[tTHEN|tLOOP|tRKzu|tEKzu|tGKzu]");
ue:			SetError( CatStr("unexpected '", CString(tokenname[d]), "'") );
		//	SetError( CatStr("expected '", CString(tokenname[c]), "' found '", CString(tokenname[d]), "'" ) );
			return ip;
		}
	}

	return ip;
}




/* ----	disassemble buffer ---------------------------
*/
uchar* Disass ( Stream* stream, uchar*p, int n )
{
	XXTRAP((long)p<0x00008000);

	XXXLog("Disass(%08x,%d)",(int)p,n);

	const int bpz = 40;

	if ( n>bpz )
	{
		while (n>bpz && *p!=tEOF)
		{
			uchar* p0 = p;
			p = Disass(stream,p,bpz); stream->PutNL(); n -= p-p0;
		}
	}

	for ( uchar*e=p+n; p<e; p=SkipToken(p) )
	{
		switch(*p)
		{
/*		case IF:
		case ELSE:
		case UNTIL:
		case WHILE:
		case LOOP:
		case EXIT:
		case NEXT:		stream->PutStr( Using( "%s ", CString(TokenName(*p)) ) ); 	break;
*/
		case tNUM:		stream->PutString( NumString( read_double(p+1) )	);	break;
		case tBYTE:		stream->PutString( NumString( (int)(char)p[1] ) 	);	break;
		case tSHORT:	stream->PutString( NumString( (int)(short)p[1] ) 	);	break;
		case tSTR:		stream->PutString( String((ptr)p+3,p[2],CharSize(p[1])).ToQuoted() ); break;
		case tSTR2:		stream->PutString( String((ptr)p+4,*(ushort*)(p+2),CharSize(p[1])).ToQuoted() );   break;
		case tIDF:		stream->PutString( String((ptr)p+3,p[2],CharSize(p[1])).ToEscaped() ); break;
		case tEOF:		stream->PutString( "end of file" );						return p;
		default:		stream->PutString( TokenName(*p)					);	break;
		}
		stream->PutSpace();
	}
	return p;
}


/* ----	disassemble proc/script ----------------------------
		used for Var::ToString( String(*disass_proc)(cString&) )
*/
String DisassString ( cString& q )
{
	XXTRAP(q.Csz()!=1);
	XXXLog("Disass(cString&)");

	String s = TokenName(tPROC)+" ";
	uptr   p = (uptr)q.Text();
	uptr   e = p + q.Len();

	while ( p<e )
	{
		switch(*p)
		{
		case tNUM:		s += NumString( read_double(p+1) );		break;
		case tBYTE:		s += NumString( (int)(char)p[1] );		break;
		case tSHORT:	s += NumString( (int)(short)p[1] );		break;
		case tSTR:		s += String((ptr)p+3,p[2],CharSize(p[1])).ToQuoted(); break;
		case tSTR2:		s += String((ptr)p+4,*(ushort*)(p+2),CharSize(p[1])).ToQuoted();   break;
		case tIDF:		s += String((ptr)p+3,p[2],CharSize(p[1])).ToEscaped(); break;
		case tEOF:		s += TokenName(tEOF);					goto x;
		default:		s += TokenName(*p);						break;
		}
		s += String(' ');
		p=SkipToken(p);
	}

x:	XXXLogOut("");
	return s;
}


String TokenName ( uint t )
{
	XXXLog("TokenName(%i)",t);
	return t<tokens ? tokenname[t] : CharString('$')+HexString(t);
}

String TokenInfo ( uint t )
{
	return t<tokens ? tokeninfo[t] : "not a valid token";
}

String TokenSyntax ( uint t )
{
	String z = String(' ')+(t<tokens?tokensyntax[t]:"")+' ';

	for ( int i=z.Len()-2; i>=1; i-- )
	{
		if ( z[i]!=' ' && z[i+1]!=' ' && (!_is_letter(z[i]) || !_is_letter(z[i+1])) )
		{
			z = z.LeftString(i+1) + " " + z.MidString(i+1);
		}

		if ( _is_letter(z[i-1]) ) continue;
		if ( _is_letter(z[i+1]) ) continue;

		str r;
		switch(z[i])
		{
		case 'x':	r = "value";	break;
		case 'n':	r = "number";	break;
		case 't':	r = "text";		break;
		case 'l':	r = "list";		break;
		case 'v':	r = "variable";	break;
		case 'i':	r = "name";		break;
		default: 	continue;
		}

		z = z.LeftString(i) + r + z.MidString(i+1);
	}

	return z.MidString(1,z.Len()-2);
}



/* ====	tokenize script ============================================
		tokenize script
		parse flow control  ((IF ELSE { } DO WHILE etc.))

		**NEW** expects UCS4String

		**NEW** idx_io for error position reporting / requesting
*/
String Tokenize ( cString& qstring, long& idx_io )
{
	XXXLogIn("Tokenize(String)");

	String qs = qstring;
	if ( qstring.Csz()!=csz4 || qs.Len()==0 || qs[qs.Len()-1]!=0 ) { qs += CharString(0xFFFFFFFF); *(qs.UCS4Text()+qs.Len()-1) = 0; }
	XXXTRAP(qs.Csz()!=csz4);

	uint     qlen = qs.Len() -1;
	cUCS4Char* qa = qs.UCS4Text();
	cUCS4Char* qe = qa + qlen;
	cUCS4Char* q  = qa;
	cUCS4Char* a  = q;

	long   zlen = qlen/4 +100;		// estimation
	String zs(zlen,csz1);
	uptr   za   = zs.UCS1Text();
	uptr   ze   = za + zlen;
	uptr   z    = za;

/*	za,z,ze,zlen  ->  destination	((z ~ "ziel"))
	qa,q,qe,qlen  ->  source    	((q ~ "quelle"))
*/

	uchar token;
	int comment = 0;						// block comment flag / block comment nesting level
	long n;

// skip line 1: "#!/usr/bin/vipsi" or the like
	if ( q[0]=='#' && q[1]=='!' ) { n=qs.Find(nl); if(n<0) q=qe; else q=qa+n+1; }

// tokenize
	while ( NoError() && (!idx_io||z-za<=idx_io) )
	{
		if (z+260>ze)
		{
			zlen=z-za;						// current used len
			zs += String(qe-q+260,csz1);	// estimation
			za = zs.UCS1Text();
			z  = za + zlen;
			zlen = zs.Len();
			ze = za + zlen;
		}

	// find start of next word
		a = q;
		while ( *a && *a<=' ' ) a++;
		if (*a==0) { q=a; break; }			// end of source
		q = a+1;

		#define T1(C,T)				case C: token=T; break;
		#define T2(C,D,T,U)			case C: if (*q!=D) { token=T; } else { token=U; q++; } break;
		#define T3(C,D,E,T,U,V)		case C: if (*q==D) { token=U; q++; } else if (*q==E) { token=V; q++; } else token=T; break;
		#define T4(C,D,E,F,T,U,V,W)	case C: if (*q==D) { token=U; q++; } else if (*q==E) { token=V; q++; } else if (*q==F) { token=W; q++; } else token=T; break;
		#define TG(C,D,DD,DGL,DDGL) case C: if (*q=='=') { token=DGL; q++; break; } if (*q!=C)   { token=D; break; } q++; if (*q!='=') { token=DD; break; } q++; token=DDGL; break;


	// convert word -> token
		switch(*a)
		{
		T1(',',tKOMMA)		T1(';',tSEMIK)		T1('(',tRKauf)		T1(')',tRKzu)
		T1('{',tGKauf)		T1('}',tGKzu)		T1('[',tEKauf)		T1(']',tEKzu)
		T1('?',tQMARK)		T1(':',tCOLON)		T1('$',tNUM)		T1('"',tSTR)
		T1('_',tIDF)		T1('~',tTILDE)		T1('\'',tNUM)		T1('.',tDOT)
		T1('@',tAT)

		T2('^','=',tXOR,tXORGL)					T2('%','=',tMOD,tMODGL)
		T2('!','=',tNOT,tNE)					T2('=','=',tGL,tEQ)

		TG('|', tOR,tOROR,tORGL,tORORGL)
		TG('&', tAND,tANDAND,tANDGL,tANDANDGL)
		TG('>', tGT,tGTGT,tGE,tGTGTGL)
		TG('<', tLT,tLTLT,tLE,tLTLTGL)
		TG('#', tHASH,tHASHHASH,tHASHGL,tHASHHASHGL)

		T3('+','+','=',tADD,tPLPL,tPLGL)
		T3('*','/','=',tMUL,tKEND,tMUGL)
		T3('-','-','=',tSUB,tMIMI,tMIGL)

		T4('/','/','*','=',tDIV,tKOMM,tKANF,tDVGL)

		default:
			if (UCS4CharIsDecimalDigit(*a))  { token=tNUM; break; }
			if (UCS4CharIsLetter(*a)) { token=tIDF; break; }
			if (comment) continue;	// ill. chars e.g. '\' in comments are legal ...
			SetError ( String("Illegal character: '")+(*a<32?'?':*a)+"', ascii=$"+HexString(*a) );
			goto x;
		}


	// store token
		switch(token)
		{
		case tKOMM:		// comment to end of line starting with '//'
			while(q<qe&&*q!=(uchar)nl) q++;
			continue;

		case tKANF:		// block comment starting with '/*'
			comment++;
			continue;

		case tKEND:		// block comment ending with '*/'
			if (!comment) SetError("Unexpected end of block comment");
			comment--;
			continue;

		case tSTR:		// string literal "text.."
			if (!comment)
			{
			// a -> "
				n = a-qa;
				String s = qs.StrVal( &n );
				if (Error()) goto x;
				q = qa+n;

				if (s.Len()<256)
				{
					*z++ = tSTR;
					*z++ = s.Csz();
					*z++ = s.Len();
				}
				else if (s.Len()<0x10000)
				{
					*z++ = tSTR2;
					*z++ = s.Csz();
					*(ushort*)z = s.Len(); z+=2;
				}
				else
				{
					SetError("Text literal too long"); goto x;
				}

				long bytes = s.Len() * s.Csz();

				if (z+bytes>ze)
				{
					zs.Truncate(z-za);
					zs+= String(bytes,csz1);
					za = (uptr)zs.Text();
					zlen = zs.Len();
					ze = za + zlen;
					z = ze-bytes;
				}
				memcpy(z,s.Text(),bytes); z += bytes;
			}
			continue;

		case tNUM:		// number literal
			if (!comment)
			{
				n = a-qa;
				double f = qs.NumVal( &n );
				if (Error()) goto x;
				q = qa+n;

				if (f==0.0) 			{ *z++ = tNUM0; }
				else if (f==1.0)		{ *z++ = tNUM1; }
				else if (f==(char)f)	{ *z++ = tBYTE;  *z++ = (char)f; }
				else if (f==(short)f)	{ *z++ = tSHORT; *(short*)z = (short)f; z+=sizeof(short); }
				else					{ *z++ = tNUM; write_double(z,f); z += sizeof(double); }
			}
			continue;

		case tIDF:				// Identifier: reserved word, variable etc.
			while ( UCS4CharIsLetter(*q) || UCS4CharIsDecimalDigit(*q) || *q=='_' ) { q++; }

			if (!comment)
			{
				String idf = String(a,q-a);
				int tok = tFROM;		// from = first non-special-char word
				do { if (tokenname[tok]==idf) { *z++ = tok; break; } } while (++tok<=tLOOP);
				if (tok<=tLOOP) { if(tok>=tIF) z+=2; continue; }	// IF .. LOOP take branch distance

			// unknown identifier; e.g. variable name
				if (idf.Len()>=256) { SetError("Identifier length exceeds limit of 255 char"); goto x; }
				*z++ = tIDF;
				*z++ = idf.Csz();
				*z++ = idf.Len();
				n = idf.Csz()*idf.Len();
				memcpy ( z, idf.Text(), n );
				z += n;
			}
			continue;

		default:			// single char word; e.g. '+' or '('
			if (!comment) *z++ = token;
			continue;
		}
	}

	if (comment) SetError("Unterminated block comment");

	if (idx_io || Error())
	{
x:		idx_io = a-qa+1;		// report source indexes starting with 1; 0 = no error
		XXXLogOut("");
		return String((UCS4Char)tEOF);
	}

	*z++ = tEOF;
	XXXTRAP(z>ze);


// calculate branch offsets and check nesting
	z=za;
	while ( NoError() && *z!=tEOF ) z = SetOffsets(z,NULL,za/*dummy*/);


// handle branch control error:
	if (Error())
	{
		PushError();
		idx_io = z-za; if(z==za) idx_io=1;
		Tokenize(qstring,idx_io);
		TRAP(!idx_io);
		PopError();
		XXXLogOut("");
		return String((UCS4Char)tEOF);
	}

// shrink tokenized buffer
// ***TODO*** does not release memory!
	zs.Truncate(z+1-za);

// return tokenized script
	XXXLogOut("");
	return zs;
}






