/*	Copyright  (c)	Günter Woigk 2010 - 2020
					mailto:kio@little-bat.de

	This file is free software.

	Permission to use, copy, modify, distribute, and sell this software
	and its documentation for any purpose is hereby granted without fee,
	provided that the above copyright notice appears in all copies and
	that both that copyright notice, this permission notice and the
	following disclaimer appear in supporting documentation.

	THIS SOFTWARE IS PROVIDED "AS IS", WITHOUT ANY WARRANTY,
	NOT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR
	A PARTICULAR PURPOSE, AND IN NO EVENT SHALL THE COPYRIGHT HOLDER
	BE LIABLE FOR ANY DAMAGES ARISING FROM THE USE OF THIS SOFTWARE,
	TO THE EXTENT PERMITTED BY APPLICABLE LAW.
*/

#include "kio/kio.h"
#include "cstrings/utf8.h"
#include "unix/files.h"
#include "cstrings/tempmem.h"
#include "Preprocessor.h"
#include "Tokenizer.h"
#include "Parser/Names.h"
#include "Error.h"
#include "Ival.h"
#include "Type.h"
#include "Word.h"
//#include "FileInfo.h"
#include "Templates/HashMap.h"
//#include "backends/TargetDescr.h"


namespace vpp
{

static const Word  eol(0,tNL);


static cstr fullpath_for_file(cstr parentfile, cstr filename, StrArray const& includepaths) noexcept
{
	cstr fpath;

	if (filename[0] == '/' || filename[0] == '~')
	{
		fpath = fullpath(filename);
	}
	else
	{
		// first guess: directory of file which contains the #include statement:
		fpath = directory_from_path(parentfile);				// directory from parent file
		fpath = fullpath(catstr(fpath,filename));				// included file's path

		for (uint i=0; errno && i<includepaths.count(); i++)
		{
			fpath = includepaths[i];							// directory from list
			fpath = fullpath(catstr(fpath,filename));			// included file's path
		}
	}
	return fpath;
}

void Preprocessor::add_macro(NameID idf)
{
	// add a defalt text macro
	// spos of words is set to 0
	// macro position is index in words[]

	macros.add(idf, std::move(Words() << Word(0,idf)));
}
void Preprocessor::add_macro(NameID idf, cstr value)
{
	macros.add(idf, std::move(Words() << Word(0,idf) << Word(0,value)));
}
void Preprocessor::add_macro(NameID idf, int value)		// value added with type int or uint
{
	macros.add(idf, std::move(Words() << Word(0,idf) << Word(0, value<0 ? tint : tuint, value)));
}
void Preprocessor::add_macro(NameID idf, uint value)	// value added with type uint
{
	macros.add(idf, std::move(Words() << Word(0,idf) << Word(0, tuint, value)));
}
void Preprocessor::init_macros()
{
	add_macro(t__FILE__, "<sourcefile>");	// replaced in expand_macro()
	add_macro(t__LINE__, 0);				// replaced in expand_macro()
	time_t now = time_t(::now());
	add_macro(t__DATE__, datestr(now));		// "2018-08-31" local time
	add_macro(t__TIME__, timestr(now));		// "23:05:01"   local time

	add_macro(t__OPTIMIZE, optimize);		// code optimization -o
	add_macro(t__TARGET, target.name);		// target identifier --cpu
	add_macro(t__BYTE_ORDER, target.byteorder == ByteOrderHiLo ? "HiLo" : "LoHi");
	if(target.byteorder == ByteOrderLoHi) add_macro(t__LITTLE_ENDIAN);
	if(target.byteorder == ByteOrderHiLo) add_macro(t__BIG_ENDIAN);

	add_macro(t__B2B, target.b2b);		// bit <-> byte shift
	add_macro(t__BPB, target.bpb);		// bits / byte
	add_macro(t__BPC, target.bpc);		// bits / char
	add_macro(t__BPS, target.bps);		// bits / short int
	add_macro(t__BPI, target.bpi);		// bits / int
	add_macro(t__BPL, target.bpl);		// bits / long int
	add_macro(t__BPF, target.bpf);		// bits / float
	add_macro(t__BPLF,target.bplf);		// bits / long float
	add_macro(t__BPSF,target.bpsf);		// bits / short float
	add_macro(t__BPP, target.bpp);		// bits / pointer
}

bool Preprocessor::test (Words& words, uint& wpos, NameID idf)
{
	Word& word = words[wpos];
	if(word.idf == idf) { wpos++; return true; }
	else return false;
}
void Preprocessor::expect (Words& words, uint& wpos, NameID idf)
{
	Word& word = words[wpos++];
	if(word.idf == idf) return;
	throw SyntaxError(word, "'%s' expected", names[idf]);
}
bool Preprocessor::test (uint& wpos, NameID idf)
{
	Word& word = words[wpos];
	if(word.idf == idf) { wpos++; return true; }
	else return false;
}
void Preprocessor::expect (uint& wpos, NameID idf)
{
	Word& word = words[wpos++];
	if(word.idf == idf) return;
	throw SyntaxError(word, "'%s' expected", names[idf]);
}
void Preprocessor::expect_eol (uint wpos)
{
	Word& word = words[wpos];
	if(word.idf == tNL) return;
	throw SyntaxError(word, "end of line expected");
}
Word& Preprocessor::get_idf (uint& wpos)
{
	Word& word = words[wpos++];
	cstr name = names[word.idf];
	if(name[0] == '_' || utf8::is_letter(name)) return word;
	else throw SyntaxError(word, "identifier expected");
}

static uint skip_argument (Words& words, uint wpos)
{
	// skip one argument to macro call
	// skip up to ',' or ')'
	// return: wpos -> tKOMMA, tRKzu or tNL

	for (;;wpos++)
	{
		Word& word = words[wpos];
		NameID idf = word.idf;
		if(idf==tNL || idf==tKOMMA || idf==tRKzu) return wpos;

		if(idf==tRKauf)
		{
			uint wpos0 = wpos;
			do { wpos = skip_argument(words, wpos+1); } while(words[wpos].idf==tKOMMA);
			if (words[wpos].idf == tNL) throw SyntaxError(words[wpos0], "mating ')' not found");
			assert(words[wpos].idf == tRKzu);
		}
	}
}
uint Preprocessor::expand_macro (Words& words, uint wpos0)
{
	// expand the macro call at words[wpos]
	// returns size of replacement text
	// => caller cann skip replacement text to avoid recursive macro expansion
	//
	// the arguments and the replacement text itself are not scanned for macros itself
	// these will be replaced if the caller does not skip the replaced text

	Word& word0 = words[wpos0];
	NameID name = word0.idf;

	if (name == t__FILE__)
	{
		word0 = Word(word0.spos, filename_from_path(included_files.fileForPos(word0.spos).filepath));
		return 1;
	}
	if (name == t__LINE__)
	{
		word0 = Word(word0.spos, tuint, included_files.lineForPos(source.getData(),word0.spos));
		return 1;
	}

	Words& macro = macros[name];
	assert(macro[0].idf == name);

	uint wpos = wpos0 + 1;			// skip macro name
	uint mpos = 1;					// ""

	Words xwords;					// replacement text

	if (macro.count() == 1)			// empty replacement text
	{
	}
	else if(!test(macro,mpos,tRKauf))  // no argument list
	{
		xwords.append(&macro[1], macro.count()-1);
	}
	else							// macro with argument list
	{
		// collect arguments:

		HashMap<NameID, Words> args;		// argument names + supplied arguments
		expect(words, wpos, tRKauf);
		if(!test(macro,mpos,tRKzu))
		{
			for(;;)
			{
				NameID idf = macro[mpos++].idf;
				uint a = wpos;
				wpos = skip_argument(words,wpos);
				uint e = wpos;

				args.add(idf,words.copyofrange(a,e));

				if(!test(macro,mpos,tKOMMA)) break;
				expect(words, wpos, tKOMMA);
			}
		}
		expect(words, wpos, tRKzu);
		expect(macro, mpos, tRKzu);

		// collect replacement text and replace macro arguments:

		while (mpos < macro.count())
		{
			Word& word = macro[mpos++];
			if(args.contains(word.idf))	// macro argument
				xwords.append(args[word.idf]);
			else if (word==tHASH)		// macro argument as string
			{
				if (mpos==macro.count() || !args.contains(macro[mpos].idf))
					throw SyntaxError(word, "macro argument name expected");
				Words& arg = args[macro[mpos++].idf];
				cstr s = "";
				if(arg.count())
				{
					uint32 a = arg[0].spos;
					uint32 e = arg.last().spos + uint32(strlen(names[arg.last().idf]));
					s = substr(&source[a],&source[e]);
				}
				xwords.append(Word(word.spos,s));
			}
			else						// nothing special: copy "as is"
				xwords.append(word);
		}
	}

	// replace macro call with replacement text:

	uint delta = xwords.count() - (wpos-wpos0);	// growth (or shrink)
	if (int(delta)>0) words.insertrange(wpos0,wpos0+delta);
	if (int(delta)<0) words.removerange(wpos0,wpos0-delta);

	for(uint i=0; i<xwords.count(); i++)
	{
		words[wpos0+i] = std::move(xwords[i]);
	}

	return xwords.count();	// return replacement text size if caller want's to prevent recursive expansion
}

enum  // Operator Priorities
{
	pAny = 0,		// whole expression: up to ')' or ','
	pTriadic,		// ?:
	pBoolean,		// && ||
	pCmp, 			// comparisions:	 lowest priority
	pAdd, 			// add, sub
	pMul, 			// mul, div, rem
	pBits, 			// bool/masks:		 higher than add/mult
	pRot, 			// rot/shift
	pUna			// unary operator:	 highest priority
};
void Preprocessor::skip_value (uint& wpos, int prio)
{
	// evaluate up to line break

a:	Word& word = words[wpos++];

	switch(int(word.idf))
	{
	case tNL:		throw SyntaxError(word, "unexpected end of line");
	case tIVAL:		break;
	case tRKauf:	skip_value(wpos,pAny); expect(wpos,tRKzu); break;
	case tEKauf:	skip_value(wpos,pAny); expect(wpos,tEKzu); break;
	case tGKauf:	skip_value(wpos,pAny); expect(wpos,tGKzu); break;
	case tNOTNOT:
	case tNOT:
	case tCPL:
	case tNEG:
	case tADD:		goto a;
	case tDEFINED:	expect(wpos,tRKauf); get_idf(wpos); expect(wpos,tRKzu); break;
	default:		get_idf(--wpos); break;		// ignore macro
	}

// expect operator

o:	NameID oper = words[wpos].idf;

	if(oper==tNL)  return;
	if(prio>=pUna) return;
	if(prio>=pRot) return;
	if(oper==tSL || oper==tSR)  { skip_value(++wpos,pRot); goto o; }
	if(prio>=pBits) return;
	if(oper==tAND || oper==tOR || oper==tXOR) { skip_value(++wpos,pBits); goto o; }
	if(prio>=pMul) return;
	if(oper==tMUL || oper==tDIV || oper==tMOD) { skip_value(++wpos,pMul); goto o; }
	if(prio>=pAdd) return;
	if(oper==tADD || oper==tSUB) { skip_value(++wpos,pAdd); goto o; }
	if(prio>=pCmp) return;
	if(oper==tEQ || oper==tNE ||oper==tLT || oper==tLE ||oper==tGT || oper==tGE) { skip_value(++wpos,pCmp); goto o; }
	if(prio>=pBoolean) return;
	if(oper==tANDAND || oper==tOROR) { skip_value(++wpos,pBoolean); goto o; }
	if(prio>=pTriadic+1) return;
	if(oper==tQMARK)
	{
		skip_value(++wpos,pTriadic);
		expect(wpos,tCOLON);
		skip_value(++wpos,pTriadic);
		//goto o;
	}
	return;	// wpos -> stopper
}
Ival Preprocessor::text_value (uint& wpos) throws
{
	Ival v = value(wpos);
	if(v.isString()) return v;
	throw SyntaxError(words[wpos], "text value expected");
}
Ival Preprocessor::numeric_value (uint& wpos, int prio) throws
{
	Ival v = value(wpos,prio);
	if(v.isNumeric()) return v;
	throw SyntaxError(words[wpos], "numeric value expected");
}
Ival Preprocessor::value (uint& wpos, int prio) throws // SyntaxError
{
	// evaluate up to line break

	Ival v(tvoid);

a:	Word& word = words[wpos++];

	switch(int(word.idf))
	{
	case tNL:
		throw SyntaxError(word, "unexpected end of line");

	case tIVAL:
		v = *word.value;
		break;

	case tRKauf:
		v = value(wpos,pAny);
		expect(wpos,tRKzu);
		break;

	case tNOTNOT:
		v = !!value(wpos,9);
		break;

	case tNOT:
		v = !value(wpos,9);
		break;

	case tCPL:
		v = ~value(wpos,9);
		break;

	case tNEG:
		v = -value(wpos,9);
		break;

	case tADD:
		v = +numeric_value(wpos,9);
		break;

	case tDEFINED:
		expect(wpos,tRKauf);
		v = Ival(tuint, macros.contains(get_idf(wpos).idf));
		expect(wpos,tRKzu);
		break;

	default:
		if (!macros.contains(word.idf))
		{
			if (word.idf < tINT8)	// operator / special char?
				throw SyntaxError(word, "unexpected '%s'", names[word.idf]);
			else
				throw SyntaxError(word, "'%s' not #defined", names[word.idf]);
		}
		expand_macro(words,--wpos);
		goto a;
	}

// expect operator

o:	NameID oper = words[wpos].idf;

	if(oper==tNL)  return v;
	if(prio>=pUna) return v;

	if(prio>=pRot) return v;
	if(oper==tSL)  { v = v << value(++wpos,pRot); goto o; }
	if(oper==tSR)  { v = v >> value(++wpos,pRot); goto o; }

	if(prio>=pBits) return v;
	if(oper==tAND) { v = v & value(++wpos,pBits); goto o; }
	if(oper==tOR)  { v = v | value(++wpos,pBits); goto o; }
	if(oper==tXOR) { v = v ^ value(++wpos,pBits); goto o; }

	if(prio>=pMul) return v;
	if(oper==tMUL) { v = v * value(++wpos,pMul); goto o; }
	if(oper==tDIV) { v = v / value(++wpos,pMul); goto o; }
	if(oper==tMOD) { v = v % value(++wpos,pMul); goto o; }

	if(prio>=pAdd) return v;
	if(oper==tADD) { v = v + value(++wpos,pAdd); goto o; }
	if(oper==tSUB) { v = v - value(++wpos,pAdd); goto o; }

	if(prio>=pCmp) return v;
	if(oper==tEQ)  { v = v == value(++wpos,pCmp); goto o; }
	if(oper==tNE)  { v = v != value(++wpos,pCmp); goto o; }
	if(oper==tLT)  { v = v <  value(++wpos,pCmp); goto o; }
	if(oper==tGT)  { v = v >  value(++wpos,pCmp); goto o; }
	if(oper==tLE)  { v = v <= value(++wpos,pCmp); goto o; }
	if(oper==tGE)  { v = v >= value(++wpos,pCmp); goto o; }

	if(prio>=pBoolean) return v;

	if(oper==tANDAND)
	{
		if(v.is0()) skip_value(++wpos,pBoolean); else v = value(++wpos,pBoolean);
		v = !!v; goto o;
	}
	if(oper==tOROR)
	{
		if(v.is0()) v = value(++wpos,pBoolean); else skip_value(++wpos,pBoolean);
		v = !!v; goto o;
	}

	if(prio >= pTriadic+1) return v;

	if(oper==tQMARK)
	{
		if(v.is0())
		{
			skip_value(++wpos,pTriadic);
			expect(wpos,tCOLON);
			v = value(++wpos,pTriadic);
		}
		else
		{
			v = value(++wpos,pTriadic);
			expect(wpos,tCOLON);
			skip_value(++wpos,pTriadic);
		}
		//goto o;
	}

	return v;	// wpos -> stopper
}

uint Preprocessor::skip (uint wpos)
{
	// skip words up to #ELIF, #ELSE or #ENDIF
	// return wpos -> #ELIF, #ELSE or #ENDIF

	uint epos = wpos;

	while (wpos < words.count())
	{
		if(words[wpos++] != tNL) continue;		// not column 1
		if(words[wpos] != tHASH) continue;		// not '#'

		switch (int(words[++wpos].idf))
		{
		default:
			wpos++;
			continue;

		case tELIF:	return wpos;
		case tELSE:	return wpos;
		case tENDIF:return wpos;

		case tIFDEF:
		case tIFNDEF:
		case tIF:
			do { epos = wpos; wpos = skip(wpos+1); }
			while (words[wpos] == tELIF);

			if(words[wpos] == tELSE)  { epos = wpos; wpos = skip(wpos+1); }
			if(words[wpos] == tENDIF) { wpos++; continue; }

			throw SyntaxError(words[wpos], "#ENDIF expected");
		}
	}

	while (words[epos-1] != tHASH) { epos--; }
	throw SyntaxError(words[epos], "#ENDIF not found");
}

void Preprocessor::include (cstr filename, uint wpos, uint wend)
{
	// add file to included_files[]
	// read file
	// tokenize file
	// append new source to source
	// remove words in range wpos .. wend
	// insert new words into words

	assert(source.last() == 0);

	// read file:
	cstr filepath = fullpath(filename); if(errno) throw FileError(words[wpos],errno,filename);
	FD fd(filepath);
	if(fd.file_size() > 256 MB) throw Error(words[wpos],dataerror,"file too long");
	uint32 filesize = uint32(fd.file_size());

	uint32 sourcesize = source.count();
	if(sourcesize+filesize > ArrayMAX) throw Error(words[wpos],dataerror,"total source size exceeds %u bytes",ArrayMAX);

	source.grow(sourcesize+filesize);
	fd.read_bytes(&source[sourcesize-1],filesize);
	source.last() = 0;

	// tokenize:
	Words new_words = tokenizer.tokenize(&source[0], sourcesize-1);
	if(new_words.count() != 0 && new_words.last() != tNL) new_words.append(eol);

	// add file to included_files[]:
	included_files.append(FileInfo(filepath, fd.file_mtime(), filesize));

	// remove "#include <filename>" from words
	words.removerange(wpos, wend);

	// insert new words into this->words:
	words.insertat(wpos,new_words);
}

uint Preprocessor::parse (uint wpos0)
{
	// handle preprocessor instructions up to tELIF, tELSE, tENDIF or tEOF:
	// should be called with wpos pointing to tNL
	// (at least not pointing to tHASH after tNL because then this won't be detected)

	while (wpos0 < words.count())
	{
		try
		{
			if(macros.contains(words[wpos0].idf))
			{
				// TODO: prevent infinite recursive expansion:
				//
				// #define A B
				// #define B A
				// if(A)
				//
				// #define A A
				// if(A)
				//
				// #define A B A
				// if(A)

				expand_macro(words,wpos0);
				continue;
			}

			if(words[wpos0++] != tNL) continue;
			if(words[wpos0] != tHASH) continue;

			uint wpos = --wpos0 + 2;			// wpos0 -> tNL, wpos -> instr
			Word& instr = get_idf(wpos);

			switch(int(instr.idf))
			{
			default:
			{
				throw SyntaxError(instr, "unknown preprocessor instruction");
			}
			case tERROR:	//	#error "text"  --  Set error with text. source not compiled.
			{
				Ival v = text_value(wpos);		// throws
				throw Error(words[wpos0], usererror, "%s", v.getString());
			}
			case tASSERT:	//	#assert <expression>
			{
				Ival v = numeric_value(wpos);	// throws
				if (v.isnot0()) goto eol;
				throw Error(words[wpos0], fatalerror, "assertion failed");
			}
			case tINCLUDE:	//	#include "filename"			Include file "filename".
			{				//	Files are only included once except if included multiple times from the same file.
				Ival v = text_value(wpos);
				expect_eol(wpos);

				cstr filename = v.getString();
				filename = unquotedstr(filename);
				cstr parentfile = included_files.fileForPos(words[wpos-1].spos).filepath;
				filename = fullpath_for_file(parentfile, filename, includedirs);
				if(errno) { errors.append(FileError(words[wpos0+2],errno,filename)); goto remove; }

				if(included_files.contains(filename)) goto remove;

				include(filename,wpos0+1,wpos+1);
				assert(words[wpos0] == tNL);
				continue;
			}
			case tREQUIRE:	//	#require "filename.c"		Also compile file "filename.c".
			{
				Ival v = text_value(wpos);
				expect_eol(wpos);

				cstr filename = v.getString();
				filename = unquotedstr(filename);
				cstr parentfile = included_files.fileForPos(words[wpos-1].spos).filepath;
				filename = fullpath_for_file(parentfile, filename, includedirs);
				if(errno) { errors.append(FileError(words[wpos0+2],errno,filename)); goto remove; }

				required_files.appendifnew(filename);
				goto remove;
			}
			case tDEFINE:	//	#define	 <name>				Define macro without value
			{				//	#define	 <name> <words>			Define macro as one or more words
							//	#define	 <name>(<args>) <words>	Define macro name with replacement arguments <args>

				// get name and verify that it's an identifier
				Word name = get_idf(wpos);
				logline("#define %s",names[name.idf]);

				// collect and validate argument names:
				Array<NameID> args;
				if(test(wpos,tRKauf) && !test(wpos,tRKzu))
				{
					do
					{	Word& arg = get_idf(wpos);
						if(args.contains(arg.idf)) throw SyntaxError(arg, "argument name redefined");
						args.append(arg.idf);
					}
					while(test(wpos,tKOMMA));
					expect(wpos,tRKzu);
				}
				uint wend = wpos;

				// copy macro definition after "\n#define" incl. name excl. eol "\n":
				Words macro;
				for(wpos=wpos0+3; words[wpos] != tNL; wpos++)
				{
					macro.append(std::move(words[wpos]));
				}

				#if 1
					// expand macros in replacement text once:
					macro << eol;	// stopper
					for(uint i = wend-(wpos0+3); i < macro.count(); )
					{
						NameID idf = macro[i].idf;
						if (macros.contains(idf) && !args.contains(idf))
							 i += expand_macro(macro,i);
						else i++;
					}
					assert(macro.last() == tNL);
					macro.drop();	// remove stopper
				#endif

				// add macro definition to macros[]:
				assert(name.idf = macro[0].idf);
				if(macros.contains(name.idf))
				{
					if(macro != macros[name.idf]) throw SyntaxError(name, "macro redefined");
				}
				else
				{
					macros.add(name.idf,std::move(macro));
				}

				goto remove;
			}
			case tUNDEF:	//	#undef	 <name>, [<name>]	Undefine macro <name>
			{
				do
				{	NameID idf = get_idf(wpos).idf;
					if (macros.contains(idf))
					{
						undefed_macros.append(std::move(macros[idf]));
						macros.remove(idf);
					}
				}
				while(test(wpos,tKOMMA));
				goto eol;
			}
			case tELIF:		instr.idf = tELIF; return wpos-1;
			case tELSE:		instr.idf = tELSE; return wpos-1;
			case tENDIF:	instr.idf = tENDIF; return wpos-1;
			case tIFDEF:	//	#ifdef	 <name>				Same as: #if defined(name)
			{
				if (macros.contains(get_idf(wpos).idf)) goto iftrue; else goto ifnot;
			}
			case tIFNDEF:	//	#ifndef	 <name>				Same as: #if !defined(name)
			{
				if (macros.contains(get_idf(wpos).idf)) goto ifnot; else goto iftrue;
			}
			case tIF:		//	#if	<expression>			can be nested.
			{				//	<expression> = macros, operators (with precedence and pruning)
							//				   and the special function <defined(macroname)>
ifvalue:		if(numeric_value(wpos).isnot0())
iftrue:			{
					expect_eol(wpos);
					uint epos = wpos;				// end for removal

					wpos = parse(wpos);				// -> tELIF, tELSE, tENDIF or tEOF
					if(words[wpos] == tEOF)
					{
						while (words[epos-1] != tHASH) { epos--; }
						throw SyntaxError(words[epos], "#ENDIF not found");
					}

					words.removerange(wpos0,epos);	// entferne Zeile "\n#IF...":
					wpos -= epos-wpos0;				// wpos -> tELIF etc. (again)
					wpos0 = wpos - 2;
					assert(words[wpos0]==tNL);

					while(words[wpos] == tELIF) { wpos = skip(wpos); }
					if(words[wpos] == tELSE) { wpos = skip(wpos); }
				}
				else
ifnot:			{
					expect_eol(wpos);
					wpos = skip(wpos);				// -> tELIF, tELSE or tENDIF

					if(words[wpos] == tELIF) { wpos++; goto ifvalue; }

					if(words[wpos] == tELSE)		// -> active branch
					{
						expect_eol(++wpos);
						uint epos = wpos;				// end pos for removal
						wpos = parse(wpos);				// -> tELIF, tELSE or tENDIF
						words.removerange(wpos0,epos);	// "\n#IFetc...\n#ELSE"
						wpos -= (epos-wpos0);
						wpos0 = wpos - 2;
					}
				}
			}

			expect(wpos,tENDIF);
eol:		expect_eol(wpos);
remove:		assert(words[wpos0].idf==tNL);
			words.removerange(wpos0,wpos);
			assert(words[wpos0].idf==tNL);
			continue;
			}
		}
		catch (Error& e)
		{
			errors.append(e);
			while (++wpos0 < words.count() && words[wpos0].idf != tNL) {}
			continue;
		}
	}

	return words.count() - 1;			// --> tEOF
}

static_assert(tEOF < tRKauf,"");
static_assert(tRKauf+2 == tEKauf,"");
static_assert(tRKauf+3 == tRKzu,"");

uint Preprocessor::skip_block (uint wpos)
{
	for (;;)
	{
		Word& word =words[wpos++];
		NameID idf = word.idf;

		if (idf > tEKzu) continue;	// identifier
		if (idf >= tRKzu) break;	// )}]

		if (idf >= tRKauf)			// ({[
		{
			NameID tKLzu = NameID(idf+3);
			uint epos = skip_block(wpos);

			if (words[epos] == tKLzu) wpos = epos + 1;
			else errors.append(SyntaxError(word, "nesting error: '%s' expected", names[tKLzu]));
		}

		if (idf == tEOF) break;
	}
	return wpos - 1;
}

cstr Preprocessor::tostr (cWord& word)
{
	if (word.idf == tIVAL)
		return ::tostr(*word.value);
	else
		return names[word.idf];		// <-- needs local names map!
}

void Preprocessor::preprocess_file (cstr sourcefile) throws // AnyError
{
	// read file
	// tokenize file
	// preprocess file

	// setup:
	init_macros();

	// read source
	source.append(0);				// 0-delimited c-string
	words.append(Word(0,tNL));		// simplify parser
	if (argv_macros) words.append(tokenizer.tokenize(argv_macros, 0));	// TODO: negative offset?
	if (errors.count()) return;
	include(sourcefile,words.count(),words.count());
	assert(words[words.count()-1]==tNL);

	// preprocess source:
	words.append(Word(0,tEOF));		// stopper
	Word& w1 = words[parse(0)];
	if (w1 != tEOF) errors.append(SyntaxError(w1, "unexpected '#%s'", tostr(w1)));

	// check nesting of (), {} and []:
	Word& w2 = words[skip_block(0)];
	if (w2 != tEOF) errors.append(SyntaxError(w2, "unexpected '%s'", tostr(w2)));
}

void Preprocessor::eraselistfiles(cstr sourcefile, cstr listdir)
{
	listfile_errors = catstr(listdir,basename_from_path(sourcefile),"___errors.txt");
	listfile_macros = catstr(listdir,basename_from_path(sourcefile),"___macros.txt");
	listfile_words  = catstr(listdir,basename_from_path(sourcefile),"___preprocessed.txt");
	listfile_included_files = catstr(listdir,basename_from_path(sourcefile),"___included_files.txt");
	listfile_required_files = catstr(listdir,basename_from_path(sourcefile),"___required_files.txt");

	remove(listfile_errors);
	remove(listfile_macros);
	remove(listfile_words);
	remove(listfile_included_files);
	remove(listfile_required_files);
}

void Preprocessor::write_macro (FD& fd, Words& macro)
{
	fd.write_str("#define");
	for (uint j=0; j<macro.count(); j++)
	{
		Word& word = macro[j];
		fd.write_char(' ');
		fd.write_str(tostr(word));
	}
	fd.write_char('\n');
}

void Preprocessor::writelistfiles ()
{
	if (required_files.count() != 0)
	{
		FD fd1(listfile_required_files,'w');
		for (uint i = 0; i<required_files.count(); i++)
		{
			fd1.write_str(required_files[i]);
			fd1.write_char('\n');
		}
	}

	if (included_files.count() > 1)
	{
		FD fd2(listfile_included_files,'w');
		for (uint i = 1; i<included_files.count(); i++)
		{
			fd2.write_str(included_files[i].filepath);
			fd2.write_char('\n');
		}
	}

	if (errors.count())
	{
		FD fd5(listfile_errors,'w');
		for(uint i=0; i<errors.count(); i++)
		{
			Error& error = errors[i];
			cstr file = included_files.fileForPos(error.pos).filepath;
			file = filename_from_path(file);
			uint line = included_files.lineForPos(&source[0],error.pos);
			fd5.write_fmt("%s line %u: %s\n",file,line,error.msg);
		}

		fd5.write_fmt("\n%u error%s\n", errors.count(), errors.count()==1?"":"s");
	}

	{
		FD fd3(listfile_macros,'w');
		for (uint i = 0; i<macros.count(); i++)
		{
			Words& macro = macros.getItems()[i];
			assert(macros.getKeys()[i] == macro[0].idf);
			write_macro(fd3,macro);
		}
		if (undefed_macros.count())
		{
			fd3.write_str("\n#undef•ined macros:\n\n");
			for (uint i = 0; i<undefed_macros.count(); i++)
			{
				write_macro(fd3,undefed_macros[i]);
			}
		}
	}

	{
		FD fd4(listfile_words,'w');

		bool nnl = 1;

		for (uint i = 0; i<words.count(); i++)
		{
			Word& word = words[i];
			if (word == tNL)
			{
				if (!nnl) fd4.write_char('\n');
				nnl = 1;
			}
			else
			{
				fd4.write_str(tostr(word));
				fd4.write_char(' ');
				nnl = 0;
			}
		}
	}
}

void Preprocessor::preprocess (cstr sourcefile, cstr listdir) throws // AnyError
{
	// read file
	// tokenize file
	// preprocess file
	// output: names[], errors[], source[], words[], included_files[] and required_files[]

	assert(sourcefile && sourcefile[0]=='/');
	assert(!listdir || listdir[0]=='/');

	TempMemPool tmp;

	// setup:
	errors.purge();  // wg. spos becomes void
	source.purge();
	words.purge();
	names.purge();
	macros.purge();
	undefed_macros.purge();
	included_files.purge();
	required_files.purge();

	if (listdir) eraselistfiles(sourcefile, listdir);

	preprocess_file(sourcefile);

	if (listdir && (debug || verbose >= 2)) writelistfiles();
}

Preprocessor::Preprocessor(Names& names, Array<char>& source, FileInfos& incl_files, StrArray& req_files) :
	names(names),
	source(source),
	included_files(incl_files),
	required_files(req_files),
	tokenizer(names,errors)
{}

void preprocess (cstr sourcefile, cstr listdir) throws
{
	Names names;
	Array<char> source;
	FileInfos fi;
	StrArray rf;
	Preprocessor pp(names,source,fi,rf);
	pp.preprocess(sourcefile,listdir);
}





#if 0

inline void remove_nl (Words& words)
{
	// remove all tNL from words[]:
	uint a=0,e=0;
	while (e < words.count())
	{
		Word& w = words[e++];
		if (w.idf != tNL) words[a++] = w;
	}
	words.shrink(a);
}

#endif


} // namespace