/// This module was generated by the StaticLang tool from
/// Goldie v0.4: http://www.semitwist.com/goldie/

// Goldie: GOLD Engine for D
// Written in the D programming language.

module samples.calculatorStatic.calc.lexer;

version = GoldieStaticLang;

import std.string;
import std.utf;

import semitwist.util.all;

import goldie.base;
import goldie.exception;
import goldie.file;
import goldie.lang;
import goldie.parser;
import goldie.token;

version(GoldieStaticLang) import goldie.lexer;
version(GoldieStaticLang) import samples.calculatorStatic.calc.token;

//TODO? Make lexer/parser a producer/consumer relationship for streamed parsing
//TODO: Make lexer steppable
//TODO? Generate recording of steps taken
public class Lexer_calc : Lexer
{
	

	//TODO? Make comment tok instead of error tok if error occurs in comment mode
	public override Token[] process(string source, Language lang, string filename="")
	{
		this.source = source;
		
		string tok;
		
		srcIndex=0;
		line=0;
		this.filename = filename;
		int  state = lang.initialDFAState;
		bool eof = false;
		bool errorFound = false;
		
		auto commentMode = CommentType.None;
		auto commentModeOfErr = CommentType.None;
		
		bool foundAcceptState;
		typeof(state)    lastAcceptState;
		typeof(srcIndex) lastAcceptIndex;
		
		getTables(lang);
		
		uint cSize;

		Token mergeErrors(Token tok1, Token tok2)
		in
		{
			mixin(deferAssert!(`tok1.type == SymbolType.Error`));
			mixin(deferAssert!(`tok2.type == SymbolType.Error`));
			mixin(deferAssert!(`tok1.id == lang.errorSymbol.id`));
			mixin(deferAssert!(`tok2.id == lang.errorSymbol.id`));
			mixin(deferEnsure!(`tok1.srcIndexStart`, `_ < tok2.srcIndexStart`));
			mixin(deferEnsure!(`tok1.srcIndexEnd`, `_ < tok2.srcIndexEnd`));
			mixin(deferEnsure!(`tok1.srcIndexEnd`, `_ == tok2.srcIndexStart`));
			flushAsserts();
		}
		body
		{
			
			version(GoldieStaticLang)
				alias Token_calc!(SymbolType.Error, "Error") _ErrorToken;
			
			return
				new _ErrorToken(
					lang,
					tok1.toString(TokenToStringMode.Full) ~ tok2.toString(TokenToStringMode.Full),
					tok1.file, tok1.line,
					tok1.srcIndexStart,
					tok2.srcIndexEnd,
					tok1.commentMode,
					tok1.debugInfo
				);
		}
		
//TODO: Make sure col values are correct when there are multi-byte chars
		bool advanceState(dchar c)
		{
			bool charAccepted = false;
			foreach(DFAStateEdge edge; dfaTable[state].edges)
			{
				//TODO: This "contains" is called a lot and takes a lot of time,
				//      try to optimize it out, maybe with character set optimization
				if(contains(charSetTable[edge.charSetIndex].chars, c))
				{
					state = edge.targetDFAStateIndex;
					charAccepted = true;
					tok ~= source[srcIndex..srcIndex+cSize];//to!(string)(c~""d);
					break;
				}
			}
			return charAccepted;
		}
		
		void addToken(Symbol symbol, string content)
		{
			int srcIndexOffset = 0;
			if(symbol.type == SymbolType.Error)
			{
				if(commentMode == CommentType.None)
					errorFound = true;

				if(!eof)
					srcIndexOffset = 1;
			}

			if(content != "" || symbol.type == SymbolType.EOF)
			{
				
				switch(symbol.id)
				{
				case 0:
				_tokens ~=
					new Token_calc!(SymbolType.EOF, "EOF")(
						lang, content,
						filename, line,
						srcIndex+srcIndexOffset-content.length,
						srcIndex+srcIndexOffset,
						commentMode,
						"commentMode: %s".format(commentMode)
					);
					break;

				case 1:
				_tokens ~=
					new Token_calc!(SymbolType.Error, "Error")(
						lang, content,
						filename, line,
						srcIndex+srcIndexOffset-content.length,
						srcIndex+srcIndexOffset,
						commentMode,
						"commentMode: %s".format(commentMode)
					);
					break;

				case 2:
				_tokens ~=
					new Token_calc!(SymbolType.Whitespace, "Whitespace")(
						lang, content,
						filename, line,
						srcIndex+srcIndexOffset-content.length,
						srcIndex+srcIndexOffset,
						commentMode,
						"commentMode: %s".format(commentMode)
					);
					break;

				case 3:
				_tokens ~=
					new Token_calc!(SymbolType.Terminal, "-")(
						lang, content,
						filename, line,
						srcIndex+srcIndexOffset-content.length,
						srcIndex+srcIndexOffset,
						commentMode,
						"commentMode: %s".format(commentMode)
					);
					break;

				case 4:
				_tokens ~=
					new Token_calc!(SymbolType.Terminal, "(")(
						lang, content,
						filename, line,
						srcIndex+srcIndexOffset-content.length,
						srcIndex+srcIndexOffset,
						commentMode,
						"commentMode: %s".format(commentMode)
					);
					break;

				case 5:
				_tokens ~=
					new Token_calc!(SymbolType.Terminal, ")")(
						lang, content,
						filename, line,
						srcIndex+srcIndexOffset-content.length,
						srcIndex+srcIndexOffset,
						commentMode,
						"commentMode: %s".format(commentMode)
					);
					break;

				case 6:
				_tokens ~=
					new Token_calc!(SymbolType.Terminal, "*")(
						lang, content,
						filename, line,
						srcIndex+srcIndexOffset-content.length,
						srcIndex+srcIndexOffset,
						commentMode,
						"commentMode: %s".format(commentMode)
					);
					break;

				case 7:
				_tokens ~=
					new Token_calc!(SymbolType.Terminal, "/")(
						lang, content,
						filename, line,
						srcIndex+srcIndexOffset-content.length,
						srcIndex+srcIndexOffset,
						commentMode,
						"commentMode: %s".format(commentMode)
					);
					break;

				case 8:
				_tokens ~=
					new Token_calc!(SymbolType.Terminal, "+")(
						lang, content,
						filename, line,
						srcIndex+srcIndexOffset-content.length,
						srcIndex+srcIndexOffset,
						commentMode,
						"commentMode: %s".format(commentMode)
					);
					break;

				case 9:
				_tokens ~=
					new Token_calc!(SymbolType.Terminal, "Number")(
						lang, content,
						filename, line,
						srcIndex+srcIndexOffset-content.length,
						srcIndex+srcIndexOffset,
						commentMode,
						"commentMode: %s".format(commentMode)
					);
					break;

				default:
					throw new InternalException("Accepting unexpected symbol #%s".format(symbol.id));
				}
			}
			
			// Merge adjacent error tokens
			if( _tokens.length >= 2 &&
			    _tokens[$-2].type == SymbolType.Error &&
			    _tokens[$-1].type == SymbolType.Error )
			{
				_tokens = _tokens[0..$-2] ~ mergeErrors(_tokens[$-2], _tokens[$-1]);
			}
				
			commentMode =
				(
					commentMode == CommentType.None &&
					symbol.type == SymbolType.CommentStart
				)? CommentType.Block :
				(
					commentMode == CommentType.None &&
					symbol.type == SymbolType.CommentLine
				)? CommentType.Line :
				(
					commentMode == CommentType.Block &&
					symbol.type == SymbolType.CommentEnd
				)? CommentType.None :
				commentMode;

			foundAcceptState = false;
		}
		
		//TODO: Unicode adds yet another EOL
		bool isNewline()
		{
			bool ret=false;
			if(srcIndex < source.length)
			{
				if(source[srcIndex] == '\n')  // Win and Unix style
					ret = true;
				else
				{
					if(srcIndex+1 < source.length)
					{
						// Old-Mac style
						if(source[srcIndex] == '\r' && source[srcIndex+1] != '\n')
							ret = true;
					}
				}
			}
			return ret;
		}
		
		_tokens.length = 0;
		foundAcceptState = false;
		lineIndicies.length = 0;
		lineIndicies ~= 0;
		uint lastAcceptIndexSize;
		
		if(source.length != 0)
		while(!eof)
		{
			dchar c;
			bool charAccepted = false;

			if(srcIndex == source.length)
				eof = true;

			if(!eof)
			{
				cSize = 0;
				
				bool invalidUtf = false;
				auto remainingSource = source[srcIndex..$];
				try
					c = decode(remainingSource, cSize);
				catch(UtfException e)
				{
					errorFound = true;
					invalidUtf = true;
				}
				if(invalidUtf)
					throw new InvalidUtfSequence(this);
					
				charAccepted = advanceState(c);
			}
			
			if(dfaTable[state].accept)
			{
				foundAcceptState = true;
				lastAcceptState  = state;
				lastAcceptIndex  = srcIndex;
				lastAcceptIndexSize = cSize;
			}
			
			bool shouldAdvance = true;
			if(!charAccepted)
			{
				if(!dfaTable[state].accept && foundAcceptState)
				{
					tok = tok[0..$+1-(srcIndex-lastAcceptIndex)];
					state    = lastAcceptState;
					srcIndex = lastAcceptIndex+lastAcceptIndexSize;
					line     = lineAtIndex(srcIndex);
					lineIndicies = lineIndicies[0..line+1];
				}

				if(dfaTable[state].accept)
				{
					auto symbolIndex = dfaTable[state].acceptSymbolIndex;
					addToken(symbolTable[symbolIndex], tok);
					shouldAdvance = false;
				}
				else
				{
					if(!eof)
						tok ~= source[srcIndex..srcIndex+cSize];//to!(string)(c~""d);
					addToken(lang.errorSymbol, tok);
				}

				state = lang.initialDFAState;
				tok = "";
			}
			
			if(shouldAdvance)
			{
				srcIndex += cSize;

				if(isNewline())
				{
					line++;
					
					//auto codePointSize = source[srcIndex..$].nextCodePointSize();
					//if(srcIndex+1 < source.length /*&& line == lineIndicies.length*/)
					lineIndicies ~= srcIndex + source[srcIndex..$].nextCodePointSize();
					
					assert(line == lineIndicies.length - 1);

					if(commentMode == CommentType.Line)
						commentMode = CommentType.None;
				}
			}
		}
		
		addToken(lang.eofSymbol(), "");
		_errors_cached = false;
		
		if(errorFound)
			throw new LexException(this);
		
		return _tokens;
	}
}
