// Goldie: GOLD Engine for D
// Written in the D programming language.

module goldie.lexer;

import tango.core.Array;
import tango.io.Stdout;
import tango.text.Regex;
import util = tango.text.Util;
import tango.util.Convert;

import semitwist.util.all;

import goldie.base;
import goldie.lang;
import goldie.file;
import goldie.parser;

private alias char schar; // Source Char

//TODO? Generate recording of steps taken
class GoldLexer
{
	// For debugging
	public schar[] remainingData;
	public schar[] processedData;
	public schar[] mixedData;
	
	public char[] source;
	mixin(getter!(TokGold[], "tokens"));

	mixin(getterLazy!(uint, "numErrors"));
	private uint _numErrors_gen()
	{
		return
			_tokens.reduceTo(
				(uint count, TokGold tok)
				{
					return
						(tok.type == GoldSymbolType.Error &&
						tok.commentMode == GoldCommentType.None)?
						count+1 : count;
				}
			);
	}
	
	mixin(getterLazy!(char[], "errorMsg"));
	private char[] _errorMsg_gen()
	{
		char[] value;
		
		foreach(TokGold tok; _tokens)
		{
			if(tok.type == GoldSymbolType.Error && tok.commentMode == GoldCommentType.None)
			{
				value ~=
					"{}({}:{}): Syntax Error: '{}'".sformatln(
						tok.file, tok.line,
						(tok.srcIndexStart-lineIndicies[tok.line-1])+1,
						tok.toStringRaw().stripNonPrintable()
					);
			}
		}
		
		return value;
	}

	public uint[] lineIndicies;

	// For debugging
	private void AppendRemainingData(T)(T data)
	{
/+		remainingData ~= data;
		mixedData ~= data;
+/	}
	private void AppendProcessedData(T)(T data)
	{
/+		processedData ~= data;
		mixedData ~= data;
+/	}
	
	// Returned value is 1-indexed
	public uint lineAtIndex(uint index)
	{
		foreach_reverse(uint line, uint lineStart; lineIndicies)
		{
			if(lineStart < index)
				return (line+1)+1; // Next line, and 1-indexed
		}
		return lineIndicies.length; // Last line, and 1-indexed
	}
	
	//TODO? Make comment tok instead of error tok if error occurs in comment mode
	//TODO: Test EOF during comment
	public TokGold[] process(schar[] source, GoldLanguage lang, char[] filename="")
	{
		this.source = source;

		schar[] tok;
		
		uint srcIndex=0;
		uint line=1; // 1-indexed //TODO? convert to 0-indexed (while displayed output is still 1-indexed)
		int  state = lang.initialDFAState;
		bool eof = false;
		
		auto commentMode = GoldCommentType.None;
		auto commentModeOfErr = GoldCommentType.None;
		
		bool foundAcceptState;
		typeof(state)    lastAcceptState;
		typeof(srcIndex) lastAcceptIndex;
		
		TokGold mergeErrors(TokGold tok1, TokGold tok2)
		in
		{
			mixin(deferAssert!(`tok1.type == GoldSymbolType.Error`));
			mixin(deferAssert!(`tok2.type == GoldSymbolType.Error`));
			mixin(deferAssert!(`tok1.id == lang.errorSymbol.id`));
			mixin(deferAssert!(`tok2.id == lang.errorSymbol.id`));
			mixin(deferAssert!(`tok1.srcIndexStart < tok2.srcIndexStart`));
			mixin(deferAssert!(`tok1.srcIndexEnd < tok2.srcIndexEnd`));
			mixin(deferAssert!(`tok1.srcIndexEnd == tok2.srcIndexStart`));
			flushAsserts();
		}
		body
		{
			return new TokGold(lang.errorSymbol,
			                   tok1.toStringRaw ~ tok2.toStringRaw,
							   tok1.file, tok1.line,
							   tok1.srcIndexStart,
							   tok2.srcIndexEnd,
							   tok1.commentMode,
							   tok1.debugInfo);
		}

		bool advanceState(schar c)
		{
			bool charAccepted = false;
			foreach(GoldDFAStateEdge edge; lang.dfaTable[state].edges)
			{
				if(lang.characterSetTable[edge.charSetIndex].chars.contains(c))
				{
					state = edge.targetDFAStateIndex;
					charAccepted = true;
					tok ~= c;
					break;
				}
			}
			return charAccepted;
		}
		
		void addToken(GoldSymbol symbol, schar[] content)
		{
			int srcIndexOffset;
			if(symbol.type == GoldSymbolType.Error)
			{
				if(commentMode == GoldCommentType.None)
				{
					// Unfortunately, can't use accessor "numErrors" with ++
					_numErrors++;
					AppendRemainingData(content);
				}

				srcIndexOffset = 1;
			}
			else
			{
				srcIndexOffset = 0;
				if(symbol.type == GoldSymbolType.Whitespace)
					AppendProcessedData("{}".sformat(content));
				else
					AppendProcessedData("[:{}:]".sformat(content));
			}

			_tokens ~= new TokGold(symbol, content,
			                       filename, line,
			                       srcIndex+srcIndexOffset-content.length,
			                       srcIndex+srcIndexOffset,
			                       commentMode,
			                       "commentMode: {}".sformat(commentMode));
			
			// Merge adjacent error tokens
			if( _tokens.length >= 2 &&
			    _tokens[$-2].type == GoldSymbolType.Error &&
			    _tokens[$-1].type == GoldSymbolType.Error )
			{
				_tokens = _tokens[0..$-2] ~ mergeErrors(_tokens[$-2], _tokens[$-1]);
			}
				
			commentMode =
				(
					commentMode == GoldCommentType.None &&
					symbol.type == GoldSymbolType.CommentStart
				)? GoldCommentType.Block :
				(
					commentMode == GoldCommentType.None &&
					symbol.type == GoldSymbolType.CommentLine
				)? GoldCommentType.Line :
				(
					commentMode == GoldCommentType.Block &&
					symbol.type == GoldSymbolType.CommentEnd
				)? GoldCommentType.None :
				commentMode;

			foundAcceptState = false;
		}
		
		bool isNewline()
		{
			bool ret=false;
			if(srcIndex < source.length)
			{
				if(source[srcIndex] == '\n')  // Win and Unix style
					ret = true;
				else
				{
					if(srcIndex+1 < source.length)
					{
						// Mac style
						if(source[srcIndex] == '\r' && source[srcIndex+1] != '\n')
							ret = true;
					}
				}
			}
			return ret;
		}
		
		remainingData = "";
		processedData = "";
		mixedData = "";
		
		_tokens.length = 0;
		foundAcceptState = false;
		lineIndicies.length = 0;
		lineIndicies ~= 0;
		if(source.length != 0)
		while(!eof)
		{
			schar c;
			bool charAccepted = false;

			if(srcIndex == source.length)
				eof = true;
				
			if(!eof)
			{
				c = source[srcIndex];
				charAccepted = advanceState(c);
			}
			
			//TODO: When newline is detected, ONLY append to lineIndicies
			//      if we're already at the last line already in there.
			if(lang.dfaTable[state].accept)
			{
				foundAcceptState = true;
				lastAcceptState  = state;
				lastAcceptIndex  = srcIndex;
			}
			
			bool shouldAdvance = true;
			if(!charAccepted)
			{
				if(!lang.dfaTable[state].accept && foundAcceptState)
				{
					tok = tok[0..$+1-(srcIndex-lastAcceptIndex)];
					state    = lastAcceptState;
					srcIndex = lastAcceptIndex+1;
					line     = lineAtIndex(srcIndex);
				}

				if(lang.dfaTable[state].accept)
				{
					auto symbolIndex = lang.dfaTable[state].acceptSymbolIndex;
					addToken(lang.symbolTable[symbolIndex], tok);
					shouldAdvance = false;
				}
				else
				{
					tok ~= c;
					addToken(lang.errorSymbol, tok);
				}

				state = lang.initialDFAState;
				tok = "";
			}
			
			if(shouldAdvance)
			{
				srcIndex++;
					
				if(isNewline())
				{
					line++;
					
					if(srcIndex+1 < source.length && line-1 == lineIndicies.length)
						lineIndicies ~= srcIndex+1;
					
					if(commentMode == GoldCommentType.Line)
						commentMode = GoldCommentType.None;
				}
			}
		}
		
		addToken(lang.eofSymbol(), "");
		_errorMsg_cached = false;
		_numErrors_cached = false;
		return _tokens;
	}
}
