{
	"0: data": {
		"srcIndexStart": "0", 
		"file": "docssrc/docs.APIRef.Lexer.tmpl", 
		"parseTreeMode": "true", 
		"srcLength": "3733", 
		"source": "[$name|Lexer$]
[$title|class Lexer$]
[$frame|HTMLPage.tmpl$]

<ul>
<li>
	[$typeLink|Lexer| class Lexer $]
	<ul>
	<li>[$typeLink|Lexer_{languageName}| class Lexer_{languageName}$][$ident| : Lexer $]</li>
	</ul>
</li>
</ul>

[$apiSectionType| Lexer | goldie.lexer | class Lexer $]
<div class=\"api-def\">
	[$apiHead|[: readonly @property Token[] tokens :]$]
	<div class=\"api-def\">
		The tokens that resulted from the last lexing.
	</div>
	[$apiHead|[: readonly @property LexError[] errors :]$]
	<div class=\"api-def\">
		The lexical errors encountered during the last lexing.
	</div>

	[$apiHead|[: string source :]$]
	<div class=\"api-def\">
		The original source that was last lexed.
	</div>

	[$apiHead|[: ptrdiff_t[] lineIndicies :]$]
	<div class=\"api-def\">
	<p>
		Indicies into [$ident|source$] where each line starts. For example,
		[$code|inline|[:lineIndicies[0]:]$] is the index where the first line
		starts (always 0), [$code|inline|[:lineIndicies[1]:]$] is the index
		where the second line starts, etc.
	</p>
	<p>
		See the example in [$ident|lineAtIndex$] below.
	</p>
	</div>

	[$apiHead|[: ptrdiff_t lineAtIndex(ptrdiff_t index) :]$]
	<div class=\"api-def\">
	<p>
		Returns the line number (zero-indexed) of a given index into the
		source that was lexed. As per Goldie
		[$pageLink|APIOver#LineColumn|conventions$], this value
		should be increased by one whenever it's displayed to the user.
	</p>
	<p>
		Example:
	</p>
		[$code|highlight|
[:string src =
\"ABC
DEF
HIJ\";

auto lexerUsed = someLanguage.lexCodeX(src);

auto numLines = lexerUsed.lineIndicies.length;

// Get everything from start of second line, to start of third line
auto secondLine = src[ lexerUsed.lineIndicies[1] .. lexerUsed.lineIndicies[2] ]

writeln(\"Number of lines: \", numLines);
writeln(\"Second line: \", secondLine);
writefln(\"src[9] is '%s' and is on line #%s\",
	src[9], 1 + lexerUsed.lineAtIndex(9));

// Output:
// Number of lines: 3
// Second line: DEF
// 
// src[9] is 'I' and is on line #3
:]$]
	</div>

	[$apiHead|[: string filename :]$]
	<div class=\"api-def\">
	<p>
		The filename (if available) of the last source lexed.
	</p>
	</div>

	[$apiHead|[: 
	ptrdiff_t srcIndex
	ptrdiff_t line
	:]$]
	<div class=\"api-def\">
	<p>
		Internal lexer state. Access is provided to these for diagnostic
		purposes.
		If an exception is thrown during lexing, then these can be inspected
		to check the lexer's internal state at the time of the exception.
		This is probably only useful for debugging Goldie.
	</p>
	</div>

	[$apiHead|[: Token[] process(string source, Language lang, string filename=\"\") :]$]
	<div class=\"api-def\">
	<p>
		Lexes (ie, tokenizes) a source. Throws a [$typeName|LexException$] if
		the source contains an error.
	</p>
	<p>
		Normally, you would use one of the
		[$typeName|Language$][$ident|.parse$] or [$typeName|Language$][$ident|.lex$]
		functions instead of calling this directly,
		but this direct access is provided in case you'd rather re-use the
		same lexer instead of instantiating a new one every time.
	</p>
	<p>
		The filename from which the source originated can be provided
		so error messages can report the appropriate filename.
	</p>
	</div>
</div>

[$apiSectionType| Lexer_{languageName} | {user-specified package}.lexer | class Lexer_{languageName} : Lexer $]
<div class=\"api-def\">
	<div class=\"api-head\">
		{languageName} = Name of static-style language
	</div>
	<p>
	This is the [$pageLink|StatVsDyn|static-style$] counterpart to
	[$pageName|Lexer$], and is generated by the
	[$pageName|StaticLang$] tool.
	</p>
	<p>
	If the name of a language is, for example, [$ident|foo$], then
	the name of this class will be [$ident|Lexer_foo$].
	</p>
	<p>
	The interface is the same as [$typeName|Lexer$].
	</p>
</div>
", 
		"0: name": {
			"srcIndexStart": "0", 
			"data": "Lexer", 
			"srcLength": "14", 
			"0: data": {
				"srcIndexStart": "7", 
				"data": "Lexer", 
				"srcLength": "5"
			}
		}, 
		"1: data": {
			"srcIndexStart": "14", 
			"data": "
", 
			"srcLength": "1"
		}, 
		"2: title": {
			"srcIndexStart": "15", 
			"data": "class Lexer", 
			"srcLength": "21", 
			"0: data": {
				"srcIndexStart": "23", 
				"data": "class Lexer", 
				"srcLength": "11"
			}
		}, 
		"3: data": {
			"srcIndexStart": "36", 
			"data": "
", 
			"srcLength": "1"
		}, 
		"4: frame": {
			"srcIndexStart": "37", 
			"data": "HTMLPage.tmpl", 
			"srcLength": "23", 
			"0: data": {
				"srcIndexStart": "45", 
				"data": "HTMLPage.tmpl", 
				"srcLength": "13"
			}
		}, 
		"5: data": {
			"srcIndexStart": "60", 
			"data": "

<ul>
<li>
	", 
			"srcLength": "13"
		}, 
		"6: typeLink": {
			"srcIndexStart": "73", 
			"param 0": "Lexer", 
			"data": "class Lexer", 
			"srcLength": "32", 
			"0: data": {
				"srcIndexStart": "90", 
				"data": "class Lexer", 
				"srcLength": "13"
			}
		}, 
		"7: data": {
			"srcIndexStart": "105", 
			"data": "
	<ul>
	<li>", 
			"srcLength": "12"
		}, 
		"8: typeLink": {
			"srcIndexStart": "117", 
			"param 0": "Lexer_{languageName}", 
			"data": "class Lexer_{languageName}", 
			"srcLength": "61", 
			"0: data": {
				"srcIndexStart": "149", 
				"data": "class Lexer_{languageName}", 
				"srcLength": "27"
			}
		}, 
		"9: ident": {
			"srcIndexStart": "178", 
			"data": ": Lexer", 
			"srcLength": "19", 
			"0: data": {
				"srcIndexStart": "186", 
				"data": ": Lexer", 
				"srcLength": "9"
			}
		}, 
		"10: data": {
			"srcIndexStart": "197", 
			"data": "</li>
	</ul>
</li>
</ul>

", 
			"srcLength": "26"
		}, 
		"11: apiSectionType": {
			"srcIndexStart": "223", 
			"param 0": "Lexer", 
			"param 1": "goldie.lexer", 
			"data": "class Lexer", 
			"srcLength": "55", 
			"0: data": {
				"srcIndexStart": "263", 
				"data": "class Lexer", 
				"srcLength": "13"
			}
		}, 
		"12: data": {
			"srcIndexStart": "278", 
			"data": "
<div class=\"api-def\">
	", 
			"srcLength": "24"
		}, 
		"13: apiHead": {
			"srcIndexStart": "302", 
			"data": " readonly @property Token[] tokens ", 
			"srcLength": "51", 
			"0: data": {
				"srcIndexStart": "312", 
				"data": " readonly @property Token[] tokens ", 
				"srcLength": "39"
			}
		}, 
		"14: data": {
			"srcIndexStart": "353", 
			"data": "
	<div class=\"api-def\">
		The tokens that resulted from the last lexing.
	</div>
	", 
			"srcLength": "82"
		}, 
		"15: apiHead": {
			"srcIndexStart": "435", 
			"data": " readonly @property LexError[] errors ", 
			"srcLength": "54", 
			"0: data": {
				"srcIndexStart": "445", 
				"data": " readonly @property LexError[] errors ", 
				"srcLength": "42"
			}
		}, 
		"16: data": {
			"srcIndexStart": "489", 
			"data": "
	<div class=\"api-def\">
		The lexical errors encountered during the last lexing.
	</div>

	", 
			"srcLength": "91"
		}, 
		"17: apiHead": {
			"srcIndexStart": "580", 
			"data": " string source ", 
			"srcLength": "31", 
			"0: data": {
				"srcIndexStart": "590", 
				"data": " string source ", 
				"srcLength": "19"
			}
		}, 
		"18: data": {
			"srcIndexStart": "611", 
			"data": "
	<div class=\"api-def\">
		The original source that was last lexed.
	</div>

	", 
			"srcLength": "77"
		}, 
		"19: apiHead": {
			"srcIndexStart": "688", 
			"data": " ptrdiff_t[] lineIndicies ", 
			"srcLength": "42", 
			"0: data": {
				"srcIndexStart": "698", 
				"data": " ptrdiff_t[] lineIndicies ", 
				"srcLength": "30"
			}
		}, 
		"20: data": {
			"srcIndexStart": "730", 
			"data": "
	<div class=\"api-def\">
	<p>
		Indicies into ", 
			"srcLength": "45"
		}, 
		"21: ident": {
			"srcIndexStart": "775", 
			"data": "source", 
			"srcLength": "16", 
			"0: data": {
				"srcIndexStart": "783", 
				"data": "source", 
				"srcLength": "6"
			}
		}, 
		"22: data": {
			"srcIndexStart": "791", 
			"data": " where each line starts. For example,
		", 
			"srcLength": "40"
		}, 
		"23: code": {
			"srcIndexStart": "831", 
			"param 0": "inline", 
			"data": "lineIndicies[0]", 
			"srcLength": "35", 
			"0: data": {
				"srcIndexStart": "845", 
				"data": "lineIndicies[0]", 
				"srcLength": "19"
			}
		}, 
		"24: data": {
			"srcIndexStart": "866", 
			"data": " is the index where the first line
		starts (always 0), ", 
			"srcLength": "56"
		}, 
		"25: code": {
			"srcIndexStart": "922", 
			"param 0": "inline", 
			"data": "lineIndicies[1]", 
			"srcLength": "35", 
			"0: data": {
				"srcIndexStart": "936", 
				"data": "lineIndicies[1]", 
				"srcLength": "19"
			}
		}, 
		"26: data": {
			"srcIndexStart": "957", 
			"data": " is the index
		where the second line starts, etc.
	</p>
	<p>
		See the example in ", 
			"srcLength": "83"
		}, 
		"27: ident": {
			"srcIndexStart": "1040", 
			"data": "lineAtIndex", 
			"srcLength": "21", 
			"0: data": {
				"srcIndexStart": "1048", 
				"data": "lineAtIndex", 
				"srcLength": "11"
			}
		}, 
		"28: data": {
			"srcIndexStart": "1061", 
			"data": " below.
	</p>
	</div>

	", 
			"srcLength": "24"
		}, 
		"29: apiHead": {
			"srcIndexStart": "1085", 
			"data": " ptrdiff_t lineAtIndex(ptrdiff_t index) ", 
			"srcLength": "56", 
			"0: data": {
				"srcIndexStart": "1095", 
				"data": " ptrdiff_t lineAtIndex(ptrdiff_t index) ", 
				"srcLength": "44"
			}
		}, 
		"30: data": {
			"srcIndexStart": "1141", 
			"data": "
	<div class=\"api-def\">
	<p>
		Returns the line number (zero-indexed) of a given index into the
		source that was lexed. As per Goldie
		", 
			"srcLength": "137"
		}, 
		"31: pageLink": {
			"srcIndexStart": "1278", 
			"param 0": "APIOver#LineColumn", 
			"data": "conventions", 
			"srcLength": "43", 
			"0: data": {
				"srcIndexStart": "1308", 
				"data": "conventions", 
				"srcLength": "11"
			}
		}, 
		"32: data": {
			"srcIndexStart": "1321", 
			"data": ", this value
		should be increased by one whenever it's displayed to the user.
	</p>
	<p>
		Example:
	</p>
		", 
			"srcLength": "109"
		}, 
		"33: code": {
			"srcIndexStart": "1430", 
			"param 0": "highlight", 
			"data": "string src =
\"ABC
DEF
HIJ\";

auto lexerUsed = someLanguage.lexCodeX(src);

auto numLines = lexerUsed.lineIndicies.length;

<span class=\"code-comment\">// Get everything from start of second line, to start of third line</span>
auto secondLine = src[ lexerUsed.lineIndicies[1] .. lexerUsed.lineIndicies[2] ]

writeln(\"Number of lines: \", numLines);
writeln(\"Second line: \", secondLine);
writefln(\"src[9] is '%s' and is on line #%s\",
    src[9], 1 + lexerUsed.lineAtIndex(9));

<span class=\"code-comment\">// Output:</span>
<span class=\"code-comment\">// Number of lines: 3</span>
<span class=\"code-comment\">// Second line: DEF</span>
<span class=\"code-comment\">//</span> 
<span class=\"code-comment\">// src[9] is 'I' and is on line #3</span>
", 
			"srcLength": "553", 
			"0: data": {
				"srcIndexStart": "1447", 
				"data": "", 
				"srcLength": "1"
			}, 
			"1: data": {
				"srcIndexStart": "1448", 
				"data": "string src =
\"ABC
DEF
HIJ\";

auto lexerUsed = someLanguage.lexCodeX(src);

auto numLines = lexerUsed.lineIndicies.length;

// Get everything from start of second line, to start of third line
auto secondLine = src[ lexerUsed.lineIndicies[1] .. lexerUsed.lineIndicies[2] ]

writeln(\"Number of lines: \", numLines);
writeln(\"Second line: \", secondLine);
writefln(\"src[9] is '%s' and is on line #%s\",
	src[9], 1 + lexerUsed.lineAtIndex(9));

// Output:
// Number of lines: 3
// Second line: DEF
// 
// src[9] is 'I' and is on line #3
", 
				"srcLength": "533"
			}
		}, 
		"34: data": {
			"srcIndexStart": "1983", 
			"data": "
	</div>

	", 
			"srcLength": "11"
		}, 
		"35: apiHead": {
			"srcIndexStart": "1994", 
			"data": " string filename ", 
			"srcLength": "33", 
			"0: data": {
				"srcIndexStart": "2004", 
				"data": " string filename ", 
				"srcLength": "21"
			}
		}, 
		"36: data": {
			"srcIndexStart": "2027", 
			"data": "
	<div class=\"api-def\">
	<p>
		The filename (if available) of the last source lexed.
	</p>
	</div>

	", 
			"srcLength": "101"
		}, 
		"37: apiHead": {
			"srcIndexStart": "2128", 
			"data": " 
	ptrdiff_t srcIndex
	ptrdiff_t line
	", 
			"srcLength": "55", 
			"0: data": {
				"srcIndexStart": "2138", 
				"data": " 
	ptrdiff_t srcIndex
	ptrdiff_t line
	", 
				"srcLength": "43"
			}
		}, 
		"38: data": {
			"srcIndexStart": "2183", 
			"data": "
	<div class=\"api-def\">
	<p>
		Internal lexer state. Access is provided to these for diagnostic
		purposes.
		If an exception is thrown during lexing, then these can be inspected
		to check the lexer's internal state at the time of the exception.
		This is probably only useful for debugging Goldie.
	</p>
	</div>

	", 
			"srcLength": "316"
		}, 
		"39: apiHead": {
			"srcIndexStart": "2499", 
			"data": " Token[] process(string source, Language lang, string filename=\"\") ", 
			"srcLength": "83", 
			"0: data": {
				"srcIndexStart": "2509", 
				"data": " Token[] process(string source, Language lang, string filename=\"\") ", 
				"srcLength": "71"
			}
		}, 
		"40: data": {
			"srcIndexStart": "2582", 
			"data": "
	<div class=\"api-def\">
	<p>
		Lexes (ie, tokenizes) a source. Throws a ", 
			"srcLength": "72"
		}, 
		"41: typeName": {
			"srcIndexStart": "2654", 
			"data": "LexException", 
			"srcLength": "25", 
			"0: data": {
				"srcIndexStart": "2665", 
				"data": "LexException", 
				"srcLength": "12"
			}
		}, 
		"42: data": {
			"srcIndexStart": "2679", 
			"data": " if
		the source contains an error.
	</p>
	<p>
		Normally, you would use one of the
		", 
			"srcLength": "86"
		}, 
		"43: typeName": {
			"srcIndexStart": "2765", 
			"data": "Language", 
			"srcLength": "21", 
			"0: data": {
				"srcIndexStart": "2776", 
				"data": "Language", 
				"srcLength": "8"
			}
		}, 
		"44: ident": {
			"srcIndexStart": "2786", 
			"data": ".parse", 
			"srcLength": "16", 
			"0: data": {
				"srcIndexStart": "2794", 
				"data": ".parse", 
				"srcLength": "6"
			}
		}, 
		"45: data": {
			"srcIndexStart": "2802", 
			"data": " or ", 
			"srcLength": "4"
		}, 
		"46: typeName": {
			"srcIndexStart": "2806", 
			"data": "Language", 
			"srcLength": "21", 
			"0: data": {
				"srcIndexStart": "2817", 
				"data": "Language", 
				"srcLength": "8"
			}
		}, 
		"47: ident": {
			"srcIndexStart": "2827", 
			"data": ".lex", 
			"srcLength": "14", 
			"0: data": {
				"srcIndexStart": "2835", 
				"data": ".lex", 
				"srcLength": "4"
			}
		}, 
		"48: data": {
			"srcIndexStart": "2841", 
			"data": "
		functions instead of calling this directly,
		but this direct access is provided in case you'd rather re-use the
		same lexer instead of instantiating a new one every time.
	</p>
	<p>
		The filename from which the source originated can be provided
		so error messages can report the appropriate filename.
	</p>
	</div>
</div>

", 
			"srcLength": "330"
		}, 
		"49: apiSectionType": {
			"srcIndexStart": "3171", 
			"param 0": "Lexer_{languageName}", 
			"param 1": "{user-specified package}.lexer", 
			"data": "class Lexer_{languageName} : Lexer", 
			"srcLength": "111", 
			"0: data": {
				"srcIndexStart": "3244", 
				"data": "class Lexer_{languageName} : Lexer", 
				"srcLength": "36"
			}
		}, 
		"50: data": {
			"srcIndexStart": "3282", 
			"data": "
<div class=\"api-def\">
	<div class=\"api-head\">
		{languageName} = Name of static-style language
	</div>
	<p>
	This is the ", 
			"srcLength": "122"
		}, 
		"51: pageLink": {
			"srcIndexStart": "3404", 
			"param 0": "StatVsDyn", 
			"data": "static-style", 
			"srcLength": "35", 
			"0: data": {
				"srcIndexStart": "3425", 
				"data": "static-style", 
				"srcLength": "12"
			}
		}, 
		"52: data": {
			"srcIndexStart": "3439", 
			"data": " counterpart to
	", 
			"srcLength": "17"
		}, 
		"53: pageName": {
			"srcIndexStart": "3456", 
			"data": "Lexer", 
			"srcLength": "18", 
			"0: data": {
				"srcIndexStart": "3467", 
				"data": "Lexer", 
				"srcLength": "5"
			}
		}, 
		"54: data": {
			"srcIndexStart": "3474", 
			"data": ", and is generated by the
	", 
			"srcLength": "27"
		}, 
		"55: pageName": {
			"srcIndexStart": "3501", 
			"data": "StaticLang", 
			"srcLength": "23", 
			"0: data": {
				"srcIndexStart": "3512", 
				"data": "StaticLang", 
				"srcLength": "10"
			}
		}, 
		"56: data": {
			"srcIndexStart": "3524", 
			"data": " tool.
	</p>
	<p>
	If the name of a language is, for example, ", 
			"srcLength": "62"
		}, 
		"57: ident": {
			"srcIndexStart": "3586", 
			"data": "foo", 
			"srcLength": "13", 
			"0: data": {
				"srcIndexStart": "3594", 
				"data": "foo", 
				"srcLength": "3"
			}
		}, 
		"58: data": {
			"srcIndexStart": "3599", 
			"data": ", then
	the name of this class will be ", 
			"srcLength": "39"
		}, 
		"59: ident": {
			"srcIndexStart": "3638", 
			"data": "Lexer_foo", 
			"srcLength": "19", 
			"0: data": {
				"srcIndexStart": "3646", 
				"data": "Lexer_foo", 
				"srcLength": "9"
			}
		}, 
		"60: data": {
			"srcIndexStart": "3657", 
			"data": ".
	</p>
	<p>
	The interface is the same as ", 
			"srcLength": "43"
		}, 
		"61: typeName": {
			"srcIndexStart": "3700", 
			"data": "Lexer", 
			"srcLength": "18", 
			"0: data": {
				"srcIndexStart": "3711", 
				"data": "Lexer", 
				"srcLength": "5"
			}
		}, 
		"62: data": {
			"srcIndexStart": "3718", 
			"data": ".
	</p>
</div>", 
			"srcLength": "15"
		}
	}
}