[$name|Token$]
[$title|class Token$]
[$frame|HTMLPage.tmpl$]
[$desc|The main interface for processing parse trees$]

<ul>
<li>[$typeLink|CommentType| enum CommentType $]</li>
<li>[$typeLink|TokenToStringMode| enum TokenToStringMode $]</li>
<li>
	[$typeLink|Token| class Token $]
	<ul>
	<li>
		[$typeLink|Token_{languageName}| class Token_{languageName}$][$ident| : Token $]
		<ul>
		<li>
			[$typeLink|Token_{languageName}!{symbol}| [:class Token_{languageName}!{symbol}:]$][$ident| : Token_{languageName} $]
			<ul>
			<li>[$typeLink|Token_{languageName}!{rule}| [:class Token_{languageName}!{rule}:]$][$ident| : Token_{languageName}!{symbol} $]</li>
			</ul>
		</li>
		</ul>
	</li>
	</ul>
</li>
</ul>

<p>
This is the main interface for processing parse trees.
</p>

<p>
See the [$pageLink|APIOver#TokenSymbol| explanation of Tokens vs Symbols $].
</p>

[$apiSectionType| CommentType | goldie.token | enum CommentType $]
<div class="api-def">
	[$apiHead| None $]
	[$apiHead| Line $]
	[$apiHead| Block $]
</div>

[$apiSectionType| TokenToStringMode | goldie.token | enum TokenToStringMode $]
<div class="api-def">
	[$apiHead| Compact $]
	<div class="api-def">
		Omit all whitespace, error and comment tokens.
	</div>

	[$apiHead| CompactWithSpaces $]
	<div class="api-def">
		Like Compact, but adds a space between each token.
	</div>
	
	[$apiHead| default Smart $]
	<div class="api-def">
		Like Compact, but adds a space between two
		tokens whenever the last character of the first
		token and the first character of the second token
		are both either alphanumeric or an underscore.
	</div>

	[$apiHead| Full $]
	<div class="api-def">
	<p>
		Includes all whitespace, error and comment tokens.
	</p>
	<p>
		Note: Doesn't currently work after the parse phase because
		whitespace, error and comment tokens are not currently
		preserved by the parser.
	</p>
	</div>
</div>

[$apiSectionType| Token | goldie.token | class Token $]
<div class="api-def">
	[$apiHead|[: this(Symbol symbol, Token[] sub, Language lang, int ruleId) :]$]
	<div class="api-def">
		Constructor for nonterminals. Normally, only GoldieLib itself needs to
		instantiate tokens, unless you want to create/modify a parse tree or
		[$typeName|Token$] array manually.
	</div>
	
	[$apiHead|[: this(Symbol symbol, Language lang, string content, string file="{unknown}", int line=0, int srcIndexStart=0, int srcIndexEnd=0, CommentType commentMode=CommentType.None, string debugInfo="") :]$]
	<div class="api-def">
		Constructor for terminals. Normally, only GoldieLib itself needs to
		instantiate tokens, unless you want to create/modify a parse tree or
		[$typeName|Token$] array manually.
	</div>

	[$apiHead| readonly property Language lang $]
	<div class="api-def">
		The [$typeName|Language$] this [$typeName|Token$] is associated with.
	</div>

	[$apiHead|[: Token[] subX :]$]
	<div class="api-def">
		The sub-tokens of this [$typeName|Token$] (if this is a nonterminal).
	</div>

	[$apiHead| readonly property int ruleId $]
	<div class="api-def">
		If this Token is a nonterminal, then this is the ID of the
		[$link| http://www.devincook.com/goldparser/doc/grammars/define-rules.htm | reduction rule $]
		that was used to create the token. This ID is
		an index into [$typeName|Language$][$ident|.ruleTable$].
	</div>

	[$apiHead| readonly property SymbolType type $]
	<div class="api-def">
		The [$typeName|SymbolType$] of this [$typeName|Token$].
		See the explanation of
		[$pageLink|APIOver#TokenSymbol| Tokens, Symbols, and Symbol Types $]
		for more information.
	</div>

	[$apiHead| readonly property Symbol symbol $]
	<div class="api-def">
		The [$typeName|Symbol$] of this [$typeName|Token$].
		See the explanation of
		[$pageLink|APIOver#TokenSymbol| Tokens, Symbols, and Symbol Types $]
		for more information.
	</div>
	
	[$apiHead| readonly property string typeName $]
	<div class="api-def">
		The name of this [$typeName|Token$]'s [$typeName|SymbolType$].
	</div>
	
	[$apiHead| readonly property string name $]
	<div class="api-def">
		The name of this [$typeName|Token$]'s [$typeName|Symbol$].
	</div>
	
	[$apiHead| readonly property string fullName $]
	<div class="api-def">
		This is just [$code|inline|typeName ~ "." ~ name$].
	</div>

	[$apiHead| readonly property int id $]
	<div class="api-def">
		The id of this [$typeName|Token$]'s [$typeName|Symbol$].
	</div>

	[$apiHead| readonly property int line $]
	[$apiHead| readonly property string file $]
	<div class="api-def">
	<p>
		The file and line number of the original source where this
		[$typeName|Token$] starts.
		See Goldie's conventions relating to 
		[$pageLink|APIOver#LineColumn| Line and Column Numbers $].
	</p>
	<p>
		For the line number where this [$typeName|Token$] ends, or
		the column number where this [$typeName|Token$] starts or ends,
		use [$ident|srcIndexStart$] or [$ident|srcIndexStart$] together
		with [$typeName|Lexer$][$ident|.lineIndicies$]
		and [$typeName|Lexer$][$ident|.lineAtIndex$].
	</p>
	</div>

	[$apiHead| readonly property int srcIndexStart $]
	[$apiHead| readonly property int srcIndexEnd $]
	[$apiHead| readonly property int srcLength $]
	<div class="api-def">
		The locations (zero-indexed) in the original source where this
		[$typeName|Token$] starts and ends, and the difference between
		them.
	</div>

	[$apiHead| readonly property CommentType commentMode $]
	<div class="api-def">
		Indicates whether or not this token exists inside a comment and,
		if so, what type of comment.
	</div>

	[$apiHead| readonly property Token firstLeaf $]
	[$apiHead| readonly property Token lastLeaf $]
	<div class="api-def">
		The first and last terminals in this [$typeName|Token$].
		If this [$typeName|Token$] isn't a nonterminal, then these
		both just return [$code|inline|this$].
	</div>

	[$apiHead| readonly property string debugInfo $]
	<div class="api-def">
		A place for extra debugging information to be stored.
	</div>

	[$apiHead|[: bool matches(string parentSymbol, string[] subSymbols...) :]$]
	<div class="api-def">
	<p>
		Determine if the [$typeName|Token$] matches (ie, was created from) a particular
		[$link| http://www.devincook.com/goldparser/doc/grammars/define-rules.htm | reduction rule$].
	</p>
	<p>
		Example:
[$code|highlight|
// Did this token come from this reduction rule?
//   &lt;Add Exp&gt; ::= &lt;Add Exp&gt; '+' &lt;Mult Exp&gt;
bool checkToken(Token tok)
{
	return tok.matches("&lt;Add Exp&gt;", "&lt;Add Exp&gt;", "+", "&lt;Mult Exp&gt;");
}
$]
	</p>
	</div>
	
	[$apiHead| 
	string toString()
	string toString(TokenToStringMode mode)
	string toStringCompact()
	string toStringCompactWithSpaces()
	string toStringSmart()
	string toStringFull()
	$]
	<div class="api-def">
	<p>
		Converts the [$typeName|Token$] to a string that resembles the
		original source. See [$typeName|TokenToStringMode$] for descriptions
		of the different modes of conversion.
	</p>
	<p>
		<em>Note:</em>
		Depending on the language and the chosen mode of conversion, the
		result <em>might not be valid code</em> in the original language, or
		<em>may have subtly changed meaning</em>.
		Not all modes of conversion are suitable for all
		purposes or all languages. Depending on the language or purpose,
		it may be that none of these are appropriate and you'll have to
		create a string by walking the [$typeName|Token$] tree manually.
		These functions are merely provided as a convenience.
	</p>
	</div>

	
	[$apiHead| semitwist.treeout.TreeNode toTreeNode() $]
	<div class="api-def">
	<p>
		[$link| http://www.dsource.org/projects/semitwist/browser/trunk/src/semitwist/treeout.d | TreeNode $]
		is a type from
		[$link| http://www.dsource.org/projects/semitwist/ | SemiTwist D Tools $]
		that provides an easy way to convert a tree to a text format such as JSON or XML.
	</p>
	<p>
		Example: To convert a Token to JSON:
[$code|highlight|
import semitwist.treeout;

string tokenToJSON(Token tok, bool prettyPrint)
{
	if(prettyPrint)
		return tok.toTreeNode().format(formatterPrettyJSON);
	else
		return tok.toTreeNode().format(formatterTrimmedJSON);
}
$]
	</p>
	<p>
		Note, however, if you wish to use the resulting JSON in
		[$pageName|JsonViewer$], and get [$pageName|JsonViewer$]'s enhanced
		source-viewing features, then you'll need to
		[$pageLink|ParseTreeMode|add a few things$] to
		the returned [$ident|TreeNode$] before formatting it to a string.
		See the
		[$link| http://www.dsource.org/projects/goldie/browser/tags/v0.4/src/tools/parse | source $]
		of the [$pageName|Parse$] tool for an example.
	</p>
	</div>
</div>

[$apiSectionType| Token_{languageName} | {user-specified package}.token | class Token_{languageName} $]
<div class="api-def">
	<div class="api-head">
		{languageName} = Name of static-style language
	</div>
	<p></p>
	<p>
		This is the common base class for all tokens in a given
		static-style language.
	</p>
	[$apiHead|[: static enum string StringOf :]$]
	<div class="api-def">
	<p>
		This is a workaround for DMD Bug [$link| http://d.puremagic.com/issues/show_bug.cgi?id=1748 | #1748$].
	</p>
	<p>
		Evaluates to [$code|inline|Token_{languageName}$]. For example, if
		the language is named [$ident|foo$], then this evaluates to
		[$code|inline|Token_foo$]
	</p>
	</div>
</div>

[$apiSectionType| Token_{languageName}!{symbol} | {user-specified package}.token | [:class Token_{languageName}!{symbol} : Token_{languageName}:] $]
<div class="api-def">
	<div class="api-head">
		{languageName} = Name of static-style language
	</div>
	<div class="api-head">
		[:{symbol} = [ SymbolType <span class="param-name">staticSymbolType</span>, ] string <span class="param-name">staticName</span>=null:]
	</div>
	<p></p>
	<p>
		This type is for tokens representing a specific [$typeName|Symbol$]
		in a [$pageLink|StatVsDyn|static-style$] language.
	</p>
	<p>
		This is a templated type. Instantiation example:
[$code|highlight|
// Assume the language is named "calc"

// For a SymbolType.Terminal symbol named "Ident":
// These are the SAME type:
Token_calc!"Ident"
Token_calc!(SymbolType.Terminal, "Ident")

// For a SymbolType.NonTerminal symbol named "&lt;Add Exp&gt;"
// These are the SAME type (but different from the above types):
Token_calc!"&lt;Add Exp&gt;"
Token_calc!(SymbolType.NonTerminal, "&lt;Add Exp&gt;")

// All the above share common base-types: Token_calc and Token.
// This only shares a common base-type of Token
// (since it's from a different language).
Token_anotherCalc!"Ident"
$]
	</p>
	<p>
		The two-parameter form is needed if there are
		[$pageLink|AmbiguousSym| two Symbols with the same name $].
	</p>
	<p>
		Attempting to instantiate a [$typeName|Token_{languageName}!{symbol}$]
		with a symbol that doesn't exist in the language will result in a
		compile-time error.
	</p>
	[$apiHead|[: static enum string StringOf :]$]
	<div class="api-def">
	<p>
		This is a workaround for DMD Bug [$link| http://d.puremagic.com/issues/show_bug.cgi?id=1748 | #1748$].
	</p>
	<p>
		Evaluates to [$code|inline|Token_{languageName}!(SymbolType.{symbolType}, "{symbolName}")$].
	</p>
	<p>
		Example:
		[$code|highlight|
[:void showStringOf(Token_foo!"Ident" tok)
{
	// Output: Token_foo!(SymbolType.Terminal, "Ident")
	writeln(typeof(tok).StringOf);
}
:]$]
	</p>
	</div>
</div>

[$apiSectionType| Token_{languageName}!{rule} | {user-specified package}.token | [:class Token_{languageName}!{rule} : Token_{languageName}!{symbol}:] $]
<div class="api-def">
	<div class="api-head">
		{languageName} = Name of static-style language
	</div>
	<div class="api-head">
		[:{rule} = string <span class="param-name">staticName</span>, ( int <span class="param-name">staticRuleId</span> | <span class="param-name">subTokenTypes</span>... ):]
	</div>
	<p></p>
	<p>
		Nonterminals have one [$typeName|Token_{languageName}!{rule}$]
		for each rule that can create them.
	</p>
	<p>
		TODO: Refer to StatVsDyn:"Types and Inheritance"
		This is a templated type. Instantiation example:
[$code|highlight|
[:// Assume the language is named "calc"

// These three are all the SAME type, and
// are for a nonterminal Token created from
// this reduction rule:
//   &lt;Add Exp&gt; ::= &lt;Add Exp&gt; '+' &lt;Mult Exp&gt;
Token_calc!("&lt;Add Exp&gt;", "&lt;Add Exp&gt;", "+", "&lt;Mult Exp&gt;")
Token_calc!("&lt;Add Exp&gt;", "&lt;Add Exp&gt;", Token_calc!(SymbolType.Terminal, "+"), "&lt;Mult Exp&gt;")
Token_calc!("&lt;Add Exp&gt;", ruleIdOf_calc!("&lt;Add Exp&gt;", "&lt;Add Exp&gt;", "+", "&lt;Mult Exp&gt;"))

// This is a different type, but shares the common
// base-class of Token_calc!"&lt;Add Exp&gt;" with the above:
Token_calc!("&lt;Add Exp&gt;", "&lt;Mult Exp&gt;")

// This is another different type, but the only base-types this
// one shares with the above are Token_calc and Token (because
// it has a different reduction symbol, ie the first argument).
Token_calc!("&lt;Mult Exp&gt;", "&lt;Negate Exp&gt;")

// The only base-type this shares with any of the above is Token,
// since it's from a different language:
Token_anotherCalc!("&lt;Add Exp&gt;", "&lt;Mult Exp&gt;")

// Use null to refer to a rule that has no sub-tokens, such as in this:
//   &lt;OptionalHello&gt; ::= 'Hello'
//                     |
Token_foo!("&lt;OptionalHello&gt;", "Hello") // First rule
Token_foo!("&lt;OptionalHello&gt;", null)    // Second rule
:]$]
	</p>
	<p>
		See also the documentation on [$pageTitle|AmbiguousSym$].
	</p>
	<p>
		Attempting to instantiate a [$typeName|Token_{languageName}!{rule}$] with a
		rule that doesn't exist in the language will result in a compile-time
		error.
	</p>
	[$apiHead|[: this(Token[] sub, Language lang) :]$]
	<div class="api-def">
		Constructor. Normally, only GoldieLib itself needs to
		instantiate tokens, unless you want to create/modify a parse tree or
		[$typeName|Token$] array manually.
	</div>
	<!-- [$apiHead| readonly property Token_{languageName}!{symbol} sub(int index) $] -->
	<div class="api-head">
		<span class="attr">readonly property</span>
		[$typeName| Token_{languageName}!{symbol} $]
		<span class="ident">sub</span>(int <span class="param-name">index</span>)
		<br />
	</div>
	<div class="api-def">
	<p>
		Type-safe [$pageLink|StatVsDyn|static-style$] counterpart to
		[$typeName|Token$][$ident|.subX$].
	</p>
	<p>
		Sample usage: [$code|inline|myToken.sub!2$]
	</p>
	<p>
		Example:
[$code|highlight|
[:// Assume the language "calc":
// &lt;Mult Exp&gt;   ::= &lt;Mult Exp&gt; '*' &lt;Negate Exp&gt;
//                | &lt;Mult Exp&gt; '/' &lt;Negate Exp&gt;
//                | &lt;Negate Exp&gt;
// &lt;Negate Exp&gt; ::= '-' &lt;Value&gt;
//                | &lt;Value&gt;

void foo(Token_calc!("&lt;Mult Exp&gt;", "&lt;Mult Exp&gt;", "*", "&lt;Negate Exp&gt;") tok)
{
	// The third subtoken is known (even at compile-time) to be
	// a &lt;Negate Exp&gt;. The others are also known.
	// These are actually checked at compile-time.
	// If you get them mixed up, you'll get a type-mismatch error when compiling.
	Token_calc!"&lt;Negate Exp&gt;" negateTok = tok.sub!2;
	Token_calc!"&lt;Mult Exp&gt;" multTok = tok.sub!0;
	Token_calc!"*" operatorTok = tok.sub!1;

	// Determine exact type of the &lt;Negate Exp&gt; subtoken:
	// Can't know this at compile-time because it depends on
	// the actual code that was parsed.
	if( cast(Token_calc!("&lt;Negate Exp&gt;", "-", "&lt;Value&gt;")) negateTok )
	{
		writeln("negateTok came from: &lt;Negate Exp&gt; ::= '-' &lt;Value&gt;");
	}
	else if( cast(Token_calc!("&lt;Negate Exp&gt;", "&lt;Value&gt;")) negateTok )
	{
		writeln("negateTok came from: &lt;Negate Exp&gt; ::= &lt;Value&gt;");
	}
	else
		writeln("Forgot to handle some other rule!");
}
:]$]
	</p>
	</div>
	[$apiHead|[: static enum string StringOf :]$]
	<div class="api-def">
	<p>
		This is a workaround for DMD Bug [$link| http://d.puremagic.com/issues/show_bug.cgi?id=1748 | #1748$].
	</p>
	<p>
		Evaluates to [$code|inline|Token_{languageName}!(SymbolType.NonTerminal, "{symbolName}", ...)$].
	</p>
	<p>
		Example:
		[$code|highlight|
[:void showStringOf(Token_calc!("&lt;Negate Exp&gt;", "-", "&lt;Value&gt;") tok)
{
	// Output: Token_calc!(SymbolType.NonTerminal, "&lt;Negate Exp&gt;", ...)
	writeln(typeof(tok).StringOf);
}
:]$]
	</p>
	</div>
</div>
