lex.dd

Ddoc

$(SPEC_S Lexical,

	The lexical analysis is independent of the syntax parsing and the
	semantic analysis. The lexical analyzer splits the source text up into
	tokens. The lexical grammar describes the syntax of those tokens. The
	grammar is designed to be suitable for high speed scanning and to make it
	easy to write a correct scanner for it. It has a minimum of special case
	rules and there is only one phase of translation. The tokens are readily
	recognizable by those familiar with C and C++.

$(H3 $(LNAME2 source_text, Source Text))

	D source text can be in one of the following formats:

	$(LIST
	ASCII,
	UTF-8,
	UTF-16BE,
	UTF-16LE,
	UTF-32BE,
	UTF-32LE
	)

	UTF-8 is a superset of traditional 7-bit ASCII.
	One of the
	following UTF BOMs (Byte Order Marks) can be present at the beginning
	of the source text:
    $(P )

    $(TABLE2 UTF Byte Order Marks,
    $(THEAD Format, BOM)
	$(TROW UTF-8, EF BB BF)
	$(TROW UTF-16BE, FE FF)
	$(TROW UTF-16LE, FF FE)
    $(TROW UTF-32BE, 00 00 FE FF)
	$(TROW UTF-32LE, FF FE 00 00)
	$(TROW ASCII, no BOM)
    )

	$(P If the source file does not start with a BOM, then the first
	character must be less than or equal to U+0000007F.)

	$(P There are no digraphs or trigraphs in D.)

	$(P The source text is decoded from its source representation
	into Unicode $(GLINK Character)s.
	The $(GLINK Character)s are further divided into:
	$(GLINK WhiteSpace),
	$(GLINK EndOfLine),
	$(GLINK Comment)s,
	$(GLINK SpecialTokenSequence)s,
	$(GLINK Token)s,
	all followed by $(GLINK EndOfFile).
	)

	$(P The source text is split into tokens using the maximal munch
	technique, i.e., the
	lexical analyzer tries to make the longest token it can. For example
	$(D >>) is a right shift token,
	not two greater than tokens. There are two exceptions to this rule:
	$(UL
		$(LI A $(D ..) embedded inside what looks like two floating point
		literals, as in $(D 1..2), is interpreted as if the $(D ..) was
		separated by a space from the first integer.)
		$(LI A $(D 1.a) is interpreted as the three tokens $(D 1), $(D .), and $(D a),
		while $(D 1. a) is interpreted as the two tokens $(D 1.) and $(D a))
	)
	)

$(H3 $(LNAME2 character_set, Character Set))

$(GRAMMAR
$(GNAME Character):
    $(I any Unicode character)
)

$(H3 $(LNAME2 end_of_file, End of File))

$(GRAMMAR
$(GNAME EndOfFile):
    $(I physical end of the file)
    $(D \u0000)
    $(D \u001A))

The source text is terminated by whichever comes first.

$(H3 $(LNAME2 end_of_line, End of Line))

$(GRAMMAR
$(GNAME EndOfLine):
    $(D \u000D)
    $(D \u000A)
    $(D \u000D \u000A)
    $(D \u2028)
    $(D \u2029)
    $(GLINK EndOfFile))

	There is no backslash line splicing, nor are there any limits
	on the length of a line.

$(H3 $(LNAME2 white_space, White Space))

$(GRAMMAR
$(GNAME WhiteSpace):
    $(GLINK Space)
    $(GLINK Space) $(I WhiteSpace)

$(GNAME Space):
    $(D \u0020)
    $(D \u0009)
    $(D \u000B)
    $(D \u000C)
)


$(H3 $(LNAME2 comment, Comments))

$(GRAMMAR
$(GNAME Comment):
    $(GLINK BlockComment)
    $(GLINK LineComment)
    $(GLINK NestingBlockComment)

$(GNAME BlockComment):
    $(B /*) $(GLINK Characters) $(B */)

$(GNAME LineComment):
    $(B //) $(GLINK Characters) $(GLINK EndOfLine)

$(GNAME NestingBlockComment):
    $(B /+) $(GLINK NestingBlockCommentCharacters) $(B +/)

$(GNAME NestingBlockCommentCharacters):
    $(GLINK NestingBlockCommentCharacter)
    $(GLINK NestingBlockCommentCharacter) $(I NestingBlockCommentCharacters)

$(GNAME NestingBlockCommentCharacter):
    $(GLINK Character)
    $(GLINK NestingBlockComment)

$(GNAME Characters):
    $(GLINK Character)
    $(GLINK Character) $(I Characters))

    $(P D has three kinds of comments:)

	$(OL
	$(LI Block comments can span multiple lines, but do not nest.)
	$(LI Line comments terminate at the end of the line.)
	$(LI Nesting block comments can span multiple lines and can nest.)
	)

	$(P
	The contents of strings and comments are not tokenized.  Consequently,
	comment openings occurring within a string do not begin a comment, and
	string delimiters within a comment do not affect the recognition of
	comment closings and nested "/+" comment openings.  With the exception
	of "/+" occurring within a "/+" comment, comment openings within a
	comment are ignored.
	)

-------------
a = /+ // +/ 1;    // parses as if 'a = 1;'
a = /+ "+/" +/ 1"; // parses as if 'a = " +/ 1";'
a = /+ /* +/ */ 3; // parses as if 'a = */ 3;'
-------------

	Comments cannot be used as token concatenators, for example,
	$(D abc/**/def) is two tokens, $(D abc) and $(D def),
	not one $(D abcdef) token.

$(H3 $(LNAME2 tokens, Tokens))

$(GRAMMAR
$(GNAME Token):
$(MULTICOLS 4,     $(GLINK Identifier)
    $(GLINK StringLiteral)
    $(GLINK CharacterLiteral)
    $(GLINK IntegerLiteral)
    $(GLINK FloatLiteral)
    $(GLINK Keyword)
    $(D /)
    $(D /=)
    $(D .)
    $(D ..)
    $(D ...)
    $(CODE_AMP)
    $(CODE_AMP)$(D =)
    $(CODE_AMP)$(CODE_AMP)
    $(D |)
    $(D |=)
    $(D ||)
    $(D -)
    $(D -=)
    $(D --)
    $(D +)
    $(D +=)
    $(D ++)
    $(D <)
    $(D <)$(D =)
    $(D <)$(D <)
    $(D <)$(D <)$(D =)
    $(D <)$(D >)
    $(D <)$(D >)$(D =)
    $(D >)
    $(D >)$(D =)
    $(D >)$(D >)$(D =)
    $(D >)$(D >)$(D >)$(D =)
    $(D >)$(D >)
    $(D >)$(D >)$(D >)
    $(D !)
    $(D !=)
    $(D !<>)
    $(D !<>=)
    $(D !<)
    $(D !<=)
    $(D !>)
    $(D !>=)
    $(D $(LPAREN))
    $(D $(RPAREN))
    $(D [)
    $(D ])
    $(CODE_LCURL)
    $(CODE_RCURL)
    $(D ?)
    $(D ,)
    $(D ;)
    $(D :)
    $(D $)
    $(D =)
    $(D ==)
    $(D *)
    $(D *=)
    $(CODE_PERCENT)
    $(CODE_PERCENT)$(D =)
    $(D ^)
    $(D ^=)
    $(D ^^)
    $(D ^^=)
    $(D ~)
    $(D ~=)
    $(D @)
    $(D =>)
    $(D #)
)
)

$(H3 $(LNAME2 identifiers, Identifiers))

$(GRAMMAR
$(GNAME Identifier):
    $(GLINK IdentifierStart)
    $(GLINK IdentifierStart) $(GLINK IdentifierChars)

$(GNAME IdentifierChars):
    $(GLINK IdentifierChar)
    $(GLINK IdentifierChar) $(I IdentifierChars)

$(GNAME IdentifierStart):
    $(D _)
    $(I Letter)
    $(I UniversalAlpha)

$(GNAME IdentifierChar):
    $(GLINK IdentifierStart)
    $(B 0)
    $(GLINK NonZeroDigit)
)


	Identifiers start with a letter, $(D _), or universal alpha,
	and are followed by any number
	of letters, $(D _), digits, or universal alphas.
	Universal alphas are as defined in ISO/IEC 9899:1999(E) Appendix D.
	(This is the C99 Standard.)
	Identifiers can be arbitrarily long, and are case sensitive.
	Identifiers starting with $(D __) (two underscores) are reserved.

$(H3 $(LNAME2 string_literals, String Literals))

$(GRAMMAR
$(GNAME StringLiteral):
    $(GLINK WysiwygString)
    $(GLINK AlternateWysiwygString)
    $(GLINK DoubleQuotedString)

    $(GLINK HexString)
    $(GLINK DelimitedString)
    $(GLINK TokenString)

$(GNAME WysiwygString):
    $(B r") $(GLINK WysiwygCharacters) $(B ") $(GLINK StringPostfix)$(OPT)

$(GNAME AlternateWysiwygString):
    $(B $(BACKTICK)) $(GLINK WysiwygCharacters) $(B $(BACKTICK)) $(GLINK StringPostfix)$(OPT)

$(GNAME WysiwygCharacters):
    $(GLINK WysiwygCharacter)
    $(GLINK WysiwygCharacter) $(I WysiwygCharacters)

$(GNAME WysiwygCharacter):
    $(GLINK Character)
    $(GLINK EndOfLine)

$(GNAME DoubleQuotedString):
    $(B ") $(GLINK DoubleQuotedCharacters) $(B ") $(GLINK StringPostfix)$(OPT)

$(GNAME DoubleQuotedCharacters):
    $(GLINK DoubleQuotedCharacter)
    $(GLINK DoubleQuotedCharacter) $(I DoubleQuotedCharacters)

$(GNAME DoubleQuotedCharacter):
    $(GLINK Character)
    $(GLINK EscapeSequence)
    $(GLINK EndOfLine)

$(GNAME EscapeSequence):
    $(D \')
    $(D \")
    $(D \?)
    $(D \\)
    $(D \0)
    $(D \a)
    $(D \b)
    $(D \f)
    $(D \n)
    $(D \r)
    $(D \t)
    $(D \v)
    $(D \x) $(GLINK HexDigit) $(GLINK HexDigit)
    $(D \) $(GLINK OctalDigit)
    $(D \) $(GLINK OctalDigit) $(GLINK OctalDigit)
    $(D \) $(GLINK OctalDigit) $(GLINK OctalDigit) $(GLINK OctalDigit)
    $(D \u) $(GLINK HexDigit) $(GLINK HexDigit) $(GLINK HexDigit) $(GLINK HexDigit)
    $(D \U) $(GLINK HexDigit) $(GLINK HexDigit) $(GLINK HexDigit) $(GLINK HexDigit) $(GLINK HexDigit) $(GLINK HexDigit) $(GLINK HexDigit) $(GLINK HexDigit)
    $(D \) $(GLINK2 entity, NamedCharacterEntity)

$(GNAME HexString):
    $(D x") $(GLINK HexStringChars) $(D ") $(GLINK StringPostfix)$(OPT)

$(GNAME HexStringChars):
    $(GLINK HexStringChar)
    $(GLINK HexStringChar) $(I HexStringChars)

$(GNAME HexStringChar):
    $(GLINK HexDigit)
    $(GLINK WhiteSpace)
    $(GLINK EndOfLine)

$(GNAME StringPostfix):
    $(B c)
    $(B w)
    $(B d)

$(GNAME DelimitedString):
    $(B q") $(I Delimiter) $(GLINK WysiwygCharacters) $(I MatchingDelimiter) $(B ")

$(GNAME TokenString):
    $(D q)$(CODE_LCURL) $(GLINK Token)s $(CODE_RCURL)

)

	$(P
	A string literal is either a double quoted string, a wysiwyg quoted
	string, a delimited string, a token string,	or a hex string.
	)

	$(P In all string literal forms, an $(GLINK EndOfLine) is regarded as a single
	$(D \n) character.)

$(H4 Wysiwyg Strings)

	$(P
	Wysiwyg ("what you see is what you get") quoted strings are enclosed by r" and ".
	All characters between
	the r" and " are part of the string.
	There are no escape sequences inside r" ":
	)

---------------
r"hello"
r"c:\root\foo.exe"
r"ab\n" // string is 4 characters,
        // 'a', 'b', '\', 'n'
---------------

	$(P
	An alternate form of wysiwyg strings are enclosed by backquotes,
	the $(BACKTICK) character. The $(BACKTICK) character is not available on some keyboards
	and the font rendering of it is sometimes indistinguishable from
	the regular ' character. Since, however, the $(BACKTICK) is rarely used,
	it is useful to delineate strings with " in them.
	)

---------------
`hello`
`c:\root\foo.exe`
`The "lazy" dog`
`a"b\n`  // string is 5 characters,
        // 'a', '"', 'b', '\', 'n'
---------------

$(H4 Double Quoted Strings)

	$(P Double quoted strings are enclosed by "". Escape sequences can be
	embedded into them with the typical $(D \) notation.)

---------------
"hello"
"c:\\root\\foo.exe"
"ab\n"   // string is 3 characters,
         // 'a', 'b', and a linefeed
"ab
"        // string is 3 characters,
         // 'a', 'b', and a linefeed
---------------



$(H4 Hex Strings)

	$(P Hex strings allow string literals to be created using hex data.
	The hex data need not form valid UTF characters.
	)

--------------
x"0A"              // same as "\x0A"
x"00 FBCD 32FD 0A" // same as
                   // "\x00\xFB\xCD\x32\xFD\x0A"
--------------

	Whitespace and newlines are ignored, so the hex data can be
	easily formatted.
	The number of hex characters must be a multiple of 2.
	$(P )

	Adjacent strings are concatenated with the ~ operator, or by simple
	juxtaposition:

--------------
"hello " ~ "world" ~ "\n" // forms the string
       // 'h','e','l','l','o',' ',
       // 'w','o','r','l','d',linefeed
--------------

	The following are all equivalent:

-----------------
"ab" "c"
r"ab" r"c"
r"a" "bc"
"a" ~ "b" ~ "c"
-----------------

	$(P The optional $(I StringPostfix) character gives a specific type
	to the string, rather than it being inferred from the context.
	This is useful when the type cannot be unambiguously inferred,
	such as when overloading based on string type. The types corresponding
	to the postfix characters are:
	)


	$(TABLE2 String Literal Postfix Characters,
	$(THEAD Postfix, Type, Aka)
	$(TROW $(B c), $(D immutable(char)[]), $(D string))
	$(TROW $(B w), $(D immutable(wchar)[]), $(D wstring))
	$(TROW $(B d), $(D immutable(dchar)[]), $(D dstring))
	)

---
"hello"c  // string
"hello"w  // wstring
"hello"d  // dstring
---

	$(P The string literals are assembled as UTF-8 char arrays,
	and the postfix is applied
	to convert to wchar or dchar as necessary as a final step.)

	$(P String literals are read only. Writes to string literals
	cannot always be detected, but cause undefined behavior.)

$(H4 Delimited Strings)

	$(P Delimited strings use various forms of delimiters.
	The delimiter, whether a character or identifer,
	must immediately follow the " without any intervening whitespace.
	The terminating delimiter must immediately precede the closing "
	without any intervening whitespace.
	A $(I nesting delimiter) nests, and is one of the
	following characters:
	)

	$(TABLE2 Nesting Delimiters,
	$(THEAD Delimiter, Matching Delimiter)
	$(TROW $(D [), $(D ]))
	$(TROW $(LPAREN), $(RPAREN))
	$(TROW $(D <), $(D >))
	$(TROW $(CODE_LCURL), $(CODE_RCURL))
	)

---
q"(foo(xxx))"   // "foo(xxx)"
q"[foo{]"       // "foo{"
---

	$(P If the delimiter is an identifier, the identifier must
	be immediately followed by a newline, and the matching
	delimiter is the same identifier starting at the beginning
	of the line:
	)
---
writeln(q"EOS
This
is a multi-line
heredoc string
EOS"
);
---
	$(P The newline following the opening identifier is not part
	of the string, but the last newline before the closing
	identifier is part of the string. The closing identifier
    must be placed on its own line at the leftmost column.
	)

	$(P Otherwise, the matching delimiter is the same as
	the delimiter character:)

---
q"/foo]/"          // "foo]"
// q"/abc/def/"    // error
---

$(H4 Token Strings)

	$(P Token strings open with the characters $(D q)$(CODE_LCURL) and close with
	the token $(CODE_RCURL). In between must be valid D tokens.
	The $(CODE_LCURL) and $(CODE_RCURL) tokens nest.
	The string is formed of all the characters between the opening
	and closing of the token string, including comments.
	)

---
q{foo}              // "foo"
q{/*}*/ }           // "/*}*/ "
q{ foo(q{hello}); } // " foo(q{hello}); "
q{ __TIME__ }       // " __TIME__ "
    // i.e. it is not replaced with the time
// q{ __EOF__ }     // error
    // __EOF__ is not a token, it's end of file
---

$(H4 Escape Sequences)

    $(P The following table explains the meaning of the escape sequences listed
    in $(GLINK EscapeSequence):)

    $(TABLE2 Escape Sequences,
    $(THEAD Sequence, Meaning)
    $(TROW $(D \'), Literal single-quote: $(D '))
    $(TROW $(D \"), Literal double-quote: $(D "))
    $(TROW $(D \?), Literal question mark: $(D ?))
    $(TROW $(D \\), Literal backslash: $(D \))
    $(TROW $(D \0), Binary zero (NUL, U+0000).)
    $(TROW $(D \a), BEL (alarm) character (U+0007).)
    $(TROW $(D \b), Backspace (U+0008).)
    $(TROW $(D \f), Form feed (FF) (U+000C).)
    $(TROW $(D \n), End-of-line (U+000A).)
    $(TROW $(D \r), Carriage return (U+000D).)
    $(TROW $(D \t), Horizontal tab (U+0009).)
    $(TROW $(D \v), Vertical tab (U+000B).)
    $(TROW $(D \x)$(I nn), Byte value in hexadecimal$(COMMA) where $(I nn) is
        specified as two hexadecimal digits.$(BR)For example: $(D \xFF)
        represents the character with the value 255.)
    $(TROW $(D \)$(I n)$(BR)$(D \)$(I nn)$(BR)$(D \)$(I nnn), Byte value in
        octal.$(BR)For example: $(D \775) represents the character with the
        value 509.)
    $(TROW $(D \u)$(I nnnn), Unicode character U+$(I nnnn)$(COMMA) where
        $(I nnnn) are four hexadecimal digits.$(BR)For example$(COMMA)
        $(D \u042F) represents the Unicode character Я (U+42F).)
    $(TROW $(D \U)$(I nnnnnnnn), Unicode character U+$(I nnnnnnnn)$(COMMA)
        where $(I nnnnnnnn) are 8 hexadecimal digits.$(BR)For example$(COMMA)
        $(D \U0001F603) represents the Unicode character U+1F603 (SMILING FACE
        WITH OPEN MOUTH).)
    $(TROW $(D \)$(I name), Named character entity from the HTML5
        specification. See $(GLINK2 entity, NamedCharacterEntity) for more
        details.)
    )


$(H3 $(LNAME2 characterliteral, Character Literals))

$(GRAMMAR
$(GNAME CharacterLiteral):
    $(B ') $(GLINK SingleQuotedCharacter) $(B ')

$(GNAME SingleQuotedCharacter):
    $(GLINK Character)
    $(GLINK EscapeSequence)
)

	Character literals are a single character or escape sequence
	enclosed by single quotes, $(D ' ').

$(H3 $(LNAME2 integerliteral, Integer Literals))

$(GRAMMAR
$(GNAME IntegerLiteral):
    $(GLINK Integer)
    $(GLINK Integer) $(GLINK IntegerSuffix)

$(GNAME Integer):
    $(GLINK DecimalInteger)
    $(GLINK BinaryInteger)
    $(GLINK HexadecimalInteger)

$(GNAME IntegerSuffix):
    $(B L)
    $(B u)
    $(B U)
    $(B Lu)
    $(B LU)
    $(B uL)
    $(B UL)

$(GNAME DecimalInteger):
    $(B 0)
    $(GLINK NonZeroDigit)
    $(GLINK NonZeroDigit) $(GLINK DecimalDigitsUS)

$(GNAME BinaryInteger):
    $(GLINK BinPrefix) $(GLINK BinaryDigitsUS)

$(GNAME BinPrefix):
    $(B 0b)
    $(B 0B)

$(GNAME HexadecimalInteger):
    $(GLINK HexPrefix) $(GLINK HexDigitsNoSingleUS)

$(GNAME NonZeroDigit):
    $(B 1)
    $(B 2)
    $(B 3)
    $(B 4)
    $(B 5)
    $(B 6)
    $(B 7)
    $(B 8)
    $(B 9)

$(GNAME DecimalDigits):
    $(GLINK DecimalDigit)
    $(GLINK DecimalDigit) $(I DecimalDigits)

$(GNAME DecimalDigitsUS):
    $(GLINK DecimalDigitUS)
    $(GLINK DecimalDigitUS) $(I DecimalDigitsUS)

$(GNAME DecimalDigitsNoSingleUS):
    $(GLINK DecimalDigit)
    $(GLINK DecimalDigit) $(GLINK DecimalDigitsUS)
    $(GLINK DecimalDigitsUS) $(GLINK DecimalDigit)

$(GNAME DecimalDigitsNoStartingUS):
    $(GLINK DecimalDigit)
    $(GLINK DecimalDigit) $(GLINK DecimalDigitsUS)

$(GNAME DecimalDigit):
    $(B 0)
    $(GLINK NonZeroDigit)

$(GNAME DecimalDigitUS):
    $(GLINK DecimalDigit)
    $(D _)

$(GNAME BinaryDigitsUS):
    $(GLINK BinaryDigitUS)
    $(GLINK BinaryDigitUS) $(I BinaryDigitsUS)

$(GNAME BinaryDigit):
    $(B 0)
    $(B 1)

$(GNAME BinaryDigitUS):
    $(GLINK BinaryDigit)
    $(D _)

$(GNAME OctalDigit):
    $(B 0)
    $(B 1)
    $(B 2)
    $(B 3)
    $(B 4)
    $(B 5)
    $(B 6)
    $(B 7)

$(GNAME HexDigits):
    $(GLINK HexDigit)
    $(GLINK HexDigit) $(I HexDigits)

$(GNAME HexDigitsUS):
    $(GLINK HexDigitUS)
    $(GLINK HexDigitUS) $(I HexDigitsUS)

$(GNAME HexDigitsNoSingleUS):
    $(GLINK HexDigit)
    $(GLINK HexDigit) $(GLINK HexDigitsUS)
    $(GLINK HexDigitsUS) $(GLINK HexDigit)

$(GNAME HexDigitsNoStartingUS):
    $(GLINK HexDigit)
    $(GLINK HexDigit) $(GLINK HexDigitsUS)

$(GNAME HexDigit):
    $(GLINK DecimalDigit)
    $(GLINK HexLetter)

$(GNAME HexDigitUS):
    $(GLINK HexDigit)
    $(D _)

$(GNAME HexLetter):
    $(B a)
    $(B b)
    $(B c)
    $(B d)
    $(B e)
    $(B f)
    $(B A)
    $(B B)
    $(B C)
    $(B D)
    $(B E)
    $(B F)
)

	$(P Integers can be specified in decimal, binary, or hexadecimal.)

	$(P Decimal integers are a sequence of decimal digits.)

	$(P $(LNAME2 binary-literals, Binary integers) are a sequence of binary digits preceded
	by a $(SINGLEQUOTE 0b) or $(SINGLEQUOTE 0B).
	)

	$(P C-style octal integer notation was deemed too easy to mix up with decimal notation;
	it is only fully supported in string literals.
	D still supports octal integer literals interpreted at compile time through the $(REF octal, std,conv)
	template, as in $(D octal!167).)

	$(P Hexadecimal integers are a sequence of hexadecimal digits preceded
	by a $(SINGLEQUOTE 0x) or $(SINGLEQUOTE 0X).
	)

	$(P Integers can have embedded $(SINGLEQUOTE $(UNDERSCORE)) characters, which are ignored.
	The embedded $(SINGLEQUOTE $(UNDERSCORE)) are useful for formatting long literals, such
	as using them as a thousands separator:
	)

-------------
123_456       // 123456
1_2_3_4_5_6_  // 123456
-------------

	$(P Integers can be immediately followed by one $(SINGLEQUOTE L) or one of
	$(SINGLEQUOTE u) or $(SINGLEQUOTE U) or both.
	Note that there is no $(SINGLEQUOTE l) suffix.
	)

	$(P The type of the integer is resolved as follows:)

	$(TABLE2 Decimal Literal Types,
	$(THEAD Literal, Type)
    $(TROW_EXPLANATORY $(I Usual decimal notation))
	$(TROW $(D 0 .. 2_147_483_647), $(D int))
	$(TROW $(D 2_147_483_648 .. 9_223_372_036_854_775_807), $(D long))
    $(MIDRULE)
    $(TROW_EXPLANATORY $(I Explicit suffixes))
	$(TROW $(D 0L .. 9_223_372_036_854_775_807L), $(D long))
	$(TROW $(D 0U .. 4_294_967_295U), $(D uint))
	$(TROW $(D 4_294_967_296U .. 18_446_744_073_709_551_615U), $(D
	ulong))
	$(TROW $(D 0UL .. 18_446_744_073_709_551_615UL), $(D ulong))
    $(MIDRULE)
    $(TROW_EXPLANATORY $(I Hexadecimal notation))
	$(TROW $(D 0x0 .. 0x7FFF_FFFF), $(D int))
	$(TROW $(D 0x8000_0000 .. 0xFFFF_FFFF), $(D uint))
	$(TROW $(D 0x1_0000_0000 .. 0x7FFF_FFFF_FFFF_FFFF), $(D long))
	$(TROW $(D 0x8000_0000_0000_0000 .. 0xFFFF_FFFF_FFFF_FFFF), $(D
	ulong))
    $(MIDRULE)
    $(TROW_EXPLANATORY $(I Hexadecimal notation with explicit suffixes))
	$(TROW $(D 0x0L .. 0x7FFF_FFFF_FFFF_FFFFL), $(D long))
	$(TROW $(D 0x8000_0000_0000_0000L .. 0xFFFF_FFFF_FFFF_FFFFL), $(D
	ulong))
	$(TROW $(D 0x0U .. 0xFFFF_FFFFU), $(D uint))
	$(TROW $(D 0x1_0000_0000U .. 0xFFFF_FFFF_FFFF_FFFFU), $(D
	ulong))
	$(TROW $(D 0x0UL .. 0xFFFF_FFFF_FFFF_FFFFUL), $(D ulong))
	)


$(H3 $(LNAME2 floatliteral, Floating Point Literals))

$(GRAMMAR
$(GNAME FloatLiteral):
    $(GLINK Float)
    $(GLINK Float) $(GLINK Suffix)
    $(GLINK Integer) $(GLINK ImaginarySuffix)
    $(GLINK Integer) $(GLINK FloatSuffix) $(GLINK ImaginarySuffix)
    $(GLINK Integer) $(GLINK RealSuffix) $(GLINK ImaginarySuffix)

$(GNAME Float):
    $(GLINK DecimalFloat)
    $(GLINK HexFloat)

$(GNAME DecimalFloat):
    $(GLINK LeadingDecimal) $(B .)
    $(GLINK LeadingDecimal) $(B .) $(GLINK DecimalDigits)
    $(GLINK DecimalDigits) $(B .) $(GLINK DecimalDigitsNoStartingUS) $(GLINK DecimalExponent)
    $(B .) $(GLINK DecimalInteger)
    $(B .) $(GLINK DecimalInteger) $(GLINK DecimalExponent)
    $(GLINK LeadingDecimal) $(GLINK DecimalExponent)

$(GNAME DecimalExponent)
    $(GLINK DecimalExponentStart) $(GLINK DecimalDigitsNoSingleUS)

$(GNAME DecimalExponentStart)
    $(B e)
    $(B E)
    $(B e+)
    $(B E+)
    $(B e-)
    $(B E-)

$(GNAME HexFloat):
    $(GLINK HexPrefix) $(GLINK HexDigitsNoSingleUS) $(B .) $(GLINK HexDigitsNoStartingUS) $(GLINK HexExponent)
    $(GLINK HexPrefix) $(B .) $(GLINK HexDigitsNoStartingUS) $(GLINK HexExponent)
    $(GLINK HexPrefix) $(GLINK HexDigitsNoSingleUS) $(GLINK HexExponent)

$(GNAME HexPrefix):
    $(B 0x)
    $(B 0X)

$(GNAME HexExponent):
    $(GLINK HexExponentStart) $(GLINK DecimalDigitsNoSingleUS)

$(GNAME HexExponentStart):
    $(B p)
    $(B P)
    $(B p+)
    $(B P+)
    $(B p-)
    $(B P-)


$(GNAME Suffix):
    $(GLINK FloatSuffix)
    $(GLINK RealSuffix)
    $(GLINK ImaginarySuffix)
    $(GLINK FloatSuffix) $(GLINK ImaginarySuffix)
    $(GLINK RealSuffix) $(GLINK ImaginarySuffix)

$(GNAME FloatSuffix):
    $(B f)
    $(B F)

$(GNAME RealSuffix):
    $(B L)

$(GNAME ImaginarySuffix):
    $(B i)

$(GNAME LeadingDecimal):
    $(GLINK DecimalInteger)
    $(B 0) $(GLINK DecimalDigitsNoSingleUS)
)

	$(P Floats can be in decimal or hexadecimal format.)

	$(P Hexadecimal floats are preceded by a $(B 0x) or $(B 0X) and the
	exponent is a $(B p) or $(B P) followed by a decimal number serving as
	the exponent of 2.
	)

	$(P Floating literals can have embedded $(SINGLEQUOTE $(UNDERSCORE)) characters, which are ignored.
	The embedded $(SINGLEQUOTE $(UNDERSCORE)) are useful for formatting long literals to
	make them more readable, such
	as using them as a thousands separator:
	)

---------
123_456.567_8         // 123456.5678
1_2_3_4_5_6_.5_6_7_8  // 123456.5678
1_2_3_4_5_6_.5e-6_    // 123456.5e-6
---------

	$(P Floating literals with no suffix are of type double.
	Floats can be followed by one $(B f), $(B F),
	or $(B L) suffix.
	The $(B f) or $(B F) suffix means it is a
	float, and $(B L) means it is a real.
	)

	$(P If a floating literal is followed by $(B i), then it is an
	$(I ireal) (imaginary) type.
	)

	$(P Examples:)

---------
0x1.FFFFFFFFFFFFFp1023 // double.max
0x1p-52                // double.epsilon
1.175494351e-38F       // float.min
6.3i                   // idouble 6.3
6.3fi                  // ifloat 6.3
6.3Li                  // ireal 6.3
---------

	$(P It is an error if the literal exceeds the range of the type.
	It is not an error if the literal is rounded to fit into
	the significant digits of the type.
	)

	$(P Complex literals are not tokens, but are assembled from
	real and imaginary expressions during semantic analysis:
	)

---------
4.5 + 6.2i  // complex number (phased out)
---------

$(H3 $(LNAME2 keywords Keywords))

	Keywords are reserved identifiers.

	See Also: Globally Defined $(GLINK Symbols).

$(GRAMMAR
$(GNAME Keyword):
$(MULTICOLS 4,
    $(LINK2 attribute.html#abstract, $(D abstract))
    $(LINK2 declaration.html#alias, $(D alias))
    $(LINK2 iasm.html#IntegerExpression, $(D align))
    $(LINK2 statement.html#AsmStatement, $(D asm))
    $(LINK2 expression.html#AssertExpression, $(D assert))
    $(LINK2 attribute.html#auto, $(D auto))

    $(LINK2 function.html#BodyStatement, $(D body))
    $(LINK2 type.html, $(D bool))
    $(LINK2 statement.html#BreakStatement, $(D break))
    $(LINK2 type.html, $(D byte))

    $(LINK2 statement.html#SwitchStatement, $(D case))
    $(LINK2 expression.html#CastExpression, $(D cast))
    $(LINK2 statement.html#TryStatement, $(D catch))
    $(LINK2 type.html, $(D cdouble))
    $(LINK2 type.html, $(D cent))
    $(LINK2 type.html, $(D cfloat))
    $(LINK2 type.html, $(D char))
    $(LINK2 class.html, $(D class))
    $(LINK2 attribute.html#const, $(D const))
    $(LINK2 statement.html#ContinueStatement, $(D continue))
    $(LINK2 type.html, $(D creal))

    $(LINK2 type.html, $(D dchar))
    $(LINK2 version.html#debug, $(D debug))
    $(LINK2 statement.html#SwitchStatement, $(D default))
    $(LINK2 type.html#delegates, $(D delegate))
    $(LINK2 expression.html#DeleteExpression, $(D delete)) ($(LINK2 $(ROOT_DIR)deprecate.html#delete, deprecated))
    $(LINK2 attribute.html#deprecated, $(D deprecated))
    $(LINK2 statement.html#DoStatement, $(D do))
    $(LINK2 type.html, $(D double))

    $(LINK2 statement.html#IfStatement, $(D else))
    $(LINK2 enum.html, $(D enum))
    $(LINK2 attribute.html#ProtectionAttribute, $(D export))
    $(LINK2 attribute.html#linkage, $(D extern))

    $(LINK2 type.html, $(D false))
    $(LINK2 class.html#final, $(D final))
    $(LINK2 statement.html#TryStatement, $(D finally))
    $(LINK2 type.html, $(D float))
    $(LINK2 statement.html#ForStatement, $(D for))
    $(LINK2 statement.html#ForeachStatement, $(D foreach))
    $(LINK2 statement.html#ForeachStatement, $(D foreach_reverse))
    $(LINK2 expression.html#FunctionLiteral, $(D function))

    $(LINK2 statement.html#GotoStatement, $(D goto))

    $(LINK2 type.html, $(D idouble))
    $(LINK2 statement.html#IfStatement, $(D if))
    $(LINK2 type.html, $(D ifloat))
    $(LINK2 attribute.html#immutable, $(D immutable))
    $(LINK2 expression.html#ImportExpression, $(D import))
    $(LINK2 expression.html#InExpression, $(D in))
    $(LINK2 function.html#inout-functions, $(D inout))
    $(LINK2 type.html, $(D int))
    $(LINK2 interface.html, $(D interface))
    $(LINK2 contracts.html, $(D invariant))
    $(LINK2 type.html, $(D ireal))
    $(LINK2 expression.html#IsExpression, $(D is))

    $(LINK2 function.html#overload-sets, $(D lazy))
    $(LINK2 type.html, $(D long))

    $(D macro) (unused)
    $(LINK2 expression.html#MixinExpression, $(D mixin))
    $(LINK2 module.html#ModuleDeclaration, $(D module))

    $(LINK2 expression.html#NewExpression, $(D new))
    $(LINK2 function.html#nothrow-functions, $(D nothrow))
    $(LINK2 expression.html#null, $(D null))

    $(LINK2 function.html#OutStatement, $(D out))
    $(LINK2 attribute.html#override, $(D override))

    $(LINK2 attribute.html#ProtectionAttribute, $(D package))
    $(LINK2 pragma.html, $(D pragma))
    $(LINK2 attribute.html#ProtectionAttribute, $(D private))
    $(LINK2 attribute.html#ProtectionAttribute, $(D protected))
    $(LINK2 attribute.html#ProtectionAttribute, $(D public))
    $(LINK2 function.html#pure-functions, $(D pure))

    $(LINK2 type.html, $(D real))
    $(LINK2 function.html#ref-functions, $(D ref))
    $(LINK2 statement.html#ReturnStatement, $(D return))

    $(LINK2 statement.html#ScopeGuardStatement, $(D scope))
    $(LINK2 attribute.html#shared, $(D shared))
    $(LINK2 type.html, $(D short))
    $(LINK2 version.html#staticif, $(D static))
    $(LINK2 struct.html, $(D struct))
    $(LINK2 expression.html#super, $(D super))
    $(LINK2 statement.html#SwitchStatement, $(D switch))
    $(LINK2 statement.html#SynchronizedStatement, $(D synchronized))

    $(LINK2 template.html, $(D template))
    $(LINK2 expression.html#this, $(D this))
    $(LINK2 statement.html#ThrowStatement, $(D throw))
    $(LINK2 type.html, $(D true))
    $(LINK2 statement.html#TryStatement, $(D try))
    $(LINK2 $(ROOT_DIR)deprecate.html#typedef, $(D typedef)) (deprecated)
    $(LINK2 expression.html#TypeidExpression, $(D typeid))
    $(LINK2 declaration.html#Typeof, $(D typeof))

    $(LINK2 type.html, $(D ubyte))
    $(LINK2 type.html, $(D ucent))
    $(LINK2 type.html, $(D uint))
    $(LINK2 type.html, $(D ulong))
    $(LINK2 struct.html, $(D union))
    $(LINK2 unittest.html, $(D unittest))
    $(LINK2 type.html, $(D ushort))

    $(LINK2 version.html#version, $(D version))
    $(LINK2 declaration.html#VoidInitializer, $(D void))
    $(LINK2 $(ROOT_DIR)ctod.html#volatile, $(D volatile)) ($(LINK2 $(ROOT_DIR)deprecate.html#volatile, deprecated))

    $(LINK2 type.html, $(D wchar))
    $(LINK2 statement.html#WhileStatement, $(D while))
    $(LINK2 statement.html#WithStatement, $(D with))

    $(LINK2 traits.html#specialkeywords, $(D __FILE__))
    $(LINK2 traits.html#specialkeywords, $(D __MODULE__))
    $(LINK2 traits.html#specialkeywords, $(D __LINE__))
    $(LINK2 traits.html#specialkeywords, $(D __FUNCTION__))
    $(LINK2 traits.html#specialkeywords, $(D __PRETTY_FUNCTION__))

    $(LINK2 attribute.html#gshared, $(D __gshared))
    $(LINK2 traits.html, $(D __traits))
    $(LINK2 $(ROOT_DIR)phobos/core_simd.html#.Vector, $(D __vector))
    $(LINK2 expression.html#IsExpression, $(D __parameters))
))

$(H3 $(LNAME2 global_symbols, Globally Defined Symbols))

	These are defined in object$(UNDERSCORE).d, which is automatically imported by the default implementation.

$(GRAMMAR
$(GNAME Symbols):
$(MULTICOLS 4,
    $(D $(LINK2 arrays.html#strings, string)) (alias to immutable(char)[]) <!-- This is important; people want to know these things up front. -->
    $(D $(LINK2 arrays.html#strings, wstring)) (alias to immutable(wchar)[])
    $(D $(LINK2 arrays.html#strings, dstring)) (alias to immutable(dchar)[])

    $(D $(LINK2 type.html#size_t, size_t))
    $(D $(LINK2 type.html#ptrdiff_t, ptrdiff_t))
))

$(H3 $(LNAME2 specialtokens, Special Tokens))

	$(P
	These tokens are replaced with other tokens according to the following
	table:
	)

	$(TABLE_2COLS Special Tokens,
	$(THEAD Special Token, Replaced with)

	$(TROW $(D __DATE__), string literal of the date of compilation "$(I mmm dd yyyy)")
	$(TROW $(D __EOF__), sets the scanner to the end of the file)

	$(TROW $(D __TIME__), string literal of the time of compilation "$(I hh:mm:ss)")
	$(TROW $(D __TIMESTAMP__), string literal of the date and time of compilation "$(I www mmm dd hh:mm:ss yyyy)")
	$(TROW $(D __VENDOR__), $(ARGS Compiler vendor string, such as "Digital Mars D"))
	$(TROW $(D __VERSION__), $(ARGS Compiler version as an integer, such as 2001))
	)

$(H3 $(LEGACY_LNAME2 Special Token Sequence, special-token-sequence, Special Token Sequences))

$(GRAMMAR
$(GNAME SpecialTokenSequence):
    $(D # line) $(GLINK IntegerLiteral) $(GLINK EndOfLine)
    $(D # line) $(GLINK IntegerLiteral) $(GLINK Filespec) $(GLINK EndOfLine)

$(GNAME Filespec):
    $(B ") $(GLINK Characters) $(B ")
)

	$(P Special token sequences are processed by the lexical analyzer, may
	appear between any other tokens, and do not affect the syntax
	parsing.
	)

	$(P There is currently only one special token sequence, $(D #line).
	)

	$(P This sets the source line number to $(GLINK IntegerLiteral),
	and optionally the source file 	name to $(GLINK Filespec),
	beginning with the next line of source text.
	The source file and line number is used for printing error messages
	and for mapping generated code back to the source for the symbolic
	debugging output.
	)

	$(P For example:
	)

-----------------
int #line 6 "foo\bar"
x;  // this is now line 6 of file foo\bar
-----------------

	$(P Note that the backslash character is not treated specially inside
	$(GLINK Filespec) strings.
	)
)

Macros:
	TITLE=Lexical