-
Notifications
You must be signed in to change notification settings - Fork 0
/
tokens.go
45 lines (38 loc) · 929 Bytes
/
tokens.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
package lexer
type Token struct {
Kind TokenKind
Value string
Err error
Start, End int
Line, Col int
}
// TokenKind tells what kind of value is in the Token
type TokenKind int
const (
// Undefined token
Undefined TokenKind = iota
// EOFToken -> EOF
EOFToken
// PunctuatorToken has special characters
PunctuatorToken
// NameToken has names
NameToken // /[_A-Za-z][_0-9A-Za-z]*/
// IntValueToken has iteger numbers
IntValueToken // NegativeSign(opt) | NonZeroDigit | Digit (list, opt)
// FloatValueToken has float numbers
FloatValueToken // Sign (opt) | IntegerPart | FractionalPart (ExponentPart)
// StringValueToken has string values
StringValueToken // "something which is a string" | """this is also valid"""
)
var tokenNames = []string{
"Undefined",
"EOF",
"Punctuator",
"Name",
"IntValue",
"FloatValue",
"StringValue",
}
func (tk TokenKind) String() string {
return tokenNames[tk]
}