/
tokens.go
155 lines (142 loc) · 2.71 KB
/
tokens.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
package tokens
import "fmt"
// TokenType identifies the type of a token
type Type int
// Known tokens
const (
Error Type = iota
Add
Assign
Colon
Comma
Div
Dot
Eq
// EqEq
Floordiv
Gt
Gteq
Lbrace
Lbracket
Lparen
Lt
Lteq
// Not
// And
// Or
// Neq
Mod
Mul
Ne
Pipe
Pow
Rbrace
Rbracket
Rparen
Semicolon
Sub
Tilde
Whitespace
Float
Integer
Name
String
Operator
BlockBegin
BlockEnd
VariableBegin
VariableEnd
RawBegin
RawEnd
CommentBegin
CommentEnd
Comment
LinestatementBegin
LinestatementEnd
LinecommentBegin
LinecommentEnd
Linecomment
Data
Initial
EOF
)
// TokenNames maps token types to their human readable name
var Names = map[Type]string{
Error: "Error",
Add: "Add",
Assign: "Assign",
Colon: "Colon",
Comma: "Comma",
Div: "Div",
Dot: "Dot",
Eq: "Eq",
// EqEq: "EqEq",
Floordiv: "Floordiv",
Gt: "Gt",
Gteq: "Gteq",
Lbrace: "Lbrace",
Lbracket: "Lbracket",
Lparen: "Lparen",
Lt: "Lt",
Lteq: "Lteq",
// Not: "Not",
// And: "And",
// Or: "Or",
// Neq: "Neq",
Mod: "Mod",
Mul: "Mul",
Ne: "Ne",
Pipe: "Pipe",
Pow: "Pow",
Rbrace: "Rbrace",
Rbracket: "Rbracket",
Rparen: "Rparen",
Semicolon: "Semicolon",
Sub: "Sub",
Tilde: "Tilde",
Whitespace: "Whitespace",
Float: "Float",
Integer: "Integer",
Name: "Name",
String: "String",
Operator: "Operator",
BlockBegin: "BlockBegin",
BlockEnd: "BlockEnd",
VariableBegin: "VariableBegin",
VariableEnd: "VariableEnd",
RawBegin: "RawBegin",
RawEnd: "RawEnd",
CommentBegin: "CommentBegin",
CommentEnd: "CommentEnd",
Comment: "Comment",
LinestatementBegin: "LinestatementBegin",
LinestatementEnd: "LinestatementEnd",
LinecommentBegin: "LinecommentBegin",
LinecommentEnd: "LinecommentEnd",
Linecomment: "Linecomment",
Data: "Data",
Initial: "Initial",
EOF: "EOF",
}
// var SymbolTokens = map[TokenType]bool {
// }
// var KeywordTokens = map[TokenType]bool {
// }
// var NumberTokens = map[TokenType]bool {
// }
// Token represents a unit of lexing
type Token struct {
Type Type
Val string
Pos int
Line int
Col int
}
func (t Token) String() string {
val := t.Val
if len(val) > 1000 {
val = fmt.Sprintf("%s...%s", val[:10], val[len(val)-5:len(val)])
}
return fmt.Sprintf("<Token[%s] Val='%s' Pos=%d Line=%d Col=%d>",
Names[t.Type], val, t.Pos, t.Line, t.Col)
}