/
token.go
387 lines (373 loc) · 13.1 KB
/
token.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
package mjingo
import (
"fmt"
"strconv"
"strings"
)
type token interface {
String() string
DebugString() string
typ() tokenType
}
type templateDataToken struct{ s string }
type variableStartToken struct{}
type variableEndToken struct{}
type blockStartToken struct{}
type blockEndToken struct{}
type identToken struct{ ident string }
type strToken struct{ s string }
type intToken struct{ n uint64 }
type int128Token struct{ n U128 }
type floatToken struct{ f float64 }
type plusToken struct{}
type minusToken struct{}
type mulToken struct{}
type divToken struct{}
type floorDivToken struct{}
type powToken struct{}
type modToken struct{}
type bangToken struct{}
type dotToken struct{}
type commaToken struct{}
type colonToken struct{}
type tildeToken struct{}
type assignToken struct{}
type pipeToken struct{}
type eqToken struct{}
type neToken struct{}
type gtToken struct{}
type gteToken struct{}
type ltToken struct{}
type lteToken struct{}
type bracketOpenToken struct{}
type bracketCloseToken struct{}
type parenOpenToken struct{}
type parenCloseToken struct{}
type braceOpenToken struct{}
type braceCloseToken struct{}
var _ = token(templateDataToken{})
var _ = token(variableStartToken{})
var _ = token(variableEndToken{})
var _ = token(blockStartToken{})
var _ = token(blockEndToken{})
var _ = token(identToken{})
var _ = token(strToken{})
var _ = token(intToken{})
var _ = token(int128Token{})
var _ = token(floatToken{})
var _ = token(plusToken{})
var _ = token(minusToken{})
var _ = token(mulToken{})
var _ = token(divToken{})
var _ = token(floorDivToken{})
var _ = token(powToken{})
var _ = token(modToken{})
var _ = token(bangToken{})
var _ = token(dotToken{})
var _ = token(commaToken{})
var _ = token(colonToken{})
var _ = token(tildeToken{})
var _ = token(assignToken{})
var _ = token(pipeToken{})
var _ = token(eqToken{})
var _ = token(neToken{})
var _ = token(gtToken{})
var _ = token(gteToken{})
var _ = token(ltToken{})
var _ = token(lteToken{})
var _ = token(bracketOpenToken{})
var _ = token(bracketCloseToken{})
var _ = token(parenOpenToken{})
var _ = token(parenCloseToken{})
var _ = token(braceOpenToken{})
var _ = token(braceCloseToken{})
func (t templateDataToken) String() string { return t.typ().String() }
func (t variableStartToken) String() string { return t.typ().String() }
func (t variableEndToken) String() string { return t.typ().String() }
func (t blockStartToken) String() string { return t.typ().String() }
func (t blockEndToken) String() string { return t.typ().String() }
func (t identToken) String() string { return t.typ().String() }
func (t strToken) String() string { return t.typ().String() }
func (t intToken) String() string { return t.typ().String() }
func (t int128Token) String() string { return t.typ().String() }
func (t floatToken) String() string { return t.typ().String() }
func (t plusToken) String() string { return t.typ().String() }
func (t minusToken) String() string { return t.typ().String() }
func (t mulToken) String() string { return t.typ().String() }
func (t divToken) String() string { return t.typ().String() }
func (t floorDivToken) String() string { return t.typ().String() }
func (t powToken) String() string { return t.typ().String() }
func (t modToken) String() string { return t.typ().String() }
func (t bangToken) String() string { return t.typ().String() }
func (t dotToken) String() string { return t.typ().String() }
func (t commaToken) String() string { return t.typ().String() }
func (t colonToken) String() string { return t.typ().String() }
func (t tildeToken) String() string { return t.typ().String() }
func (t assignToken) String() string { return t.typ().String() }
func (t pipeToken) String() string { return t.typ().String() }
func (t eqToken) String() string { return t.typ().String() }
func (t neToken) String() string { return t.typ().String() }
func (t gtToken) String() string { return t.typ().String() }
func (t gteToken) String() string { return t.typ().String() }
func (t ltToken) String() string { return t.typ().String() }
func (t lteToken) String() string { return t.typ().String() }
func (t bracketOpenToken) String() string { return t.typ().String() }
func (t bracketCloseToken) String() string { return t.typ().String() }
func (t parenOpenToken) String() string { return t.typ().String() }
func (t parenCloseToken) String() string { return t.typ().String() }
func (t braceOpenToken) String() string { return t.typ().String() }
func (t braceCloseToken) String() string { return t.typ().String() }
func (t templateDataToken) DebugString() string { return fmt.Sprintf("TemplateData(%q)", t.s) }
func (t variableStartToken) DebugString() string { return "VariableStart" }
func (t variableEndToken) DebugString() string { return "VariableEnd" }
func (t blockStartToken) DebugString() string { return "BlockStart" }
func (t blockEndToken) DebugString() string { return "BlockEnd" }
func (t identToken) DebugString() string { return fmt.Sprintf("Ident(%q)", t.ident) }
func (t strToken) DebugString() string { return fmt.Sprintf("Str(%q)", t.s) }
func (t intToken) DebugString() string { return fmt.Sprintf("Int(%d)", t.n) }
func (t int128Token) DebugString() string { return fmt.Sprintf("Int128(%s)", t.n) }
func (t floatToken) DebugString() string {
s := strconv.FormatFloat(t.f, 'f', -1, 64)
if strings.ContainsRune(s, '.') {
return fmt.Sprintf("Float(%s)", s)
}
return fmt.Sprintf("Float(%s.0)", s)
}
func (t plusToken) DebugString() string { return "Plus" }
func (t minusToken) DebugString() string { return "Minus" }
func (t mulToken) DebugString() string { return "Mul" }
func (t divToken) DebugString() string { return "Div" }
func (t floorDivToken) DebugString() string { return "FloorDiv" }
func (t powToken) DebugString() string { return "Pow" }
func (t modToken) DebugString() string { return "Mod" }
func (t bangToken) DebugString() string { return "Bang" }
func (t dotToken) DebugString() string { return "Dot" }
func (t commaToken) DebugString() string { return "Comma" }
func (t colonToken) DebugString() string { return "Colon" }
func (t tildeToken) DebugString() string { return "Tilde" }
func (t assignToken) DebugString() string { return "Assign" }
func (t pipeToken) DebugString() string { return "Pipe" }
func (t eqToken) DebugString() string { return "Eq" }
func (t neToken) DebugString() string { return "Ne" }
func (t gtToken) DebugString() string { return "Gt" }
func (t gteToken) DebugString() string { return "Gte" }
func (t ltToken) DebugString() string { return "Lt" }
func (t lteToken) DebugString() string { return "Lte" }
func (t bracketOpenToken) DebugString() string { return "BracketOpen" }
func (t bracketCloseToken) DebugString() string { return "BracketClose" }
func (t parenOpenToken) DebugString() string { return "ParenOpen" }
func (t parenCloseToken) DebugString() string { return "ParenClose" }
func (t braceOpenToken) DebugString() string { return "BraceOpen" }
func (t braceCloseToken) DebugString() string { return "BraceClose" }
func (templateDataToken) typ() tokenType { return tokenTypeTemplateData }
func (variableStartToken) typ() tokenType { return tokenTypeVariableStart }
func (variableEndToken) typ() tokenType { return tokenTypeVariableEnd }
func (blockStartToken) typ() tokenType { return tokenTypeBlockStart }
func (blockEndToken) typ() tokenType { return tokenTypeBlockEnd }
func (identToken) typ() tokenType { return tokenTypeIdent }
func (strToken) typ() tokenType { return tokenTypeString }
func (intToken) typ() tokenType { return tokenTypeInt }
func (int128Token) typ() tokenType { return tokenTypeInt128 }
func (floatToken) typ() tokenType { return tokenTypeFloat }
func (plusToken) typ() tokenType { return tokenTypePlus }
func (minusToken) typ() tokenType { return tokenTypeMinus }
func (mulToken) typ() tokenType { return tokenTypeMul }
func (divToken) typ() tokenType { return tokenTypeDiv }
func (floorDivToken) typ() tokenType { return tokenTypeFloorDiv }
func (powToken) typ() tokenType { return tokenTypePow }
func (modToken) typ() tokenType { return tokenTypeMod }
func (bangToken) typ() tokenType { return tokenTypeBang }
func (dotToken) typ() tokenType { return tokenTypeDot }
func (commaToken) typ() tokenType { return tokenTypeComma }
func (colonToken) typ() tokenType { return tokenTypeColon }
func (tildeToken) typ() tokenType { return tokenTypeTilde }
func (assignToken) typ() tokenType { return tokenTypeAssign }
func (pipeToken) typ() tokenType { return tokenTypePipe }
func (eqToken) typ() tokenType { return tokenTypeEq }
func (neToken) typ() tokenType { return tokenTypeNe }
func (gtToken) typ() tokenType { return tokenTypeGt }
func (gteToken) typ() tokenType { return tokenTypeGte }
func (ltToken) typ() tokenType { return tokenTypeLt }
func (lteToken) typ() tokenType { return tokenTypeLte }
func (bracketOpenToken) typ() tokenType { return tokenTypeBracketOpen }
func (bracketCloseToken) typ() tokenType { return tokenTypeBracketClose }
func (parenOpenToken) typ() tokenType { return tokenTypeParenOpen }
func (parenCloseToken) typ() tokenType { return tokenTypeParenClose }
func (braceOpenToken) typ() tokenType { return tokenTypeBraceOpen }
func (braceCloseToken) typ() tokenType { return tokenTypeBraceClose }
func isTokenOfType[T any](tkn token) bool {
_, ok := tkn.(T)
return ok
}
func isIdentTokenWithName(name ...string) func(token) bool {
return func(tkn token) bool {
identTkn, ok := tkn.(identToken)
if ok {
for _, n := range name {
if identTkn.ident == n {
return true
}
}
}
return false
}
}
type tokenType int
const (
// Raw template data.
tokenTypeTemplateData tokenType = iota + 1
// Variable block start.
tokenTypeVariableStart
// Variable block end
tokenTypeVariableEnd
// Statement block start
tokenTypeBlockStart
// Statement block end
tokenTypeBlockEnd
// An identifier.
tokenTypeIdent
// A borrowed string.
tokenTypeStr
// An allocated string.
// NOTE: not used in mjingo
tokenTypeString
// An integer (limited to i64)
tokenTypeInt
// A large integer
tokenTypeInt128
// A float
tokenTypeFloat
// A plus (`+`) operator.
tokenTypePlus
// A plus (`-`) operator.
tokenTypeMinus
// A mul (`*`) operator.
tokenTypeMul
// A div (`/`) operator.
tokenTypeDiv
// A floor division (`//`) operator.
tokenTypeFloorDiv
// Power operator (`**`).
tokenTypePow
// A mod (`%`) operator.
tokenTypeMod
// The bang (`!`) operator.
tokenTypeBang
// A dot operator (`.`)
tokenTypeDot
// The comma operator (`,`)
tokenTypeComma
// The colon operator (`:`)
tokenTypeColon
// The tilde operator (`~`)
tokenTypeTilde
// The assignment operator (`=`)
tokenTypeAssign
// The pipe symbol.
tokenTypePipe
// `==` operator
tokenTypeEq
// `!=` operator
tokenTypeNe
// `>` operator
tokenTypeGt
// `>=` operator
tokenTypeGte
// `<` operator
tokenTypeLt
// `<=` operator
tokenTypeLte
// Open Bracket
tokenTypeBracketOpen
// Close Bracket
tokenTypeBracketClose
// Open Parenthesis
tokenTypeParenOpen
// Close Parenthesis
tokenTypeParenClose
// Open Brace
tokenTypeBraceOpen
// Close Brace
tokenTypeBraceClose
)
func (k tokenType) String() string {
switch k {
case tokenTypeTemplateData:
return "template-data"
case tokenTypeVariableStart:
return "start of variable block"
case tokenTypeVariableEnd:
return "end of variable block"
case tokenTypeBlockStart:
return "start of block"
case tokenTypeBlockEnd:
return "end of block"
case tokenTypeIdent:
return "identifier"
case tokenTypeStr, tokenTypeString:
return "string"
case tokenTypeInt, tokenTypeInt128:
return "integer"
case tokenTypeFloat:
return "float"
case tokenTypePlus:
return "`+`"
case tokenTypeMinus:
return "`-`"
case tokenTypeMul:
return "`*`"
case tokenTypeDiv:
return "`/`"
case tokenTypeFloorDiv:
return "`//`"
case tokenTypePow:
return "`**`"
case tokenTypeMod:
return "`%`"
case tokenTypeBang:
return "`!`"
case tokenTypeDot:
return "`.`"
case tokenTypeComma:
return "`,`"
case tokenTypeColon:
return "`:`"
case tokenTypeTilde:
return "`~`"
case tokenTypeAssign:
return "`=`"
case tokenTypePipe:
return "`|`"
case tokenTypeEq:
return "`==`"
case tokenTypeNe:
return "`!=`"
case tokenTypeGt:
return "`>`"
case tokenTypeGte:
return "`>=`"
case tokenTypeLt:
return "`<`"
case tokenTypeLte:
return "`<=`"
case tokenTypeBracketOpen:
return "`[`"
case tokenTypeBracketClose:
return "`]`"
case tokenTypeParenOpen:
return "`(`"
case tokenTypeParenClose:
return "`)`"
case tokenTypeBraceOpen:
return "`{`"
case tokenTypeBraceClose:
return "`}`"
default:
return "unknown"
}
}
type loc struct {
line uint32
col uint32
offset uint32
}