-
Notifications
You must be signed in to change notification settings - Fork 0
/
lexer.go
111 lines (100 loc) · 1.92 KB
/
lexer.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
package lexer
type Lexer struct {
input *Input
}
func NewLexer(in *Input) *Lexer {
return &Lexer{
input: in,
}
}
func (l *Lexer) Read() (t Token) {
defer func() {
if t.Kind != EOFToken && t.Value == "" {
t.Value = string(l.input.raw[t.Start:t.End])
}
}()
l.ignore()
t.Start = l.input.Pos
if l.isSingleCharacterToken(&t) {
return
}
r := l.input.PeekOne(0)
switch {
case runeDot == r:
l.readDot(&t)
return
case runeQuotation == r:
l.readStringValue(&t)
return
case isDigit(r) || runeNegativeSign == r:
if l.readIntValue(&t) {
return
} else if l.readFloatValue(&t) {
return
}
// TODO: undefined token
return
}
l.readName(&t)
return
}
func (l *Lexer) ignore() {
if canIgnore(l.input.PeekOne(0)) {
l.input.Pos++
l.ignore()
} else if l.input.PeekOne(0) == runeHashtag {
for isCommentCharacter(l.input.PeekOne(0)) {
l.input.Pos++
}
l.ignore()
}
}
func (l *Lexer) peekEqual(bs ...rune) bool {
for i := 0; i < len(bs); i++ {
if l.input.PeekOne(i+1) != bs[i] {
return false
}
}
return true
}
func (l *Lexer) peekRules(bs ...func(rune) bool) bool {
for i := 0; i < len(bs); i++ {
if !bs[i](l.input.PeekOne(i)) {
return false
}
}
return true
}
func (l *Lexer) isSingleCharacterToken(t *Token) bool {
if l.input.PeekOne(0) == runeEOF {
t.Kind = EOFToken
} else if isPunctuator(l.input.PeekOne(0)) {
t.Kind = PunctuatorToken
t.Value = string(l.input.raw[l.input.Pos : l.input.Pos+1])
} else {
return false
}
l.input.Pos++
t.End = l.input.Pos
return true
}
func (l *Lexer) readName(t *Token) {
t.Kind = NameToken
defer func() {
t.End = l.input.Pos
}()
for {
if l.input.Pos > t.Start && isName(l.input.PeekOne(0)) {
l.input.Pos++
} else if l.input.Pos == t.Start && isNameStart(l.input.PeekOne(0)) {
l.input.Pos++
} else {
return
}
}
}
func (l *Lexer) readDot(t *Token) {
t.Kind = PunctuatorToken
l.input.Pos += 3
t.End = l.input.Pos
}