forked from teivah/golang-good-code-bad-code
/
lexer.go
60 lines (50 loc) · 901 Bytes
/
lexer.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
package parser
type lexeme []byte
func (x lexeme) isNewline() bool {
return len(x) == 1 && x[0] == '\n'
}
func (x lexeme) isCommand() bool {
return len(x) >= 2 && x[0] == '-'
}
const (
start = iota
word
)
var newLine = []byte("\n")
func lex(src []byte) (words []lexeme) {
words = make([]lexeme, 0, len(src)/4)
state := start
// commandType := noCommand
i := 0
for j := 0; ; j++ {
if j == len(src) {
if i != j {
words = append(words, src[i:j])
}
break
}
switch state {
case start:
switch src[j] {
case ' ', '\t', '\n':
// Ignore
default:
state = word
}
i = j
case word:
switch src[j] {
case ' ', '\t':
words = append(words, src[i:j])
state = start
case '\n':
words = append(words, src[i:j])
words = append(words, newLine)
state = start
default:
// Keeping reading the word
}
}
}
return words
}