-
Notifications
You must be signed in to change notification settings - Fork 8
/
lexer.go
99 lines (84 loc) · 2.11 KB
/
lexer.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
package fasta
import (
"bufio"
"bytes"
"io"
)
// Scanner represents a lexical scanner.
type Scanner struct {
r *bufio.Reader
}
// NewScanner returns a new instance of Scanner.
func NewScanner(r io.Reader) *Scanner {
return &Scanner{r: bufio.NewReader(r)}
}
// read reads the next rune from the bufferred reader.
// Returns the rune(0) if an error occurs (or io.EOF is returned).
func (s *Scanner) read() rune {
ch, _, err := s.r.ReadRune()
if err != nil {
return eof
}
return ch
}
// unread places the previously read rune back on the reader.
func (s *Scanner) unread() {
_ = s.r.UnreadRune()
}
// Scan returns the next token and literal value.
func (s *Scanner) Scan() (tok Token, lit string) {
// Read the next rune.
ch := s.read()
// If we see whitespace then consume all contiguous whitespace.
// If we see a letter then consume as an ident or reserved word.
if isEndOfLine(ch) {
s.unread()
return s.scanEndOfLine()
}
switch ch {
case eof:
return EOF, ""
case '>':
return STARTIDENT, string(ch)
}
s.unread()
return s.scanIdent()
}
// scanEndOfLine consumes the current rune and all contiguous \n\r.
func (s *Scanner) scanEndOfLine() (tok Token, lit string) {
// Create a buffer and read the current character into it.
var buf bytes.Buffer
buf.WriteRune(s.read())
// Read every subsequent whitespace character into the buffer.
// Non-whitespace characters and EOF will cause the loop to exit.
for {
if ch := s.read(); ch == eof {
break
} else if !isEndOfLine(ch) {
s.unread()
break
} else {
buf.WriteRune(ch)
}
}
return ENDOFLINE, buf.String()
}
// scanIdent consumes the current rune and all contiguous ident runes.
func (s *Scanner) scanIdent() (tok Token, lit string) {
// Create a buffer and read the current character into it.
var buf bytes.Buffer
buf.WriteRune(s.read())
// Read every subsequent ident character into the buffer.
// Non-ident characters and EOF will cause the loop to exit.
for {
if ch := s.read(); ch == eof {
break
} else if !isIdent(ch) {
s.unread()
break
} else {
_, _ = buf.WriteRune(ch)
}
}
return IDENTIFIER, buf.String()
}