-
Notifications
You must be signed in to change notification settings - Fork 248
/
peekable_lex.go
59 lines (48 loc) · 1.38 KB
/
peekable_lex.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
package lexer
import (
"container/list"
"fmt"
)
// PeekableLexer wraps a lexer and provides the ability to peek forward without
// losing state.
type PeekableLexer struct {
lex *Lexer // a reference to the lexer used for tokenization
readTokens *list.List // tokens already read from the lexer during a lookahead.
}
// NewPeekableLexer returns a new PeekableLexer for the given lexer.
func NewPeekableLexer(lex *Lexer) *PeekableLexer {
return &PeekableLexer{
lex: lex,
readTokens: list.New(),
}
}
// NextToken returns the next token found in the lexer.
func (l *PeekableLexer) NextToken() Lexeme {
frontElement := l.readTokens.Front()
if frontElement != nil {
return l.readTokens.Remove(frontElement).(Lexeme)
}
return l.lex.nextToken()
}
// PeekToken performs lookahead of the given count on the token stream.
func (l *PeekableLexer) PeekToken(count int) Lexeme {
if count < 1 {
panic(fmt.Sprintf("Expected count > 1, received: %v", count))
}
// Ensure that the readTokens has at least the requested number of tokens.
if l.readTokens.Len() < count {
for {
l.readTokens.PushBack(l.lex.nextToken())
if l.readTokens.Len() == count {
break
}
}
}
// Retrieve the count-th token from the list.
var element *list.Element
element = l.readTokens.Front()
for i := 1; i < count; i++ {
element = element.Next()
}
return element.Value.(Lexeme)
}