/
scanner.go
161 lines (138 loc) · 4.18 KB
/
scanner.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
package scanner
import (
"github.com/jsightapi/jsight-schema-core/bytes"
"github.com/jsightapi/jsight-schema-core/fs"
"github.com/jsightapi/jsight-api-core/jerr"
)
type stepFunc func(*Scanner, byte) *jerr.JApiError
type Scanner struct {
data bytes.Bytes
file *fs.File
// step a function, that will be evaluated for next byte.
step stepFunc
// stepStack keeping previous step when going into comments to return to when
// comment is done.
stepStack stepFuncStack
// finds gather beginnings or ends of lexemes during steps.
finds []LexemeEvent
// stack to keep the beginning of a lexeme until ending is found.
stack eventStack
lastDirectiveParameters []*Lexeme
curIndex bytes.Index
dataSize bytes.Index
}
func NewJApiScanner(file *fs.File) *Scanner {
s := Scanner{
step: stateRoot,
file: file,
data: file.Content(),
finds: make([]LexemeEvent, 0, 5),
stepStack: make(stepFuncStack, 0, 5),
lastDirectiveParameters: make([]*Lexeme, 0, 5),
stack: make(eventStack, 0, 5),
}
s.dataSize = s.data.LenIndex()
return &s
}
func (s *Scanner) File() *fs.File {
return s.file
}
// Next reads japi file by bytes, detects lexemes beginnings and ends and returns them as soon as they found
// returns false for the end of file
func (s *Scanner) Next() (*Lexeme, *jerr.JApiError) {
if len(s.finds) != 0 { // found beginning or end of lexeme
lex, je := s.processLexemeEvent(s.shiftFound())
if je != nil {
return nil, je
}
if lex != nil {
return lex, nil
}
}
for s.curIndex <= s.dataSize {
var c byte
if s.curIndex == s.dataSize { // file ended
c = EOF
} else {
c = s.data.Byte(s.curIndex)
if c == EOF { // we use it to imitate end of file but strictly under our control
return nil, s.japiErrorBasic("File cannot contain byte zero")
}
}
je := s.step(s, c) // evaluate byte
if je != nil {
return nil, je
}
s.curIndex++
for range s.finds { // sometimes one char means two finds
lex, je := s.processLexemeEvent(s.shiftFound())
if je != nil {
return nil, je
}
if lex != nil {
switch lex.Type() {
case Parameter:
s.lastDirectiveParameters = append(s.lastDirectiveParameters, lex)
case Keyword:
s.lastDirectiveParameters = s.lastDirectiveParameters[:0]
default:
// none
}
return lex, nil
}
}
}
return nil, nil
}
func (s *Scanner) CurrentIndex() bytes.Index {
return s.curIndex
}
// SetCurrentIndex to continue scanning from certain position
func (s *Scanner) SetCurrentIndex(i bytes.Index) {
s.curIndex = i
}
// it is important to rely here on Event index only, not current scanner index
func (s *Scanner) processLexemeEvent(lexEvent LexemeEvent) (*Lexeme, *jerr.JApiError) {
eventType := lexEvent.type_
switch {
case eventType.IsBeginning():
s.stack.Push(lexEvent)
return nil, nil
case eventType.IsEnding():
startEvent := s.stack.Pop()
startType := startEvent.type_
switch {
case startType == KeywordBegin && eventType == KeywordEnd,
startType == AnnotationBegin && eventType == AnnotationEnd,
startType == SchemaBegin && eventType == SchemaEnd,
startType == TextBegin && eventType == TextEnd,
startType == ParameterBegin && eventType == ParameterEnd,
startType == EnumBegin && eventType == EnumEnd:
lex := NewLexeme(eventType.ToLexemeType(), startEvent.position, lexEvent.position, s.file)
return lex, nil
default:
return nil, s.japiErrorBasic("Ending lexeme event does not match beginning event")
}
case eventType.IsSingle():
lex := NewLexeme(eventType.ToLexemeType(), lexEvent.position, lexEvent.position, s.file)
return lex, nil
default:
return nil, s.japiErrorBasic("Unsupported lexeme event type")
}
}
func (s *Scanner) foundAt(i bytes.Index, t LexemeEventType) {
s.finds = append(s.finds, LexemeEvent{t, i})
}
func (s *Scanner) found(t LexemeEventType) {
s.foundAt(s.curIndex, t)
}
func (s *Scanner) shiftFound() LexemeEvent {
length := len(s.finds)
if length == 0 {
panic("Empty set of found lexemes")
}
lexEvent := s.finds[0]
copy(s.finds[0:], s.finds[1:])
s.finds = s.finds[:length-1]
return lexEvent
}