-
Notifications
You must be signed in to change notification settings - Fork 28
/
scanner.go
80 lines (72 loc) · 1.78 KB
/
scanner.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
// Copyright 2021 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package slang
import (
"fmt"
"go/scanner"
"go/token"
"os"
)
type tokPos struct {
tok token.Token
pos token.Position
lit string
ellipsis bool
}
var supportedTokens = map[token.Token]bool{}
func init() {
for _, tok := range []token.Token{
token.IDENT, token.SEMICOLON, token.DEFINE, token.ASSIGN,
token.LPAREN, token.RPAREN, token.COMMA,
token.STRING, token.INT, token.ELLIPSIS,
} {
supportedTokens[tok] = true
}
}
func scan(fset *token.FileSet, file *token.File, src []byte) ([]tokPos, error) {
errs := &scanner.ErrorList{}
s := &scanner.Scanner{}
errHandler := func(pos token.Position, msg string) {
errs.Add(pos, msg)
}
tokens := make([]tokPos, 0, 100)
s.Init(file, src, errHandler, scanner.ScanComments)
for {
pos, tok, lit := s.Scan()
if tok == token.EOF {
break
}
if supportedTokens[tok] {
tokens = append(tokens, tokPos{tok: tok, pos: file.Position(pos), lit: lit})
continue
}
if tok == token.COMMENT {
continue
}
s.ErrorCount++
if tok.IsKeyword() {
errs.Add(fset.Position(pos), fmt.Sprintf("reserved keyword '%s'", tok))
} else {
errs.Add(fset.Position(pos), fmt.Sprintf("invalid token '%s'", tok))
}
if s.ErrorCount > 5 {
break
}
}
return tokens, errs.Err()
}
func scanFile(filename string) ([]tokPos, error) {
src, err := os.ReadFile(filename)
if err != nil {
return nil, err
}
fset := token.NewFileSet()
file := fset.AddFile(filename, fset.Base(), len(src))
return scan(fset, file, src)
}
func scanBytes(src []byte) ([]tokPos, error) {
fset := token.NewFileSet()
file := fset.AddFile("", fset.Base(), len(src))
return scan(fset, file, src)
}