forked from unidoc/unioffice
-
Notifications
You must be signed in to change notification settings - Fork 0
/
lex.go
88 lines (74 loc) · 1.74 KB
/
lex.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
// Copyright 2017 Baliance. All rights reserved.
//
// Use of this source code is governed by the terms of the Affero GNU General
// Public License version 3.0 as published by the Free Software Foundation and
// appearing in the file LICENSE included in the packaging of this file. A
// commercial license can be purchased by contacting sales@baliance.com.
package formula
// Based off of http://ieeexplore.ieee.org/document/7335408/
import (
"fmt"
"io"
"strings"
"sync"
)
var debugLex = false
type tokenType int
func (t tokenType) String() string {
return yyTokname(int(t))
}
type node struct {
token tokenType
val string
}
func printable(s string) string {
s = strings.Replace(s, "\n", "\\n", -1)
s = strings.Replace(s, "\r", "\\r", -1)
s = strings.Replace(s, "\t", "\\t", -1)
return s
}
func (n node) String() string {
return fmt.Sprintf("{%s %s}", n.token, printable(string(n.val)))
}
//go:generate ragel -G2 -Z lexer.rl
//go:generate goimports -w lexer.go
type Lexer struct {
nodes chan *node
lock sync.Mutex
injected []chan *node
peeked []*node
}
func NewLexer() *Lexer {
return &Lexer{nodes: make(chan *node)}
}
func LexReader(r io.Reader) chan *node {
l := NewLexer()
go l.lex(r)
return l.nodes
}
func (l *Lexer) emit(typ tokenType, val []byte) {
if debugLex {
fmt.Println("emit", typ, printable(string(val)))
}
l.nodes <- &node{typ, string(val)}
}
func (l *Lexer) nextRaw() *node {
for len(l.injected) != 0 {
n := <-l.injected[len(l.injected)-1]
if n != nil {
return n
}
l.injected = l.injected[0 : len(l.injected)-1]
}
return <-l.nodes
}
func (l *Lexer) Next() *node {
l.lock.Lock()
defer l.lock.Unlock()
if len(l.peeked) > 0 {
n := l.peeked[0]
l.peeked = l.peeked[1:]
return n
}
return l.nextRaw()
}