/
tokens.go
112 lines (106 loc) · 2.09 KB
/
tokens.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
package sb
import "io"
type Tokens []Token
func TokensFromStream(stream Stream) (tokens Tokens, err error) {
for {
var token Token
err := stream.Next(&token)
if err != nil {
return nil, err
}
if token.Invalid() {
break
}
tokens = append(tokens, token)
}
return
}
func MustTokensFromStream(stream Stream) Tokens {
tokens, err := TokensFromStream(stream)
if err != nil {
panic(err)
}
return tokens
}
func CollectTokens(tokens *Tokens) Sink {
var sink Sink
sink = func(token *Token) (Sink, error) {
if token.Invalid() {
return nil, nil
}
*tokens = append(*tokens, *token)
return sink, nil
}
return sink
}
func CollectValueTokens(tokens *Tokens) Sink {
var sink Sink
type Frame struct {
Kind Kind
N int
}
var stack []*Frame
sink = func(token *Token) (Sink, error) {
if len(stack) > 0 {
parent := stack[len(stack)-1]
if parent.Kind == KindTypeName && parent.N > 0 {
stack = stack[:len(stack)-1]
if len(stack) > 0 {
parent = stack[len(stack)-1]
}
}
parent.N++
}
if token.Invalid() {
if len(stack) > 0 {
return nil, io.ErrUnexpectedEOF
}
return nil, nil // NOCOVER
}
*tokens = append(*tokens, *token)
switch token.Kind {
case KindArrayEnd, KindObjectEnd, KindMapEnd, KindTupleEnd:
if len(stack) == 0 {
return nil, UnexpectedEndToken
}
if token.Kind != stack[len(stack)-1].Kind {
return nil, UnexpectedEndToken
}
stack = stack[:len(stack)-1]
if len(stack) == 0 {
return nil, nil
}
return sink, nil
case KindArray:
stack = append(stack, &Frame{
Kind: KindArrayEnd,
})
return sink, nil
case KindObject:
stack = append(stack, &Frame{
Kind: KindObjectEnd,
})
return sink, nil
case KindMap:
stack = append(stack, &Frame{
Kind: KindMapEnd,
})
return sink, nil
case KindTuple:
stack = append(stack, &Frame{
Kind: KindTupleEnd,
})
return sink, nil
case KindTypeName:
stack = append(stack, &Frame{
Kind: KindTypeName,
})
return sink, nil
}
if len(stack) > 0 {
return sink, nil
}
return nil, nil
}
return sink
}