-
Notifications
You must be signed in to change notification settings - Fork 4
/
token.go
60 lines (50 loc) · 1.06 KB
/
token.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
// Copyright (C) 2014 Constantin Schomburg <me@cschomburg.com>
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.
package nlp
type Token struct {
Value string `json:"value,omitempty"`
Lemma string `json:"lemma,omitempty"`
Tags map[string]struct{} `json:"tags,omitempty"`
}
func (t Token) Is(tag string) bool {
_, ok := t.Tags[tag]
return ok
}
func (t *Token) Tag(tag string) {
t.Tags[tag] = struct{}{}
}
type tokenIterator struct {
tokens []*Token
pos int
}
func newTokenIterator(tokens []*Token) *tokenIterator {
return &tokenIterator{tokens, -1}
}
func (it *tokenIterator) Peek() *Token {
if it.pos+1 >= len(it.tokens) {
return nil
}
return it.tokens[it.pos+1]
}
func (it *tokenIterator) Next() *Token {
it.pos++
if it.pos >= len(it.tokens) {
return nil
}
return it.tokens[it.pos]
}
func (it *tokenIterator) Reset() {
it.pos = -1
}
func JoinTokens(ts []*Token) string {
s := ""
for i, t := range ts {
if i > 0 {
s += " "
}
s += t.Value
}
return s
}