forked from alecthomas/chroma
-
Notifications
You must be signed in to change notification settings - Fork 1
/
remap.go
80 lines (73 loc) · 1.69 KB
/
remap.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
package chroma
type remappingLexer struct {
lexer Lexer
mapper func(*Token) []*Token
}
// RemappingLexer remaps a token to a set of, potentially empty, tokens.
func RemappingLexer(lexer Lexer, mapper func(*Token) []*Token) Lexer {
return &remappingLexer{lexer, mapper}
}
func (r *remappingLexer) Config() *Config {
return r.lexer.Config()
}
func (r *remappingLexer) Tokenise(options *TokeniseOptions, text string) (Iterator, error) {
it, err := r.lexer.Tokenise(options, text)
if err != nil {
return nil, err
}
buffer := []*Token{}
return func() *Token {
for {
if len(buffer) > 0 {
t := buffer[0]
buffer = buffer[1:]
return t
}
t := it()
if t == nil {
return t
}
buffer = r.mapper(t)
}
}, nil
}
type TypeMapping []struct {
From, To TokenType
Words []string
}
// TypeRemappingLexer remaps types of tokens coming from a parent Lexer.
//
// eg. Map "defvaralias" tokens of type NameVariable to NameFunction:
//
// mapping := TypeMapping{
// {NameVariable, NameFunction, []string{"defvaralias"},
// }
// lexer = TypeRemappingLexer(lexer, mapping)
func TypeRemappingLexer(lexer Lexer, mapping TypeMapping) Lexer {
// Lookup table for fast remapping.
lut := map[TokenType]map[string]TokenType{}
for _, rt := range mapping {
km, ok := lut[rt.From]
if !ok {
km = map[string]TokenType{}
lut[rt.From] = km
}
if len(rt.Words) == 0 {
km[""] = rt.To
} else {
for _, k := range rt.Words {
km[k] = rt.To
}
}
}
return RemappingLexer(lexer, func(t *Token) []*Token {
if k, ok := lut[t.Type]; ok {
if tt, ok := k[t.Value]; ok {
t.Type = tt
} else if tt, ok := k[""]; ok {
t.Type = tt
}
}
return []*Token{t}
})
}