-
Notifications
You must be signed in to change notification settings - Fork 1.2k
/
filter_any_case_phrase.go
161 lines (137 loc) · 4.29 KB
/
filter_any_case_phrase.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
package logstorage
import (
"fmt"
"strings"
"sync"
"unicode/utf8"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
)
// filterAnyCasePhrase filters field entries by case-insensitive phrase match.
//
// An example LogsQL query: `fieldName:i(word)` or `fieldName:i("word1 ... wordN")`
type filterAnyCasePhrase struct {
fieldName string
phrase string
phraseLowercaseOnce sync.Once
phraseLowercase string
tokensOnce sync.Once
tokens []string
}
func (fp *filterAnyCasePhrase) String() string {
return fmt.Sprintf("%si(%s)", quoteFieldNameIfNeeded(fp.fieldName), quoteTokenIfNeeded(fp.phrase))
}
func (fp *filterAnyCasePhrase) updateNeededFields(neededFields fieldsSet) {
neededFields.add(fp.fieldName)
}
func (fp *filterAnyCasePhrase) getTokens() []string {
fp.tokensOnce.Do(fp.initTokens)
return fp.tokens
}
func (fp *filterAnyCasePhrase) initTokens() {
fp.tokens = tokenizeStrings(nil, []string{fp.phrase})
}
func (fp *filterAnyCasePhrase) getPhraseLowercase() string {
fp.phraseLowercaseOnce.Do(fp.initPhraseLowercase)
return fp.phraseLowercase
}
func (fp *filterAnyCasePhrase) initPhraseLowercase() {
fp.phraseLowercase = strings.ToLower(fp.phrase)
}
func (fp *filterAnyCasePhrase) applyToBlockResult(br *blockResult, bm *bitmap) {
phraseLowercase := fp.getPhraseLowercase()
applyToBlockResultGeneric(br, bm, fp.fieldName, phraseLowercase, matchAnyCasePhrase)
}
func (fp *filterAnyCasePhrase) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
fieldName := fp.fieldName
phraseLowercase := fp.getPhraseLowercase()
// Verify whether fp matches const column
v := bs.csh.getConstColumnValue(fieldName)
if v != "" {
if !matchAnyCasePhrase(v, phraseLowercase) {
bm.resetBits()
}
return
}
// Verify whether fp matches other columns
ch := bs.csh.getColumnHeader(fieldName)
if ch == nil {
// Fast path - there are no matching columns.
// It matches anything only for empty phrase.
if len(phraseLowercase) > 0 {
bm.resetBits()
}
return
}
tokens := fp.getTokens()
switch ch.valueType {
case valueTypeString:
matchStringByAnyCasePhrase(bs, ch, bm, phraseLowercase)
case valueTypeDict:
matchValuesDictByAnyCasePhrase(bs, ch, bm, phraseLowercase)
case valueTypeUint8:
matchUint8ByExactValue(bs, ch, bm, phraseLowercase, tokens)
case valueTypeUint16:
matchUint16ByExactValue(bs, ch, bm, phraseLowercase, tokens)
case valueTypeUint32:
matchUint32ByExactValue(bs, ch, bm, phraseLowercase, tokens)
case valueTypeUint64:
matchUint64ByExactValue(bs, ch, bm, phraseLowercase, tokens)
case valueTypeFloat64:
matchFloat64ByPhrase(bs, ch, bm, phraseLowercase, tokens)
case valueTypeIPv4:
matchIPv4ByPhrase(bs, ch, bm, phraseLowercase, tokens)
case valueTypeTimestampISO8601:
phraseUppercase := strings.ToUpper(fp.phrase)
matchTimestampISO8601ByPhrase(bs, ch, bm, phraseUppercase, tokens)
default:
logger.Panicf("FATAL: %s: unknown valueType=%d", bs.partPath(), ch.valueType)
}
}
func matchValuesDictByAnyCasePhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phraseLowercase string) {
bb := bbPool.Get()
for _, v := range ch.valuesDict.values {
c := byte(0)
if matchAnyCasePhrase(v, phraseLowercase) {
c = 1
}
bb.B = append(bb.B, c)
}
matchEncodedValuesDict(bs, ch, bm, bb.B)
bbPool.Put(bb)
}
func matchStringByAnyCasePhrase(bs *blockSearch, ch *columnHeader, bm *bitmap, phraseLowercase string) {
visitValues(bs, ch, bm, func(v string) bool {
return matchAnyCasePhrase(v, phraseLowercase)
})
}
func matchAnyCasePhrase(s, phraseLowercase string) bool {
if len(phraseLowercase) == 0 {
// Special case - empty phrase matches only empty string.
return len(s) == 0
}
if len(phraseLowercase) > len(s) {
return false
}
if isASCIILowercase(s) {
// Fast path - s is in lowercase
return matchPhrase(s, phraseLowercase)
}
// Slow path - convert s to lowercase before matching
bb := bbPool.Get()
bb.B = stringsutil.AppendLowercase(bb.B, s)
sLowercase := bytesutil.ToUnsafeString(bb.B)
ok := matchPhrase(sLowercase, phraseLowercase)
bbPool.Put(bb)
return ok
}
func isASCIILowercase(s string) bool {
for i := 0; i < len(s); i++ {
c := s[i]
if c >= utf8.RuneSelf || (c >= 'A' && c <= 'Z') {
return false
}
}
return true
}