forked from huichen/wukong
-
Notifications
You must be signed in to change notification settings - Fork 10
/
segmenter_worker.go
79 lines (70 loc) · 2.07 KB
/
segmenter_worker.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
package engine
import (
"github.com/huichen/wukong/types"
)
type segmenterRequest struct {
docId uint64
shard int
data types.DocumentIndexData
}
func (engine *Engine) segmenterWorker() {
for {
request := <-engine.segmenterChannel
tokensMap := make(map[string][]int)
numTokens := 0
if !engine.initOptions.NotUsingSegmenter && request.data.Content != "" {
// 当文档正文不为空时,优先从内容分词中得到关键词
segments := engine.segmenter.Segment([]byte(request.data.Content))
for _, segment := range segments {
token := segment.Token().Text()
if !engine.stopTokens.IsStopToken(token) {
tokensMap[token] = append(tokensMap[token], segment.Start())
}
}
numTokens = len(segments)
} else {
// 否则载入用户输入的关键词
for _, t := range request.data.Tokens {
if !engine.stopTokens.IsStopToken(t.Text) {
tokensMap[t.Text] = t.Locations
}
}
numTokens = len(request.data.Tokens)
}
// 加入非分词的文档标签
for _, label := range request.data.Labels {
if !engine.initOptions.NotUsingSegmenter {
if !engine.stopTokens.IsStopToken(label) {
tokensMap[label] = []int{}
}
} else {
tokensMap[label] = []int{}
}
}
indexerRequest := indexerAddDocumentRequest{
document: &types.DocumentIndex{
DocId: request.docId,
TokenLength: float32(numTokens),
Keywords: make([]types.KeywordIndex, len(tokensMap)),
},
}
iTokens := 0
for k, v := range tokensMap {
indexerRequest.document.Keywords[iTokens] = types.KeywordIndex{
Text: k,
// 非分词标注的词频设置为0,不参与tf-idf计算
Frequency: float32(len(v)),
Starts: v}
iTokens++
}
var dealDocInfoChan = make(chan bool, 1)
indexerRequest.dealDocInfoChan = dealDocInfoChan
engine.indexerAddDocumentChannels[request.shard] <- indexerRequest
rankerRequest := rankerAddDocRequest{
docId: request.docId,
fields: request.data.Fields,
dealDocInfoChan: dealDocInfoChan,
}
engine.rankerAddDocChannels[request.shard] <- rankerRequest
}
}