-
Notifications
You must be signed in to change notification settings - Fork 4
/
merger.go
52 lines (42 loc) · 1.05 KB
/
merger.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
package search
import (
"log"
"sort"
"strconv"
"time"
"github.com/awesomefly/easysearch/index"
btree "github.com/awesomefly/gobtree"
)
func Merge(srcPath, dstPath string) {
log.Println("Starting merge ...")
start := time.Now()
idx := index.NewBTreeIndex(srcPath)
log.Printf("Source index loaded %d keys in %v", idx.BT.Count(), time.Since(start))
start = time.Now()
dstIdx := index.NewBTreeIndex(dstPath)
log.Printf("Dst index loaded %d keys in %v", dstIdx.BT.Count(), time.Since(start))
start = time.Now()
ch := idx.BT.FullSet()
for {
k := <-ch
d := <-ch
v := <-ch
if k == nil || d == nil || v == nil {
break
}
var src index.PostingList
src.FromBytes(v)
dst := dstIdx.Lookup(string(k), true)
dst = append(dst, src...)
sort.Sort(dst)
id, err := strconv.ParseInt(string(d), 10, 64) //TestKey.Docid()对应
if err != nil {
panic(err)
}
key := &btree.TestKey{K: string(k), Id: id}
dstIdx.BT.Insert(key, &dst)
}
log.Printf("merge %s to %s in %v", srcPath, dstPath, time.Since(start))
idx.Close()
dstIdx.Close()
}