forked from daviddengcn/gcse
/
filecache.go
82 lines (68 loc) · 2 KB
/
filecache.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
package spider
import (
"log"
"github.com/golangplus/bytes"
"github.com/golangplus/errors"
"github.com/daviddengcn/bolthelper"
"github.com/golang/protobuf/proto"
)
type FileCache interface {
Get(signature string, contents proto.Message) bool
Set(signature string, contents proto.Message)
}
type NullFileCache struct{}
func (NullFileCache) Get(string, proto.Message) bool { return false }
func (NullFileCache) Set(string, proto.Message) {}
var _ FileCache = NullFileCache{}
type BoltFileCache struct {
bh.DB
IncCounter func(string)
}
var _ FileCache = BoltFileCache{}
// Filecache folders:
// s/<path> - signature of this path
// c/<signature> - contents of a signagure
// p/<signature>/<path> - list of paths referencing this signature
var (
cacheSignatureKey = []byte("s")
cacheContentsKey = []byte("c")
cachePathsKey = []byte("p")
)
func (bc BoltFileCache) inc(name string) {
if bc.IncCounter == nil {
return
}
bc.IncCounter(name)
}
func (bc BoltFileCache) Get(sign string, contents proto.Message) bool {
found := false
if err := bc.View(func(tx bh.Tx) error {
return tx.Value([][]byte{cacheContentsKey, []byte(sign)}, func(v bytesp.Slice) error {
found = true
return errorsp.WithStacks(proto.Unmarshal(v, contents))
})
}); err != nil {
log.Printf("Reading from file cache DB for %v failed: %v", sign, err)
bc.inc("crawler.filecache.get_error")
return false
}
if found {
bc.inc("crawler.filecache.hit")
} else {
bc.inc("crawler.filecache.missed")
}
return found
}
func (bc BoltFileCache) Set(signature string, contents proto.Message) {
if err := bc.Update(func(tx bh.Tx) error {
bs, err := proto.Marshal(contents)
if err != nil {
return errorsp.WithStacksAndMessage(err, "Marshal %v failed", contents)
}
return tx.Put([][]byte{cacheContentsKey, []byte(signature)}, bs)
}); err != nil {
bc.inc("crawler.filecache.set_error")
log.Printf("Updating to file cache DB for %v failed: %v", signature, err)
}
bc.inc("crawler.filecache.sign_saved")
}