-
Notifications
You must be signed in to change notification settings - Fork 0
/
bloom.go
116 lines (104 loc) · 4.41 KB
/
bloom.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
package pruner
import (
"encoding/binary"
"errors"
"os"
bloomfilter "github.com/holiman/bloomfilter/v2"
"github.com/pecoin/go-fullnode/common"
"github.com/pecoin/go-fullnode/core/rawdb"
"github.com/pecoin/go-fullnode/log"
)
// stateBloomHasher is a wrapper around a byte blob to satisfy the interface API
// requirements of the bloom library used. It's used to convert a trie hash or
// contract code hash into a 64 bit mini hash.
type stateBloomHasher []byte
func (f stateBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") }
func (f stateBloomHasher) Sum(b []byte) []byte { panic("not implemented") }
func (f stateBloomHasher) Reset() { panic("not implemented") }
func (f stateBloomHasher) BlockSize() int { panic("not implemented") }
func (f stateBloomHasher) Size() int { return 8 }
func (f stateBloomHasher) Sum64() uint64 { return binary.BigEndian.Uint64(f) }
// stateBloom is a bloom filter used during the state convesion(snapshot->state).
// The keys of all generated entries will be recorded here so that in the pruning
// stage the entries belong to the specific version can be avoided for deletion.
//
// The false-positive is allowed here. The "false-positive" entries means they
// actually don't belong to the specific version but they are not deleted in the
// pruning. The downside of the false-positive allowance is we may leave some "dangling"
// nodes in the disk. But in practice the it's very unlike the dangling node is
// state root. So in theory this pruned state shouldn't be visited anymore. Another
// potential issue is for fast sync. If we do another fast sync upon the pruned
// database, it's problematic which will stop the expansion during the syncing.
// TODO address it @rjl493456442 @holiman @karalabe.
//
// After the entire state is generated, the bloom filter should be persisted into
// the disk. It indicates the whole generation procedure is finished.
type stateBloom struct {
bloom *bloomfilter.Filter
}
// newStateBloomWithSize creates a brand new state bloom for state generation.
// The bloom filter will be created by the passing bloom filter size. According
// to the https://hur.st/bloomfilter/?n=600000000&p=&m=2048MB&k=4, the parameters
// are picked so that the false-positive rate for mainnet is low enough.
func newStateBloomWithSize(size uint64) (*stateBloom, error) {
bloom, err := bloomfilter.New(size*1024*1024*8, 4)
if err != nil {
return nil, err
}
log.Info("Initialized state bloom", "size", common.StorageSize(float64(bloom.M()/8)))
return &stateBloom{bloom: bloom}, nil
}
// NewStateBloomFromDisk loads the state bloom from the given file.
// In this case the assumption is held the bloom filter is complete.
func NewStateBloomFromDisk(filename string) (*stateBloom, error) {
bloom, _, err := bloomfilter.ReadFile(filename)
if err != nil {
return nil, err
}
return &stateBloom{bloom: bloom}, nil
}
// Commit flushes the bloom filter content into the disk and marks the bloom
// as complete.
func (bloom *stateBloom) Commit(filename, tempname string) error {
// Write the bloom out into a temporary file
_, err := bloom.bloom.WriteFile(tempname)
if err != nil {
return err
}
// Ensure the file is synced to disk
f, err := os.Open(tempname)
if err != nil {
return err
}
if err := f.Sync(); err != nil {
f.Close()
return err
}
f.Close()
// Move the teporary file into it's final location
return os.Rename(tempname, filename)
}
// Put implements the KeyValueWriter interface. But here only the key is needed.
func (bloom *stateBloom) Put(key []byte, value []byte) error {
// If the key length is not 32bytes, ensure it's contract code
// entry with new scheme.
if len(key) != common.HashLength {
isCode, codeKey := rawdb.IsCodeKey(key)
if !isCode {
return errors.New("invalid entry")
}
bloom.bloom.Add(stateBloomHasher(codeKey))
return nil
}
bloom.bloom.Add(stateBloomHasher(key))
return nil
}
// Delete removes the key from the key-value data store.
func (bloom *stateBloom) Delete(key []byte) error { panic("not supported") }
// Contain is the wrapper of the underlying contains function which
// reports whether the key is contained.
// - If it says yes, the key may be contained
// - If it says no, the key is definitely not contained.
func (bloom *stateBloom) Contain(key []byte) (bool, error) {
return bloom.bloom.Contains(stateBloomHasher(key)), nil
}