-
Notifications
You must be signed in to change notification settings - Fork 175
/
chunkDataPacks.go
144 lines (119 loc) · 3.92 KB
/
chunkDataPacks.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
package badger
import (
"fmt"
"github.com/dgraph-io/badger/v2"
"github.com/onflow/flow-go/model/flow"
"github.com/onflow/flow-go/module"
"github.com/onflow/flow-go/module/metrics"
"github.com/onflow/flow-go/storage"
badgermodel "github.com/onflow/flow-go/storage/badger/model"
"github.com/onflow/flow-go/storage/badger/operation"
"github.com/onflow/flow-go/storage/badger/transaction"
)
type ChunkDataPacks struct {
db *badger.DB
collections storage.Collections
byChunkIDCache *Cache
}
func NewChunkDataPacks(collector module.CacheMetrics, db *badger.DB, collections storage.Collections, byChunkIDCacheSize uint) *ChunkDataPacks {
store := func(key interface{}, val interface{}) func(*transaction.Tx) error {
chdp := val.(*badgermodel.StoredChunkDataPack)
return transaction.WithTx(operation.SkipDuplicates(operation.InsertChunkDataPack(chdp)))
}
retrieve := func(key interface{}) func(tx *badger.Txn) (interface{}, error) {
chunkID := key.(flow.Identifier)
var c badgermodel.StoredChunkDataPack
return func(tx *badger.Txn) (interface{}, error) {
err := operation.RetrieveChunkDataPack(chunkID, &c)(tx)
return &c, err
}
}
cache := newCache(collector, metrics.ResourceChunkDataPack,
withLimit(byChunkIDCacheSize),
withStore(store),
withRetrieve(retrieve),
)
ch := ChunkDataPacks{
db: db,
byChunkIDCache: cache,
collections: collections,
}
return &ch
}
func (ch *ChunkDataPacks) Store(c *flow.ChunkDataPack) error {
sc := toStoredChunkDataPack(c)
err := operation.RetryOnConflictTx(ch.db, transaction.Update, ch.byChunkIDCache.PutTx(sc.ChunkID, sc))
if err != nil {
return fmt.Errorf("could not store chunk datapack: %w", err)
}
return nil
}
func (ch *ChunkDataPacks) Remove(chunkID flow.Identifier) error {
err := operation.RetryOnConflict(ch.db.Update, operation.RemoveChunkDataPack(chunkID))
if err != nil {
return fmt.Errorf("could not remove chunk datapack: %w", err)
}
// TODO Integrate cache removal in a similar way as storage/retrieval is
ch.byChunkIDCache.Remove(chunkID)
return nil
}
func (ch *ChunkDataPacks) BatchStore(c *flow.ChunkDataPack, batch storage.BatchStorage) error {
sc := toStoredChunkDataPack(c)
writeBatch := batch.GetWriter()
batch.OnSucceed(func() {
ch.byChunkIDCache.Insert(sc.ChunkID, sc)
})
return operation.BatchInsertChunkDataPack(sc)(writeBatch)
}
func (ch *ChunkDataPacks) ByChunkID(chunkID flow.Identifier) (*flow.ChunkDataPack, error) {
schdp, err := ch.byChunkID(chunkID)
if err != nil {
return nil, err
}
chdp := &flow.ChunkDataPack{
ChunkID: schdp.ChunkID,
StartState: schdp.StartState,
Proof: schdp.Proof,
}
if !schdp.SystemChunk {
collection, err := ch.collections.ByID(schdp.CollectionID)
if err != nil {
return nil, fmt.Errorf("could not retrive collection (id: %x) for stored chunk data pack: %w", schdp.CollectionID, err)
}
chdp.Collection = collection
}
return chdp, nil
}
func (ch *ChunkDataPacks) byChunkID(chunkID flow.Identifier) (*badgermodel.StoredChunkDataPack, error) {
tx := ch.db.NewTransaction(false)
defer tx.Discard()
schdp, err := ch.retrieveCHDP(chunkID)(tx)
if err != nil {
return nil, fmt.Errorf("could not retrive stored chunk data pack: %w", err)
}
return schdp, nil
}
func (ch *ChunkDataPacks) retrieveCHDP(chunkID flow.Identifier) func(*badger.Txn) (*badgermodel.StoredChunkDataPack, error) {
return func(tx *badger.Txn) (*badgermodel.StoredChunkDataPack, error) {
val, err := ch.byChunkIDCache.Get(chunkID)(tx)
if err != nil {
return nil, err
}
return val.(*badgermodel.StoredChunkDataPack), nil
}
}
func toStoredChunkDataPack(c *flow.ChunkDataPack) *badgermodel.StoredChunkDataPack {
sc := &badgermodel.StoredChunkDataPack{
ChunkID: c.ChunkID,
StartState: c.StartState,
Proof: c.Proof,
SystemChunk: false,
}
if c.Collection != nil {
// non system chunk
sc.CollectionID = c.Collection.ID()
} else {
sc.SystemChunk = true
}
return sc
}