-
Notifications
You must be signed in to change notification settings - Fork 916
/
eds.go
275 lines (245 loc) · 8.43 KB
/
eds.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
package eds
import (
"bytes"
"context"
"crypto/sha256"
"errors"
"fmt"
"io"
"math"
"github.com/ipfs/go-cid"
"github.com/ipld/go-car"
"github.com/ipld/go-car/util"
"github.com/celestiaorg/celestia-app/pkg/da"
"github.com/celestiaorg/celestia-app/pkg/wrapper"
"github.com/celestiaorg/nmt"
"github.com/celestiaorg/rsmt2d"
"github.com/celestiaorg/celestia-node/libs/utils"
"github.com/celestiaorg/celestia-node/share"
"github.com/celestiaorg/celestia-node/share/ipld"
)
var ErrEmptySquare = errors.New("share: importing empty data")
// WriteEDS writes the entire EDS into the given io.Writer as CARv1 file.
// This includes all shares in quadrant order, followed by all inner nodes of the NMT tree.
// Order: [ Carv1Header | Q1 | Q2 | Q3 | Q4 | inner nodes ]
// For more information about the header: https://ipld.io/specs/transport/car/carv1/#header
func WriteEDS(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Writer) (err error) {
ctx, span := tracer.Start(ctx, "write-eds")
defer func() {
utils.SetStatusAndEnd(span, err)
}()
// Creates and writes Carv1Header. Roots are the eds Row + Col roots
err = writeHeader(eds, w)
if err != nil {
return fmt.Errorf("share: writing carv1 header: %w", err)
}
// Iterates over shares in quadrant order via eds.GetCell
err = writeQuadrants(eds, w)
if err != nil {
return fmt.Errorf("share: writing shares: %w", err)
}
// Iterates over proofs and writes them to the CAR
err = writeProofs(ctx, eds, w)
if err != nil {
return fmt.Errorf("share: writing proofs: %w", err)
}
return nil
}
// writeHeader creates a CarV1 header using the EDS's Row and Column roots as the list of DAG roots.
func writeHeader(eds *rsmt2d.ExtendedDataSquare, w io.Writer) error {
rootCids, err := rootsToCids(eds)
if err != nil {
return fmt.Errorf("getting root cids: %w", err)
}
return car.WriteHeader(&car.CarHeader{
Roots: rootCids,
Version: 1,
}, w)
}
// writeQuadrants reorders the shares to quadrant order and writes them to the CARv1 file.
func writeQuadrants(eds *rsmt2d.ExtendedDataSquare, w io.Writer) error {
hasher := nmt.NewNmtHasher(sha256.New(), share.NamespaceSize, ipld.NMTIgnoreMaxNamespace)
shares := quadrantOrder(eds)
for _, share := range shares {
leaf, err := hasher.HashLeaf(share)
if err != nil {
return fmt.Errorf("hashing share: %w", err)
}
cid, err := ipld.CidFromNamespacedSha256(leaf)
if err != nil {
return fmt.Errorf("getting cid from share: %w", err)
}
err = util.LdWrite(w, cid.Bytes(), share)
if err != nil {
return fmt.Errorf("writing share to file: %w", err)
}
}
return nil
}
// writeProofs iterates over the in-memory blockstore's keys and writes all inner nodes to the
// CARv1 file.
func writeProofs(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Writer) error {
// check if proofs are collected by ipld.ProofsAdder in previous reconstructions of eds
proofs, err := getProofs(ctx, eds)
if err != nil {
return fmt.Errorf("recomputing proofs: %w", err)
}
for id, proof := range proofs {
err := util.LdWrite(w, id.Bytes(), proof)
if err != nil {
return fmt.Errorf("writing proof to the car: %w", err)
}
}
return nil
}
func getProofs(ctx context.Context, eds *rsmt2d.ExtendedDataSquare) (map[cid.Cid][]byte, error) {
// check if there are proofs collected by ipld.ProofsAdder in previous reconstruction of eds
if adder := ipld.ProofsAdderFromCtx(ctx); adder != nil {
defer adder.Purge()
return adder.Proofs(), nil
}
// recompute proofs from eds
shares := eds.Flattened()
shareCount := len(shares)
if shareCount == 0 {
return nil, ErrEmptySquare
}
odsWidth := int(math.Sqrt(float64(shareCount)) / 2)
// this adder ignores leaves, so that they are not added to the store we iterate through in
// writeProofs
adder := ipld.NewProofsAdder(odsWidth * 2)
defer adder.Purge()
eds, err := rsmt2d.ImportExtendedDataSquare(
shares,
share.DefaultRSMT2DCodec(),
wrapper.NewConstructor(uint64(odsWidth),
nmt.NodeVisitor(adder.VisitFn())),
)
if err != nil {
return nil, fmt.Errorf("recomputing data square: %w", err)
}
// compute roots
if _, err = eds.RowRoots(); err != nil {
return nil, fmt.Errorf("computing row roots: %w", err)
}
return adder.Proofs(), nil
}
// quadrantOrder reorders the shares in the EDS to quadrant row-by-row order, prepending the
// respective namespace to the shares.
// e.g. [ Q1 R1 | Q1 R2 | Q1 R3 | Q1 R4 | Q2 R1 | Q2 R2 .... ]
func quadrantOrder(eds *rsmt2d.ExtendedDataSquare) [][]byte {
size := eds.Width() * eds.Width()
shares := make([][]byte, size)
quadrantWidth := int(eds.Width() / 2)
quadrantSize := quadrantWidth * quadrantWidth
for i := 0; i < quadrantWidth; i++ {
for j := 0; j < quadrantWidth; j++ {
cells := getQuadrantCells(eds, uint(i), uint(j))
innerOffset := i*quadrantWidth + j
for quadrant := 0; quadrant < 4; quadrant++ {
shares[(quadrant*quadrantSize)+innerOffset] = prependNamespace(quadrant, cells[quadrant])
}
}
}
return shares
}
// getQuadrantCells returns the cell of each EDS quadrant with the passed inner-quadrant coordinates
func getQuadrantCells(eds *rsmt2d.ExtendedDataSquare, i, j uint) [][]byte {
cells := make([][]byte, 4)
quadrantWidth := eds.Width() / 2
cells[0] = eds.GetCell(i, j)
cells[1] = eds.GetCell(i, j+quadrantWidth)
cells[2] = eds.GetCell(i+quadrantWidth, j)
cells[3] = eds.GetCell(i+quadrantWidth, j+quadrantWidth)
return cells
}
// prependNamespace adds the namespace to the passed share if in the first quadrant,
// otherwise it adds the ParitySharesNamespace to the beginning.
func prependNamespace(quadrant int, shr share.Share) []byte {
namespacedShare := make([]byte, 0, share.NamespaceSize+share.Size)
switch quadrant {
case 0:
return append(append(namespacedShare, share.GetNamespace(shr)...), shr...)
case 1, 2, 3:
return append(append(namespacedShare, share.ParitySharesNamespace...), shr...)
default:
panic("invalid quadrant")
}
}
// rootsToCids converts the EDS's Row and Column roots to CIDs.
func rootsToCids(eds *rsmt2d.ExtendedDataSquare) ([]cid.Cid, error) {
rowRoots, err := eds.RowRoots()
if err != nil {
return nil, err
}
colRoots, err := eds.ColRoots()
if err != nil {
return nil, err
}
roots := make([][]byte, 0, len(rowRoots)+len(colRoots))
roots = append(roots, rowRoots...)
roots = append(roots, colRoots...)
rootCids := make([]cid.Cid, len(roots))
for i, r := range roots {
rootCids[i], err = ipld.CidFromNamespacedSha256(r)
if err != nil {
return nil, fmt.Errorf("getting cid from root: %w", err)
}
}
return rootCids, nil
}
// ReadEDS reads the first EDS quadrant (1/4) from an io.Reader CAR file.
// Only the first quadrant will be read, which represents the original data.
// The returned EDS is guaranteed to be full and valid against the DataRoot, otherwise ReadEDS
// errors.
func ReadEDS(ctx context.Context, r io.Reader, root share.DataHash) (eds *rsmt2d.ExtendedDataSquare, err error) {
_, span := tracer.Start(ctx, "read-eds")
defer func() {
utils.SetStatusAndEnd(span, err)
}()
carReader, err := car.NewCarReader(r)
if err != nil {
return nil, fmt.Errorf("share: reading car file: %w", err)
}
// car header includes both row and col roots in header
odsWidth := len(carReader.Header.Roots) / 4
odsSquareSize := odsWidth * odsWidth
shares := make([][]byte, odsSquareSize)
// the first quadrant is stored directly after the header,
// so we can just read the first odsSquareSize blocks
for i := 0; i < odsSquareSize; i++ {
block, err := carReader.Next()
if err != nil {
return nil, fmt.Errorf("share: reading next car entry: %w", err)
}
// the stored first quadrant shares are wrapped with the namespace twice.
// we cut it off here, because it is added again while importing to the tree below
shares[i] = share.GetData(block.RawData())
}
// use proofs adder if provided, to cache collected proofs while recomputing the eds
var opts []nmt.Option
visitor := ipld.ProofsAdderFromCtx(ctx).VisitFn()
if visitor != nil {
opts = append(opts, nmt.NodeVisitor(visitor))
}
eds, err = rsmt2d.ComputeExtendedDataSquare(
shares,
share.DefaultRSMT2DCodec(),
wrapper.NewConstructor(uint64(odsWidth), opts...),
)
if err != nil {
return nil, fmt.Errorf("share: computing eds: %w", err)
}
newDah, err := da.NewDataAvailabilityHeader(eds)
if err != nil {
return nil, err
}
if !bytes.Equal(newDah.Hash(), root) {
return nil, fmt.Errorf(
"share: content integrity mismatch: imported root %s doesn't match expected root %s",
newDah.Hash(),
root,
)
}
return eds, nil
}