-
Notifications
You must be signed in to change notification settings - Fork 3
/
shard_creator.go
62 lines (54 loc) · 1.47 KB
/
shard_creator.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
package rsutils
import (
"crypto/sha256"
"fmt"
"hash"
"io"
"github.com/klauspost/reedsolomon"
)
type ShardCreator struct {
dataSources []io.Reader
size int64
dataShards int
parityShards int
}
func NewShardCreator(src []io.Reader, size int64, dataShards, parityShards int) *ShardCreator {
return &ShardCreator{
dataSources: src,
size: size,
dataShards: dataShards,
parityShards: parityShards,
}
}
func (p *ShardCreator) Encode(parityDst []io.Writer) (*Metadata, error) {
RSEncoder, err := reedsolomon.NewStream(p.dataShards, p.parityShards)
if err != nil {
return nil, fmt.Errorf("Error creating reedsolomon encoder: %s", err)
}
hashers := make([]hash.Hash, p.dataShards+p.parityShards)
for i := range hashers {
hashers[i] = sha256.New()
}
hashingReaders := make([]io.Reader, p.dataShards)
for i := range hashingReaders {
hashingReaders[i] = io.TeeReader(p.dataSources[i], hashers[i])
}
hashingWriters := make([]io.Writer, p.parityShards)
for i := range hashingWriters {
hashingWriters[i] = io.MultiWriter(parityDst[i], hashers[p.dataShards+i])
}
err = RSEncoder.Encode(hashingReaders, hashingWriters)
if err != nil {
return nil, fmt.Errorf("Error encoding: %s", err)
}
hashes := make([]string, len(hashers))
for i := range hashers {
hashes[i] = fmt.Sprintf("%x", hashers[i].Sum(nil))
}
return &Metadata{
Size: p.size,
Hashes: hashes,
DataShards: p.dataShards,
ParityShards: p.parityShards,
}, nil
}