forked from alanshaw/go-carbites
/
simple.go
88 lines (80 loc) 路 1.68 KB
/
simple.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
package carbites
import (
"bufio"
"bytes"
"io"
"github.com/ipfs/go-cid"
car "github.com/ipld/go-car"
util "github.com/ipld/go-car/util"
)
var emptyHd *car.CarHeader
func init() {
emptyCid, _ := cid.Decode("bafkqaaa")
emptyHd = &car.CarHeader{
Roots: []cid.Cid{emptyCid},
Version: 1,
}
}
type SimpleSplitter struct {
reader *car.CarReader
header *car.CarHeader
targetSize int
}
// Create a new CAR file splitter to create multiple smaller CAR files using the
// "simple" strategy.
func NewSimpleSplitter(in io.Reader, targetSize int) (*SimpleSplitter, error) {
r, err := car.NewCarReader(in)
if err != nil {
return nil, err
}
h := r.Header
return &SimpleSplitter{r, h, targetSize}, nil
}
func (spltr *SimpleSplitter) Next() (io.Reader, error) {
var b []byte
buf := bytes.NewBuffer(b)
err := car.WriteHeader(spltr.header, buf)
if err != nil {
return nil, err
}
spltr.header = emptyHd
empty := true
total := buf.Len()
for {
bl, err := spltr.reader.Next()
if err != nil {
if err == io.EOF {
break
}
return nil, err
}
empty = false
util.LdWrite(buf, bl.Cid().Bytes(), bl.RawData())
total += len(bl.RawData())
if total >= spltr.targetSize {
break
}
}
if empty {
return nil, io.EOF
}
return buf, nil
}
// Join together multiple CAR files that were split using the "simple" strategy
// into a single CAR file.
func JoinSimple(in []io.Reader) (io.Reader, error) {
var brs []io.Reader
for i, r := range in {
br := bufio.NewReader(r)
brs = append(brs, br)
if i == 0 {
continue
}
// discard header from other CARs
_, err := util.LdRead(br)
if err != nil {
return nil, err
}
}
return io.MultiReader(brs...), nil
}