forked from ipfs/kubo
/
importer.go
95 lines (78 loc) · 2.18 KB
/
importer.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
// package importer implements utilities used to create ipfs DAGs from files
// and readers
package importer
import (
"fmt"
"os"
"github.com/ipfs/go-ipfs/commands/files"
bal "github.com/ipfs/go-ipfs/importer/balanced"
"github.com/ipfs/go-ipfs/importer/chunk"
h "github.com/ipfs/go-ipfs/importer/helpers"
trickle "github.com/ipfs/go-ipfs/importer/trickle"
dag "github.com/ipfs/go-ipfs/merkledag"
"github.com/ipfs/go-ipfs/pin"
logging "github.com/ipfs/go-ipfs/vendor/QmQg1J6vikuXF9oDvm4wpdeAUvvkVEKW1EYDw9HhTMnP2b/go-log"
)
var log = logging.Logger("importer")
// Builds a DAG from the given file, writing created blocks to disk as they are
// created
func BuildDagFromFile(fpath string, ds dag.DAGService, mp pin.ManualPinner) (*dag.Node, error) {
stat, err := os.Lstat(fpath)
if err != nil {
return nil, err
}
if stat.IsDir() {
return nil, fmt.Errorf("`%s` is a directory", fpath)
}
f, err := files.NewSerialFile(fpath, fpath, stat)
if err != nil {
return nil, err
}
defer f.Close()
return BuildDagFromReader(ds, chunk.NewSizeSplitter(f, chunk.DefaultBlockSize), BasicPinnerCB(mp))
}
func BuildDagFromReader(ds dag.DAGService, spl chunk.Splitter, ncb h.NodeCB) (*dag.Node, error) {
// Start the splitter
blkch, errch := chunk.Chan(spl)
dbp := h.DagBuilderParams{
Dagserv: ds,
Maxlinks: h.DefaultLinksPerBlock,
NodeCB: ncb,
}
return bal.BalancedLayout(dbp.New(blkch, errch))
}
func BuildTrickleDagFromReader(ds dag.DAGService, spl chunk.Splitter, ncb h.NodeCB) (*dag.Node, error) {
// Start the splitter
blkch, errch := chunk.Chan(spl)
dbp := h.DagBuilderParams{
Dagserv: ds,
Maxlinks: h.DefaultLinksPerBlock,
NodeCB: ncb,
}
return trickle.TrickleLayout(dbp.New(blkch, errch))
}
func BasicPinnerCB(p pin.ManualPinner) h.NodeCB {
return func(n *dag.Node, last bool) error {
k, err := n.Key()
if err != nil {
return err
}
if last {
p.PinWithMode(k, pin.Recursive)
return p.Flush()
} else {
p.PinWithMode(k, pin.Indirect)
return nil
}
}
}
func PinIndirectCB(p pin.ManualPinner) h.NodeCB {
return func(n *dag.Node, last bool) error {
k, err := n.Key()
if err != nil {
return err
}
p.PinWithMode(k, pin.Indirect)
return nil
}
}