-
Notifications
You must be signed in to change notification settings - Fork 5
/
directory.go
147 lines (139 loc) · 3.83 KB
/
directory.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
package builder
import (
"fmt"
"io/fs"
"os"
"path"
"github.com/ipfs/go-unixfsnode/data"
dagpb "github.com/ipld/go-codec-dagpb"
"github.com/ipld/go-ipld-prime"
cidlink "github.com/ipld/go-ipld-prime/linking/cid"
"github.com/multiformats/go-multihash"
)
// https://github.com/ipfs/go-ipfs/pull/8114/files#diff-eec963b47a6e1080d9d8023b4e438e6e3591b4154f7379a7e728401d2055374aR319
const shardSplitThreshold = 262144
// https://github.com/ipfs/go-unixfs/blob/ec6bb5a4c5efdc3a5bce99151b294f663ee9c08d/io/directory.go#L29
const defaultShardWidth = 256
// BuildUnixFSRecursive returns a link pointing to the UnixFS node representing
// the file or directory tree pointed to by `root`
func BuildUnixFSRecursive(root string, ls *ipld.LinkSystem) (ipld.Link, uint64, error) {
info, err := os.Lstat(root)
if err != nil {
return nil, 0, err
}
m := info.Mode()
switch {
case m.IsDir():
var tsize uint64
entries, err := os.ReadDir(root)
if err != nil {
return nil, 0, err
}
lnks := make([]dagpb.PBLink, 0, len(entries))
for _, e := range entries {
lnk, sz, err := BuildUnixFSRecursive(path.Join(root, e.Name()), ls)
if err != nil {
return nil, 0, err
}
tsize += sz
entry, err := BuildUnixFSDirectoryEntry(e.Name(), int64(sz), lnk)
if err != nil {
return nil, 0, err
}
lnks = append(lnks, entry)
}
return BuildUnixFSDirectory(lnks, ls)
case m.Type() == fs.ModeSymlink:
content, err := os.Readlink(root)
if err != nil {
return nil, 0, err
}
outLnk, sz, err := BuildUnixFSSymlink(content, ls)
if err != nil {
return nil, 0, err
}
return outLnk, sz, nil
case m.IsRegular():
fp, err := os.Open(root)
if err != nil {
return nil, 0, err
}
defer fp.Close()
outLnk, sz, err := BuildUnixFSFile(fp, "", ls)
if err != nil {
return nil, 0, err
}
return outLnk, sz, nil
default:
return nil, 0, fmt.Errorf("cannot encode non regular file: %s", root)
}
}
// estimateDirSize estimates if a directory is big enough that it warrents sharding.
// The estimate is the sum over the len(linkName) + bytelen(linkHash)
// https://github.com/ipfs/go-unixfs/blob/master/io/directory.go#L152-L162
func estimateDirSize(entries []dagpb.PBLink) int {
s := 0
for _, e := range entries {
s += len(e.Name.Must().String())
lnk := e.Hash.Link()
cl, ok := lnk.(cidlink.Link)
if ok {
s += cl.ByteLen()
} else if lnk == nil {
s += 0
} else {
s += len(lnk.Binary())
}
}
return s
}
// BuildUnixFSDirectory creates a directory link over a collection of entries.
func BuildUnixFSDirectory(entries []dagpb.PBLink, ls *ipld.LinkSystem) (ipld.Link, uint64, error) {
if estimateDirSize(entries) > shardSplitThreshold {
return BuildUnixFSShardedDirectory(defaultShardWidth, multihash.MURMUR3X64_64, entries, ls)
}
ufd, err := BuildUnixFS(func(b *Builder) {
DataType(b, data.Data_Directory)
})
if err != nil {
return nil, 0, err
}
pbb := dagpb.Type.PBNode.NewBuilder()
pbm, err := pbb.BeginMap(2)
if err != nil {
return nil, 0, err
}
if err = pbm.AssembleKey().AssignString("Data"); err != nil {
return nil, 0, err
}
if err = pbm.AssembleValue().AssignBytes(data.EncodeUnixFSData(ufd)); err != nil {
return nil, 0, err
}
if err = pbm.AssembleKey().AssignString("Links"); err != nil {
return nil, 0, err
}
lnks, err := pbm.AssembleValue().BeginList(int64(len(entries)))
if err != nil {
return nil, 0, err
}
// sorting happens in codec-dagpb
var totalSize uint64
for _, e := range entries {
totalSize += uint64(e.Tsize.Must().Int())
if err := lnks.AssembleValue().AssignNode(e); err != nil {
return nil, 0, err
}
}
if err := lnks.Finish(); err != nil {
return nil, 0, err
}
if err := pbm.Finish(); err != nil {
return nil, 0, err
}
node := pbb.Build()
lnk, sz, err := sizedStore(ls, fileLinkProto, node)
if err != nil {
return nil, 0, err
}
return lnk, totalSize + sz, err
}