-
Notifications
You must be signed in to change notification settings - Fork 1.2k
/
systar.go
163 lines (132 loc) · 4.57 KB
/
systar.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
package tarutil
import (
"archive/tar"
"io"
"os"
"path/filepath"
"strings"
logging "github.com/ipfs/go-log/v2"
"golang.org/x/xerrors"
)
var log = logging.Logger("tarutil") // nolint
var CacheFileConstraints = map[string]int64{
"p_aux": 64,
"t_aux": 10240,
"sc-02-data-tree-r-last.dat": 10_000_000, // small sectors
"sc-02-data-tree-r-last-0.dat": 10_000_000,
"sc-02-data-tree-r-last-1.dat": 10_000_000,
"sc-02-data-tree-r-last-2.dat": 10_000_000,
"sc-02-data-tree-r-last-3.dat": 10_000_000,
"sc-02-data-tree-r-last-4.dat": 10_000_000,
"sc-02-data-tree-r-last-5.dat": 10_000_000,
"sc-02-data-tree-r-last-6.dat": 10_000_000,
"sc-02-data-tree-r-last-7.dat": 10_000_000,
"sc-02-data-tree-r-last-8.dat": 10_000_000,
"sc-02-data-tree-r-last-9.dat": 10_000_000,
"sc-02-data-tree-r-last-10.dat": 10_000_000,
"sc-02-data-tree-r-last-11.dat": 10_000_000,
"sc-02-data-tree-r-last-12.dat": 10_000_000,
"sc-02-data-tree-r-last-13.dat": 10_000_000,
"sc-02-data-tree-r-last-14.dat": 10_000_000,
"sc-02-data-tree-r-last-15.dat": 10_000_000,
"sc-02-data-layer-1.dat": 65 << 30, // 1x sector size + small buffer
"sc-02-data-layer-2.dat": 65 << 30, // 1x sector size + small buffer
"sc-02-data-layer-3.dat": 65 << 30, // 1x sector size + small buffer
"sc-02-data-layer-4.dat": 65 << 30, // 1x sector size + small buffer
"sc-02-data-layer-5.dat": 65 << 30, // 1x sector size + small buffer
"sc-02-data-layer-6.dat": 65 << 30, // 1x sector size + small buffer
"sc-02-data-layer-7.dat": 65 << 30, // 1x sector size + small buffer
"sc-02-data-layer-8.dat": 65 << 30, // 1x sector size + small buffer
"sc-02-data-layer-9.dat": 65 << 30, // 1x sector size + small buffer
"sc-02-data-layer-10.dat": 65 << 30, // 1x sector size + small buffer
"sc-02-data-layer-11.dat": 65 << 30, // 1x sector size + small buffer
"sc-02-data-tree-c-0.dat": 5 << 30, // ~4.6G
"sc-02-data-tree-c-1.dat": 5 << 30,
"sc-02-data-tree-c-2.dat": 5 << 30,
"sc-02-data-tree-c-3.dat": 5 << 30,
"sc-02-data-tree-c-4.dat": 5 << 30,
"sc-02-data-tree-c-5.dat": 5 << 30,
"sc-02-data-tree-c-6.dat": 5 << 30,
"sc-02-data-tree-c-7.dat": 5 << 30,
"sc-02-data-tree-c-8.dat": 5 << 30,
"sc-02-data-tree-c-9.dat": 5 << 30,
"sc-02-data-tree-c-10.dat": 5 << 30,
"sc-02-data-tree-c-11.dat": 5 << 30,
"sc-02-data-tree-c-12.dat": 5 << 30,
"sc-02-data-tree-c-13.dat": 5 << 30,
"sc-02-data-tree-c-14.dat": 5 << 30,
"sc-02-data-tree-c-15.dat": 5 << 30,
"sc-02-data-tree-d.dat": 130 << 30, // 2x sector size, ~130G accunting for small buffer on 64G sectors
}
func ExtractTar(body io.Reader, dir string, buf []byte) (int64, error) {
if err := os.MkdirAll(dir, 0755); err != nil { // nolint
return 0, xerrors.Errorf("mkdir: %w", err)
}
tr := tar.NewReader(body)
var read int64
for {
header, err := tr.Next()
switch err {
default:
return read, err
case io.EOF:
return read, nil
case nil:
}
sz, found := CacheFileConstraints[header.Name]
if !found {
return read, xerrors.Errorf("tar file %#v isn't expected")
}
if header.Size > sz {
return read, xerrors.Errorf("tar file %#v is bigger than expected: %d > %d", header.Name, header.Size, sz)
}
out := filepath.Join(dir, header.Name) //nolint:gosec
if !strings.HasPrefix(out, filepath.Clean(dir)) {
return read, xerrors.Errorf("unsafe tar path %#v (must be within %#v)", out, filepath.Clean(dir))
}
f, err := os.Create(out)
if err != nil {
return read, xerrors.Errorf("creating file %s: %w", out, err)
}
ltr := io.LimitReader(tr, header.Size)
r, err := io.CopyBuffer(f, ltr, buf)
read += r
if err != nil {
return read, err
}
if err := f.Close(); err != nil {
return read, err
}
}
}
func TarDirectory(dir string, w io.Writer, buf []byte) error {
tw := tar.NewWriter(w)
files, err := os.ReadDir(dir)
if err != nil {
return err
}
for _, file := range files {
info, err := file.Info()
if err != nil {
return xerrors.Errorf("getting file info for file %s: %w", file.Name(), err)
}
h, err := tar.FileInfoHeader(info, "")
if err != nil {
return xerrors.Errorf("getting header for file %s: %w", file.Name(), err)
}
if err := tw.WriteHeader(h); err != nil {
return xerrors.Errorf("wiritng header for file %s: %w", file.Name(), err)
}
f, err := os.OpenFile(filepath.Join(dir, file.Name()), os.O_RDONLY, 644) // nolint
if err != nil {
return xerrors.Errorf("opening %s for reading: %w", file.Name(), err)
}
if _, err := io.CopyBuffer(tw, f, buf); err != nil {
return xerrors.Errorf("copy data for file %s: %w", file.Name(), err)
}
if err := f.Close(); err != nil {
return err
}
}
return nil
}