/
dir_inserter.go
92 lines (80 loc) · 2.16 KB
/
dir_inserter.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
package fileset
import (
"context"
"io"
"path"
"strings"
"github.com/pachyderm/pachyderm/v2/src/internal/errors"
"github.com/pachyderm/pachyderm/v2/src/internal/storage/chunk"
"github.com/pachyderm/pachyderm/v2/src/internal/storage/fileset/index"
)
type dirInserter struct {
x FileSet
lower string
}
// NewDirInserter creates a file set that inserts directory entries.
func NewDirInserter(x FileSet, lower string) FileSet {
return &dirInserter{
x: x,
lower: lower,
}
}
// Iterate calls cb once for every file in lexicographical order by path
func (s *dirInserter) Iterate(ctx context.Context, cb func(File) error, opts ...index.Option) error {
lastPath := ""
var emit func(p string, f File) error
emit = func(p string, f File) error {
parent := Clean(parentOf(p), true)
if lastPath >= parent || p == "/" || parent < s.lower {
if err := cb(f); err != nil {
return err
}
lastPath = p
return nil
}
// need to create entry for parent
df := dirFile{
path: parent,
}
if err := emit(parent, df); err != nil {
return err
}
return emit(p, f)
}
return s.x.Iterate(ctx, func(f File) error {
return emit(f.Index().Path, f)
}, opts...)
}
func (s *dirInserter) IterateDeletes(_ context.Context, _ func(File) error, _ ...index.Option) error {
return errors.Errorf("iterating deletes in a directory inserter file set")
}
func parentOf(x string) string {
x = strings.TrimRight(x, "/")
y := path.Dir(x)
if y == x {
return "/"
}
if !IsDir(y) {
y += "/"
}
return y
}
func (s *dirInserter) Shards(_ context.Context, _ ...index.Option) ([]*index.PathRange, error) {
return nil, errors.Errorf("sharding a directory inserter file set")
}
type dirFile struct {
path string
}
func (d dirFile) Index() *index.Index {
return &index.Index{
Path: d.path,
File: &index.File{},
}
}
func (d dirFile) Content(_ context.Context, _ io.Writer, _ ...chunk.ReaderOption) error {
return nil
}
func (d dirFile) Hash(_ context.Context) ([]byte, error) {
// TODO: It may make sense to move the generation of directory metadata (size / hash) into the directory inserter.
panic("we should not be using the Hash function for dirFile, this is a bug")
}