-
-
Notifications
You must be signed in to change notification settings - Fork 12
/
s3fscache.go
158 lines (130 loc) · 3.41 KB
/
s3fscache.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
package s3browser
import (
"path"
"strings"
"sync"
"time"
"github.com/dustin/go-humanize"
"github.com/minio/minio-go/v6"
"go.uber.org/zap"
)
type S3FsCache struct {
lock sync.RWMutex
s3 S3Client
sorter *S3FsSorter
logger *zap.Logger
bucket string
data map[string]Directory
}
type Directory struct {
Path string
Folders []string
Filenames []string
files map[string]File
}
type File struct {
Bytes int64
Date time.Time
}
// Caller must ensure file exists.
func (d Directory) GetFile(fileName string) File {
return d.files[fileName]
}
// HumanSize returns the size of the file as a human-readable string
// in IEC format (i.e. power of 2 or base 1024).
func (f File) HumanSize() string {
return humanize.IBytes(uint64(f.Bytes))
}
// HumanModTime returns the modified time of the file as a human-readable string.
func (f File) HumanModTime(format string) string {
return f.Date.Format(format)
}
func NewS3FsCache(client S3Client, sorter *S3FsSorter, l *zap.Logger) S3FsCache {
return S3FsCache{
s3: client,
sorter: sorter,
logger: l,
}
}
func (fs *S3FsCache) GetDir(dirPath string) (Directory, bool) {
dir, ok := fs.data[normalizePath(dirPath)]
return dir, ok
}
func (fs *S3FsCache) GetFile(filePath string) (File, bool) {
dirPath, fileName := path.Split(filePath)
dirPath = normalizePath(dirPath)
dir, ok := fs.GetDir(dirPath)
if !ok {
return File{}, false
}
file := dir.GetFile(fileName)
return file, file != File{}
}
func (fs *S3FsCache) Refresh() (err error) {
fs.logger.Info("Refreshing S3 cache")
newData := map[string]Directory{}
addDirectory(fs.logger, newData, "/")
fs.s3.ForEachObject(func(obj minio.ObjectInfo) {
objDir, objName := path.Split(obj.Key)
objDir = normalizePath(objDir)
// Add any missing parent directories in `newData`
if _, ok := newData[objDir]; !ok {
addDirectory(fs.logger, newData, objDir)
}
// Add the object
if objName != "" { // "": obj is the directory itself
fs.logger.Debug("file", zap.String("dir", objDir), zap.String("name", objName))
fsCopy := newData[objDir]
fsCopy.files[objName] = File{
Bytes: obj.Size,
Date: obj.LastModified,
}
fsCopy.Filenames = append(fsCopy.Filenames, objName)
newData[objDir] = fsCopy
}
})
if fs.sorter != nil {
for _, dir := range newData {
fs.sorter.Sort(dir.Folders)
fs.sorter.Sort(dir.Filenames)
}
}
fs.data = newData
fs.logger.Info("S3 cache updated")
return nil
}
// Ensure path starts with / and doesn't end with one
func normalizePath(p string) string {
if p == "" {
return "/"
}
return "/" + strings.Trim(path.Clean(p), "/")
}
// Add directory
// `dirPath` must be normalized
func addDirectory(logger *zap.Logger, outData map[string]Directory, dirPath string) {
// Split dirPath into its path components
dirs := strings.Split(dirPath[1:], "/") // [1:]: skip leading /
parentPath := "/"
for _, curr := range dirs {
currPath := path.Join(parentPath, curr)
if _, ok := outData[currPath]; !ok {
logger.Debug("dir", zap.String("path", currPath))
// Add to parent Node
parentNode := outData[parentPath]
parentNode.Folders = append(parentNode.Folders, curr)
outData[parentPath] = parentNode
// Add own Node
outData[currPath] = Directory{
Path: currPath,
Folders: []string{},
Filenames: []string{},
files: map[string]File{},
}
}
if parentPath != "/" {
parentPath += "/"
}
parentPath += curr
}
}