-
Notifications
You must be signed in to change notification settings - Fork 342
/
gcs.go
73 lines (62 loc) · 1.45 KB
/
gcs.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
package store
import (
"context"
"encoding/hex"
"errors"
"fmt"
"net/url"
"strings"
"cloud.google.com/go/storage"
"google.golang.org/api/iterator"
)
type gcsWalker struct {
client *storage.Client
mark Mark
}
func NewGCSWalker(client *storage.Client) *gcsWalker {
return &gcsWalker{client: client}
}
func (w *gcsWalker) Walk(ctx context.Context, storageURI *url.URL, op WalkOptions, walkFn func(e ObjectStoreEntry) error) error {
prefix := strings.TrimLeft(storageURI.Path, "/")
iter := w.client.
Bucket(storageURI.Host).
Objects(ctx, &storage.Query{
Prefix: prefix,
StartOffset: op.After,
})
for {
attrs, err := iter.Next()
if errors.Is(err, iterator.Done) {
break
}
if err != nil {
return fmt.Errorf("error listing objects at storage uri %s: %w", storageURI, err)
}
// skipping first key (without forgetting the possible empty string key!)
if op.After != "" && attrs.Name <= op.After {
continue
}
w.mark = Mark{
LastKey: attrs.Name,
HasMore: true,
}
if err := walkFn(ObjectStoreEntry{
FullKey: attrs.Name,
RelativeKey: strings.TrimPrefix(attrs.Name, prefix),
Address: fmt.Sprintf("gs://%s/%s", attrs.Bucket, attrs.Name),
ETag: hex.EncodeToString(attrs.MD5),
Mtime: attrs.Updated,
Size: attrs.Size,
}); err != nil {
return err
}
}
w.mark = Mark{
LastKey: "",
HasMore: false,
}
return nil
}
func (w *gcsWalker) Marker() Mark {
return w.mark
}