Skip to content

Commit

Permalink
Pass correct prefix to restrict file listing in S3 object storage
Browse files Browse the repository at this point in the history
We had unit tests in place, but our fake storage implementation
did not handle prefix restrictions in its fake `ListObjects()` method.
Fixed the fake storage, so now we’ll detect this problem in unit tests.

#33
  • Loading branch information
brawer committed May 8, 2024
1 parent 4511934 commit 252aa5e
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 2 deletions.
2 changes: 1 addition & 1 deletion cmd/qrank-builder/pageentities.go
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ func buildSitePageEntities(site WikiSite, ctx context.Context, dumps string, s3
func storedPageEntities(ctx context.Context, s3 S3) (map[string][]string, error) {
re := regexp.MustCompile(`^page_entities/([a-z0-9_\-]+)-(\d{8})-page_entities.zst$`)
result := make(map[string][]string, 1000)
opts := minio.ListObjectsOptions{Prefix: "pageviews/"}
opts := minio.ListObjectsOptions{Prefix: "page_entities/"}
for obj := range s3.ListObjects(ctx, "qrank", opts) {
if obj.Err != nil {
return nil, obj.Err
Expand Down
6 changes: 5 additions & 1 deletion cmd/qrank-builder/s3_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"context"
"fmt"
"os"
"strings"

"github.com/minio/minio-go/v7"
)
Expand All @@ -26,9 +27,12 @@ func (s3 *FakeS3) ListObjects(ctx context.Context, bucketName string, opts minio
ch := make(chan minio.ObjectInfo, 2)
go func() {
defer close(ch)
prefix := opts.Prefix
if bucketName == "qrank" {
for key, _ := range s3.data {
ch <- minio.ObjectInfo{Key: key}
if strings.HasPrefix(key, prefix) {
ch <- minio.ObjectInfo{Key: key}
}
}
}
}()
Expand Down

0 comments on commit 252aa5e

Please sign in to comment.