Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

S3 tests #3713

Merged
merged 5 commits into from
Aug 21, 2023
Merged

S3 tests #3713

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
f78d81e
Remove test as S3 does not support empty directories
Jamstah Aug 12, 2022
2d316a1
We don't use gocheck in these tests
Jamstah Aug 12, 2022
6ceb904
Don't check returned storage class if we use NONE
Jamstah Aug 12, 2022
f7bdd91
Don't test the OUTPOSTS storage class
Jamstah Aug 12, 2022
7622d0a
Don't return the from of a walk
Jamstah Aug 12, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 14 additions & 14 deletions registry/storage/driver/s3-aws/s3.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1039,18 +1039,8 @@ func (d *driver) URLFor(ctx context.Context, path string, options map[string]int
// Walk traverses a filesystem defined within driver, starting
// from the given path, calling f on each file
func (d *driver) Walk(ctx context.Context, from string, f storagedriver.WalkFn) error {
path := from
if !strings.HasSuffix(path, "/") {
path = path + "/"
}

prefix := ""
if d.s3Path("") == "" {
prefix = "/"
}

var objectCount int64
if err := d.doWalk(ctx, &objectCount, d.s3Path(path), prefix, f); err != nil {
if err := d.doWalk(ctx, &objectCount, from, f); err != nil {
return err
}

Expand All @@ -1062,19 +1052,29 @@ func (d *driver) Walk(ctx context.Context, from string, f storagedriver.WalkFn)
return nil
}

func (d *driver) doWalk(parentCtx context.Context, objectCount *int64, path, prefix string, f storagedriver.WalkFn) error {
func (d *driver) doWalk(parentCtx context.Context, objectCount *int64, from string, f storagedriver.WalkFn) error {
var (
retError error
// the most recent directory walked for de-duping
prevDir string
// the most recent skip directory to avoid walking over undesirable files
prevSkipDir string
)
prevDir = strings.Replace(path, d.s3Path(""), prefix, 1)
prevDir = from

path := from
if !strings.HasSuffix(path, "/") {
path = path + "/"
}

prefix := ""
if d.s3Path("") == "" {
prefix = "/"
}

listObjectsInput := &s3.ListObjectsV2Input{
Bucket: aws.String(d.Bucket),
Prefix: aws.String(path),
Prefix: aws.String(d.s3Path(path)),
MaxKeys: aws.Int64(listMax),
}

Expand Down
54 changes: 10 additions & 44 deletions registry/storage/driver/s3-aws/s3_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,6 @@ import (
"strings"
"testing"

"gopkg.in/check.v1"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"

Expand All @@ -23,13 +21,8 @@ import (
"github.com/distribution/distribution/v3/registry/storage/driver/testsuites"
)

// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) { check.TestingT(t) }

var (
s3DriverConstructor func(rootDirectory, storageClass string) (*Driver, error)
skipS3 func() string
)
var s3DriverConstructor func(rootDirectory, storageClass string) (*Driver, error)
var skipS3 func() string
Jamstah marked this conversation as resolved.
Show resolved Hide resolved

func init() {
var (
Expand Down Expand Up @@ -205,39 +198,6 @@ func TestEmptyRootList(t *testing.T) {
}
}

// TestWalkEmptySubDirectory assures we list an empty sub directory only once when walking
// through its parent directory.
func TestWalkEmptySubDirectory(t *testing.T) {
if skipS3() != "" {
t.Skip(skipS3())
}

drv, err := s3DriverConstructor("", s3.StorageClassStandard)
if err != nil {
t.Fatalf("unexpected error creating rooted driver: %v", err)
}

// create an empty sub directory.
s3driver := drv.StorageDriver.(*driver)
if _, err := s3driver.S3.PutObject(&s3.PutObjectInput{
Bucket: aws.String(os.Getenv("S3_BUCKET")),
Key: aws.String("/testdir/emptydir/"),
}); err != nil {
t.Fatalf("error creating empty directory: %s", err)
}

bucketFiles := []string{}
s3driver.Walk(context.Background(), "/testdir", func(fileInfo storagedriver.FileInfo) error {
bucketFiles = append(bucketFiles, fileInfo.Path())
return nil
})

expected := []string{"/testdir/emptydir"}
if !reflect.DeepEqual(bucketFiles, expected) {
t.Errorf("expecting files %+v, found %+v instead", expected, bucketFiles)
}
}

func TestStorageClass(t *testing.T) {
if skipS3() != "" {
t.Skip(skipS3())
Expand All @@ -253,6 +213,11 @@ func TestStorageClass(t *testing.T) {
t.Fatalf("unexpected error creating driver with storage class %v: %v", storageClass, err)
}

// Can only test outposts if using s3 outposts
if storageClass == s3.StorageClassOutposts {
continue
}

err = s3Driver.PutContent(ctx, filename, contents)
if err != nil {
t.Fatalf("unexpected error creating content with storage class %v: %v", storageClass, err)
Expand All @@ -269,7 +234,9 @@ func TestStorageClass(t *testing.T) {
}
defer resp.Body.Close()
// Amazon only populates this header value for non-standard storage classes
if storageClass == s3.StorageClassStandard && resp.StorageClass != nil {
if storageClass == noStorageClass {
// We haven't specified a storage class so we can't confirm what it is
} else if storageClass == s3.StorageClassStandard && resp.StorageClass != nil {
t.Fatalf(
"unexpected response storage class for file with storage class %v: %v",
storageClass,
Expand Down Expand Up @@ -637,7 +604,6 @@ func TestWalk(t *testing.T) {
name: "from folder",
fn: func(fileInfo storagedriver.FileInfo) error { return nil },
expected: []string{
"/folder1",
"/folder1/file1",
},
from: "/folder1",
Expand Down