From d411f709087f1c79b2fdf007779d30c7bd21dfd2 Mon Sep 17 00:00:00 2001 From: Katrina Rogan Date: Wed, 10 Mar 2021 15:23:02 -0800 Subject: [PATCH] Always try to create a bucket when loading a container (#76) --- storage/stow_store.go | 38 +++++----------- storage/stow_store_test.go | 91 +++++++++++++++++++++++++++++++++++++- 2 files changed, 102 insertions(+), 27 deletions(-) diff --git a/storage/stow_store.go b/storage/stow_store.go index 6f856ca542..b484bc2588 100644 --- a/storage/stow_store.go +++ b/storage/stow_store.go @@ -53,19 +53,6 @@ var fQNFn = map[string]func(string) DataReference{ }, } -// Checks if the error is AWS S3 bucket not found error -func awsBucketIsNotFound(err error) bool { - if IsNotFound(err) { - return true - } - - if awsErr, errOk := errs.Cause(err).(awserr.Error); errOk { - return awsErr.Code() == s32.ErrCodeNoSuchBucket - } - - return false -} - // Checks if the error is AWS S3 bucket already exists error. func awsBucketAlreadyExists(err error) bool { if IsExists(err) { @@ -124,21 +111,20 @@ type StowStore struct { } func (s *StowStore) LoadContainer(ctx context.Context, container string, createIfNotFound bool) (stow.Container, error) { + // TODO: As of stow v0.2.6 elides the container lookup when a bucket region is set, + // so we always just attempt to create it when createIfNotFound is true. + + if createIfNotFound { + logger.Infof(ctx, "Attempting to create container [%s]", container) + _, err := s.loc.CreateContainer(container) + if err != nil && !awsBucketAlreadyExists(err) && !IsExists(err) { + return nil, fmt.Errorf("unable to initialize container [%v]. Error: %v", container, err) + } + } + c, err := s.loc.Container(container) if err != nil { - if createIfNotFound { - logger.Infof(ctx, "Container [%s] lookup failed, err [%s], will try to create a new one", container, err) - if IsNotFound(err) || awsBucketIsNotFound(err) { - c, err := s.loc.CreateContainer(container) - // If the container's already created, move on. Otherwise, fail with error. - if err != nil && !awsBucketAlreadyExists(err) && !IsExists(err) { - return nil, fmt.Errorf("unable to initialize container [%v]. Error: %v", container, err) - } - return c, nil - } - } else { - logger.Errorf(ctx, "Container [%s] lookup failed. Error %s", container, err) - } + logger.Errorf(ctx, "Container [%s] lookup failed. Error %s", container, err) return nil, err } return c, nil diff --git a/storage/stow_store_test.go b/storage/stow_store_test.go index 2b18e966a2..c3131eb759 100644 --- a/storage/stow_store_test.go +++ b/storage/stow_store_test.go @@ -28,13 +28,18 @@ import ( type mockStowLoc struct { stow.Location - ContainerCb func(id string) (stow.Container, error) + ContainerCb func(id string) (stow.Container, error) + CreateContainerCb func(name string) (stow.Container, error) } func (m mockStowLoc) Container(id string) (stow.Container, error) { return m.ContainerCb(id) } +func (m mockStowLoc) CreateContainer(name string) (stow.Container, error) { + return m.CreateContainerCb(name) +} + type mockStowContainer struct { id string items map[string]mockStowItem @@ -133,6 +138,12 @@ func TestStowStore_ReadRaw(t *testing.T) { } return nil, fmt.Errorf("container is not supported") }, + CreateContainerCb: func(name string) (stow.Container, error) { + if name == container { + return newMockStowContainer(container), nil + } + return nil, fmt.Errorf("container is not supported") + }, }, false, testScope) assert.NoError(t, err) err = s.WriteRaw(context.TODO(), DataReference("s3://container/path"), 0, Options{}, bytes.NewReader([]byte{})) @@ -158,6 +169,12 @@ func TestStowStore_ReadRaw(t *testing.T) { } return nil, fmt.Errorf("container is not supported") }, + CreateContainerCb: func(name string) (stow.Container, error) { + if name == container { + return newMockStowContainer(container), nil + } + return nil, fmt.Errorf("container is not supported") + }, }, false, testScope) assert.NoError(t, err) err = s.WriteRaw(context.TODO(), DataReference("s3://container/path"), 3*MiB, Options{}, bytes.NewReader([]byte{})) @@ -183,6 +200,12 @@ func TestStowStore_ReadRaw(t *testing.T) { } return nil, fmt.Errorf("container is not supported") }, + CreateContainerCb: func(name string) (stow.Container, error) { + if name == container { + return newMockStowContainer(container), nil + } + return nil, fmt.Errorf("container is not supported") + }, }, true, testScope) assert.NoError(t, err) err = s.WriteRaw(context.TODO(), "s3://bad-container/path", 0, Options{}, bytes.NewReader([]byte{})) @@ -209,6 +232,12 @@ func TestStowStore_ReadRaw(t *testing.T) { } return nil, fmt.Errorf("container is not supported") }, + CreateContainerCb: func(name string) (stow.Container, error) { + if name == container { + return newMockStowContainer(container), nil + } + return nil, fmt.Errorf("container is not supported") + }, }, true, testScope) assert.NoError(t, err) err = s.WriteRaw(context.TODO(), "s3://bad-container/path", 0, Options{}, bytes.NewReader([]byte{})) @@ -373,3 +402,63 @@ func Test_newStowRawStore(t *testing.T) { }) } } + +func TestLoadContainer(t *testing.T) { + container := "container" + t.Run("Create if not found", func(t *testing.T) { + stowStore := StowStore{ + loc: &mockStowLoc{ + ContainerCb: func(id string) (stow.Container, error) { + if id == container { + return newMockStowContainer(container), nil + } + return nil, fmt.Errorf("container is not supported") + }, + CreateContainerCb: func(name string) (stow.Container, error) { + if name == container { + return newMockStowContainer(container), nil + } + return nil, fmt.Errorf("container is not supported") + }, + }, + } + stowContainer, err := stowStore.LoadContainer(context.Background(), "container", true) + assert.NoError(t, err) + assert.Equal(t, container, stowContainer.ID()) + }) + t.Run("Create if not found with error", func(t *testing.T) { + stowStore := StowStore{ + loc: &mockStowLoc{ + ContainerCb: func(id string) (stow.Container, error) { + if id == container { + return newMockStowContainer(container), nil + } + return nil, fmt.Errorf("container is not supported") + }, + CreateContainerCb: func(name string) (stow.Container, error) { + if name == container { + return nil, fmt.Errorf("foo") + } + return nil, fmt.Errorf("container is not supported") + }, + }, + } + _, err := stowStore.LoadContainer(context.Background(), "container", true) + assert.EqualError(t, err, "unable to initialize container [container]. Error: foo") + }) + t.Run("No create if not found", func(t *testing.T) { + stowStore := StowStore{ + loc: &mockStowLoc{ + ContainerCb: func(id string) (stow.Container, error) { + if id == container { + return newMockStowContainer(container), nil + } + return nil, fmt.Errorf("container is not supported") + }, + }, + } + stowContainer, err := stowStore.LoadContainer(context.Background(), "container", false) + assert.NoError(t, err) + assert.Equal(t, container, stowContainer.ID()) + }) +}