Fail backup if it already exists in object storage

pkg/cloudprovider/aws/object_store.go

@@ -23,7 +23,9 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/endpoints"
+	"github.com/aws/aws-sdk-go/aws/request"
 	v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
@@ -42,10 +44,18 @@ const (
 	signatureVersionKey = "signatureVersion"
 )
 
+type s3Interface interface {
+	HeadObject(input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
+	GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error)
+	ListObjectsV2Pages(input *s3.ListObjectsV2Input, fn func(*s3.ListObjectsV2Output, bool) bool) error
+	DeleteObject(input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error)
+	GetObjectRequest(input *s3.GetObjectInput) (req *request.Request, output *s3.GetObjectOutput)
+}
+
 type ObjectStore struct {
 	log              logrus.FieldLogger
-	s3               *s3.S3
-	preSignS3        *s3.S3
+	s3               s3Interface
+	preSignS3        s3Interface
 	s3Uploader       *s3manager.Uploader
 	kmsKeyID         string
 	signatureVersion string
@@ -191,6 +201,51 @@ func (o *ObjectStore) PutObject(bucket, key string, body io.Reader) error {
 	return errors.Wrapf(err, "error putting object %s", key)
 }
 
+const notFoundCode = "NotFound"
+
+// ObjectExists checks if there is an object with the given key in the object storage bucket.
+func (o *ObjectStore) ObjectExists(bucket, key string) (bool, error) {
+	log := o.log.WithFields(
+		logrus.Fields{
+			"bucket": bucket,
+			"key":    key,
+		},
+	)
+
+	req := &s3.HeadObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	}
+
+	log.Debug("Checking if object exists")
+	_, err := o.s3.HeadObject(req)
+
+	if err == nil {
+		log.Debug("Object exists")
+		return true, nil
+	}
+
+	log.Debug("Checking for AWS specific error information")
+	if aerr, ok := err.(awserr.Error); ok {
+		log.WithFields(
+			logrus.Fields{
+				"code":    aerr.Code(),
+				"message": aerr.Message(),
+			},
+		).Debugf("awserr.Error contents (origErr=%v)", aerr.OrigErr())
+
+		// The code will be NotFound if the key doesn't exist.
+		// See https://github.com/aws/aws-sdk-go/issues/1208 and https://github.com/aws/aws-sdk-go/pull/1213.
+		log.Debugf("Checking for code=%s", notFoundCode)
+		if aerr.Code() == notFoundCode {
+			log.Debug("Object doesn't exist - got not found")
+			return false, nil
+		}
+	}
+
+	return false, errors.WithStack(err)
+}
+
 func (o *ObjectStore) GetObject(bucket, key string) (io.ReadCloser, error) {
 	req := &s3.GetObjectInput{
 		Bucket: &bucket,

pkg/cloudprovider/aws/object_store_test.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2018 the Velero contributors.
+Copyright 2018 the Heptio Ark contributors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -19,11 +19,107 @@ package aws
 import (
 	"testing"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+
+	"github.com/heptio/velero/pkg/util/test"
 )
 
 func TestIsValidSignatureVersion(t *testing.T) {
 	assert.True(t, isValidSignatureVersion("1"))
 	assert.True(t, isValidSignatureVersion("4"))
 	assert.False(t, isValidSignatureVersion("3"))
 }
+
+type mockS3 struct {
+	mock.Mock
+}
+
+func (m *mockS3) HeadObject(input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	args := m.Called(input)
+	return args.Get(0).(*s3.HeadObjectOutput), args.Error(1)
+}
+
+func (m *mockS3) GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+	args := m.Called(input)
+	return args.Get(0).(*s3.GetObjectOutput), args.Error(1)
+}
+
+func (m *mockS3) ListObjectsV2Pages(input *s3.ListObjectsV2Input, fn func(*s3.ListObjectsV2Output, bool) bool) error {
+	args := m.Called(input, fn)
+	return args.Error(0)
+}
+
+func (m *mockS3) DeleteObject(input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+	args := m.Called(input)
+	return args.Get(0).(*s3.DeleteObjectOutput), args.Error(1)
+}
+
+func (m *mockS3) GetObjectRequest(input *s3.GetObjectInput) (req *request.Request, output *s3.GetObjectOutput) {
+	args := m.Called(input)
+	return args.Get(0).(*request.Request), args.Get(1).(*s3.GetObjectOutput)
+}
+
+func TestObjectExists(t *testing.T) {
+	tests := []struct {
+		name           string
+		errorResponse  error
+		expectedExists bool
+		expectedError  string
+	}{
+		{
+			name:           "exists",
+			errorResponse:  nil,
+			expectedExists: true,
+		},
+		{
+			name:           "doesn't exist",
+			errorResponse:  awserr.New(s3.ErrCodeNoSuchKey, "no such key", nil),
+			expectedExists: false,
+			expectedError:  "NoSuchKey: no such key",
+		},
+		{
+			name:           "error checking for existence",
+			errorResponse:  errors.Errorf("bad"),
+			expectedExists: false,
+			expectedError:  "bad",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			s := new(mockS3)
+			defer s.AssertExpectations(t)
+
+			o := &ObjectStore{
+				log: test.NewLogger(),
+				s3:  s,
+			}
+
+			bucket := "b"
+			key := "k"
+			req := &s3.HeadObjectInput{
+				Bucket: aws.String(bucket),
+				Key:    aws.String(key),
+			}
+
+			s.On("HeadObject", req).Return(&s3.HeadObjectOutput{}, tc.errorResponse)
+
+			exists, err := o.ObjectExists(bucket, key)
+
+			if tc.expectedError != "" {
+				assert.EqualError(t, err, tc.expectedError)
+				return
+			}
+			require.NoError(t, err)
+
+			assert.Equal(t, tc.expectedExists, exists)
+		})
+	}
+}