Merge pull request #75 from swapnilgm/swift-client

Add mock tests for Openstack swift

Gopkg.toml

@@ -68,4 +68,8 @@ required = ["github.com/coreos/bbolt"]
 
 [[constraint]]
   name = "github.com/gophercloud/gophercloud"
-  revision = "c7ca48da8eafab13e1835090a1147e64cfc10174"
+  revision = "c7ca48da8eafab13e1835090a1147e64cfc10174"
+
+[[override]]
+name = "gopkg.in/fsnotify.v1"
+source = "https://github.com/fsnotify/fsnotify.git"

pkg/etcdutil/defrag_test.go

@@ -20,17 +20,18 @@ var _ = Describe("Defrag", func() {
 	tlsConfig = NewTLSConfig("", "", "", true, true, endpoints)
 	Context("Defragmentation", func() {
 		BeforeEach(func() {
+			now := time.Now().Unix()
 			client, err := GetTLSClientForEtcd(tlsConfig)
 			defer client.Close()
 			Expect(err).ShouldNot(HaveOccurred())
 			for index := 0; index <= 1000; index++ {
 				ctx, cancel := context.WithTimeout(context.TODO(), etcdConnectionTimeout)
-				client.Put(ctx, fmt.Sprintf("%s%d", keyPrefix, index), valuePrefix)
+				client.Put(ctx, fmt.Sprintf("%s%d%d", keyPrefix, now, index), valuePrefix)
 				cancel()
 			}
 			for index := 0; index <= 500; index++ {
 				ctx, cancel := context.WithTimeout(context.TODO(), etcdConnectionTimeout)
-				client.Delete(ctx, fmt.Sprintf("%s%d", keyPrefix, index))
+				client.Delete(ctx, fmt.Sprintf("%s%d%d", keyPrefix, now, index))
 				cancel()
 			}
 		})

pkg/snapshot/restorer/restorer.go

@@ -163,10 +163,12 @@ func makeDB(snapdir string, snap snapstore.Snapshot, commit int, ss snapstore.Sn
 		if _, err = db.Seek(-sha256.Size, io.SeekEnd); err != nil {
 			return err
 		}
-		sha := make([]byte, sha256.Size)
-		if _, err := db.Read(sha); err != nil {
-			return err
+
+		sha, err := ioutil.ReadAll(db)
+		if err != nil {
+			return fmt.Errorf("failed to read sha from db %v", err)
 		}
+
 		// truncate away integrity hash
 		if err = db.Truncate(off - sha256.Size); err != nil {
 			return err

pkg/snapstore/s3_snapstore_test.go

@@ -17,10 +17,10 @@ package snapstore_test
 import (
 	"bytes"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -32,9 +32,10 @@ import (
 // Define a mock struct to be used in your unit tests of myFunc.
 type mockS3Client struct {
 	s3iface.S3API
-	objects          map[string]*[]byte
-	prefix           string
-	multiPartUploads map[string]*[][]byte
+	objects               map[string]*[]byte
+	prefix                string
+	multiPartUploads      map[string]*[][]byte
+	multiPartUploadsMutex sync.Mutex
 }
 
 // GetObject returns the object from map for mock test
@@ -51,10 +52,10 @@ func (m *mockS3Client) GetObject(in *s3.GetObjectInput) (*s3.GetObjectOutput, er
 
 // PutObject adds the object to the map for mock test
 func (m *mockS3Client) PutObject(in *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
-	off, _ := in.Body.Seek(0, io.SeekEnd)
-	in.Body.Seek(0, io.SeekStart)
-	temp := make([]byte, off)
-	in.Body.Read(temp)
+	temp, err := ioutil.ReadAll(in.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read complete body %v", err)
+	}
 	m.objects[*in.Key] = &temp
 	out := s3.PutObjectOutput{}
 	return &out, nil
@@ -79,18 +80,17 @@ func (m *mockS3Client) UploadPartWithContext(ctx aws.Context, in *s3.UploadPartI
 		return nil, fmt.Errorf("part number should be positive integer")
 	}
 	if *in.PartNumber > int64(len(*m.multiPartUploads[*in.UploadId])) {
+		m.multiPartUploadsMutex.Lock()
 		t := make([][]byte, *in.PartNumber)
 		copy(t, *m.multiPartUploads[*in.UploadId])
 		delete(m.multiPartUploads, *in.UploadId)
 		m.multiPartUploads[*in.UploadId] = &t
+		m.multiPartUploadsMutex.Unlock()
 	}
-	off, err := in.Body.Seek(0, io.SeekEnd)
+	temp, err := ioutil.ReadAll(in.Body)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to read complete body %v", err)
 	}
-	in.Body.Seek(0, io.SeekStart)
-	temp := make([]byte, off)
-	in.Body.Read(temp)
 	(*m.multiPartUploads[*in.UploadId])[*in.PartNumber-1] = temp
 	eTag := string(*in.PartNumber)
 	out := &s3.UploadPartOutput{
@@ -153,7 +153,7 @@ func (m *mockS3Client) ListObjects(in *s3.ListObjectsInput) (*s3.ListObjectsOutp
 func (m *mockS3Client) ListObjectsPages(in *s3.ListObjectsInput, callback func(*s3.ListObjectsOutput, bool) bool) error {
 	var (
 		count    int64 = 0
-		limit    int64 = 1000
+		limit    int64 = 1 // aws default is 1000.
 		lastPage bool  = false
 		keys     []string
 		out      = &s3.ListObjectsOutput{

pkg/snapstore/snapstore_suite_test.go

@@ -15,13 +15,36 @@
 package snapstore_test
 
 import (
+	"testing"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/gardener/etcd-backup-restore/pkg/snapstore"
+	th "github.com/gophercloud/gophercloud/testhelper"
+	fake "github.com/gophercloud/gophercloud/testhelper/client"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+)
 
-	"testing"
+var (
+	testObj    *testing.T
+	swiftStore snapstore.SnapStore
 )
 
 func TestSnapstore(t *testing.T) {
 	RegisterFailHandler(Fail)
+	testObj = t
 	RunSpecs(t, "Snapstore Suite")
 }
+
+var _ = BeforeSuite(func() {
+	logrus.Infof("Starting test server...")
+	th.SetupHTTP()
+	initializeMockSwiftServer(testObj)
+	swiftStore = snapstore.NewSwiftSnapstoreFromClient(bucket, prefix, "/tmp", 5, fake.ServiceClient())
+})
+
+var _ = AfterSuite(func() {
+	logrus.Infof("Shutting down test server...")
+	th.TeardownHTTP()
+})

pkg/snapstore/snapstore_test.go

@@ -15,30 +15,36 @@
 package snapstore_test
 
 import (
+	"bytes"
+	"io/ioutil"
 	"path"
-	"strings"
 	"time"
 
+	"github.com/sirupsen/logrus"
+
 	. "github.com/gardener/etcd-backup-restore/pkg/snapstore"
+	fake "github.com/gophercloud/gophercloud/testhelper/client"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
+var (
+	bucket    string = "mock-bucket"
+	objectMap        = map[string]*[]byte{}
+	prefix    string = "v1"
+)
+
 var _ = Describe("Snapstore", func() {
 	var (
-		bucket       string
 		snap1        Snapshot
 		snap2        Snapshot
 		snap3        Snapshot
-		expectedVal1 string
-		expectedVal2 string
-		m            mockS3Client
+		expectedVal1 []byte
+		expectedVal2 []byte
 		snapstores   map[string]SnapStore
 	)
-	// S3SnapStore is snapstore with local disk as backend
+
 	BeforeEach(func() {
-		bucket = "mock-bucket"
-		prefix := "v1"
 		now := time.Now().Unix()
 		snap1 = Snapshot{
 			CreatedOn:     time.Unix(now, 0).UTC(),
@@ -64,63 +70,81 @@ var _ = Describe("Snapstore", func() {
 		snap2.GenerateSnapshotDirectory()
 		snap3.GenerateSnapshotName()
 		snap3.GenerateSnapshotDirectory()
-		expectedVal1 = "value1"
-		expectedVal2 = "value2"
-		expectedVal1Bytes := []byte(expectedVal1)
-		expectedVal2Bytes := []byte(expectedVal2)
-		m = mockS3Client{
-			objects: map[string]*[]byte{
-				path.Join(prefix, snap1.SnapDir, snap1.SnapName): &expectedVal1Bytes,
-				path.Join(prefix, snap2.SnapDir, snap2.SnapName): &expectedVal2Bytes,
-			},
-			prefix:           prefix,
-			multiPartUploads: map[string]*[][]byte{},
-		}
+		expectedVal1 = []byte("value1")
+		expectedVal2 = []byte("value2")
+
 		snapstores = map[string]SnapStore{
-			"s3": NewS3FromClient(bucket, prefix, "/tmp", 5, &m),
+			"s3": NewS3FromClient(bucket, prefix, "/tmp", 5, &mockS3Client{
+				objects:          objectMap,
+				prefix:           prefix,
+				multiPartUploads: map[string]*[][]byte{},
+			}),
+			"swift": NewSwiftSnapstoreFromClient(bucket, prefix, "/tmp", 5, fake.ServiceClient()),
 		}
 	})
 
 	Describe("Fetch operation", func() {
 		It("fetches snapshot", func() {
-			for _, snapStore := range snapstores {
+			for key, snapStore := range snapstores {
+				logrus.Infof("Running tests for %s", key)
+				resetObjectMap()
+				objectMap[path.Join(prefix, snap1.SnapDir, snap1.SnapName)] = &expectedVal1
+				objectMap[path.Join(prefix, snap2.SnapDir, snap2.SnapName)] = &expectedVal2
 				rc, err := snapStore.Fetch(snap1)
 				Expect(err).ShouldNot(HaveOccurred())
-				temp := make([]byte, len(expectedVal1))
-				n, err := rc.Read(temp)
+				temp, err := ioutil.ReadAll(rc)
 				Expect(err).ShouldNot(HaveOccurred())
-				Expect(string(temp[:n])).To(Equal(expectedVal1))
+				Expect(temp).To(Equal(expectedVal1))
 			}
 		})
 	})
+
 	Describe("Save snapshot", func() {
 		It("saves snapshot", func() {
-			for _, snapStore := range snapstores {
-				prevLen := len(m.objects)
-				err := snapStore.Save(snap3, strings.NewReader("value3"))
+			for key, snapStore := range snapstores {
+				logrus.Infof("Running tests for %s", key)
+				resetObjectMap()
+				dummyData := make([]byte, 6*1024*1024)
+				err := snapStore.Save(snap3, bytes.NewReader(dummyData))
 				Expect(err).ShouldNot(HaveOccurred())
-				Expect(len(m.objects)).To(Equal(prevLen + 1))
+				Expect(len(objectMap)).Should(BeNumerically(">", 0))
 			}
 		})
 	})
+
 	Describe("List snapshot", func() {
 		It("gives sorted list of snapshot", func() {
-			for _, snapStore := range snapstores {
+			for key, snapStore := range snapstores {
+				logrus.Infof("Running tests for %s", key)
+				resetObjectMap()
+				objectMap[path.Join(prefix, snap1.SnapDir, snap1.SnapName)] = &expectedVal1
+				objectMap[path.Join(prefix, snap2.SnapDir, snap2.SnapName)] = &expectedVal2
 				snapList, err := snapStore.List()
 				Expect(err).ShouldNot(HaveOccurred())
 				Expect(snapList.Len()).To(Equal(2))
 				Expect(snapList[0].SnapName).To(Equal(snap1.SnapName))
 			}
 		})
 	})
+
 	Describe("Delete snapshot", func() {
 		It("deletes snapshot", func() {
-			for _, snapStore := range snapstores {
-				prevLen := len(m.objects)
+			for key, snapStore := range snapstores {
+				logrus.Infof("Running tests for %s", key)
+				resetObjectMap()
+				objectMap[path.Join(prefix, snap1.SnapDir, snap1.SnapName)] = &expectedVal1
+				objectMap[path.Join(prefix, snap2.SnapDir, snap2.SnapName)] = &expectedVal2
+				prevLen := len(objectMap)
 				err := snapStore.Delete(snap2)
 				Expect(err).ShouldNot(HaveOccurred())
-				Expect(len(m.objects)).To(Equal(prevLen - 1))
+				Expect(len(objectMap)).To(Equal(prevLen - 1))
 			}
 		})
 	})
 })
+
+func resetObjectMap() {
+	for k := range objectMap {
+		delete(objectMap, k)
+	}
+}