CSPL-1027: Utility to download or upload file to s3

test/licensemaster/lm_c3_test.go

@@ -46,7 +46,7 @@ var _ = Describe("Licensemaster test", func() {
 		It("licensemaster, integration: Splunk Operator can configure License Master with Indexers and Search Heads in C3 SVA", func() {
 
 			// Download License File
-			licenseFilePath, err := testenv.DownloadFromS3Bucket()
+			licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket()
 			Expect(err).To(Succeed(), "Unable to download license file")
 
 			// Create License Config Map

test/licensemaster/lm_m4_test.go

@@ -46,7 +46,7 @@ var _ = Describe("Licensemaster test", func() {
 		It("licensemaster: Splunk Operator can configure license master with indexers and search head in M4 SVA", func() {
 
 			// Download License File
-			licenseFilePath, err := testenv.DownloadFromS3Bucket()
+			licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket()
 			Expect(err).To(Succeed(), "Unable to download license file")
 
 			// Create License Config Map

test/licensemaster/lm_s1_test.go

@@ -46,7 +46,7 @@ var _ = Describe("Licensemaster test", func() {
 		It("licensemaster: Splunk Operator can configure License Master with Standalone in S1 SVA", func() {
 
 			// Download License File
-			licenseFilePath, err := testenv.DownloadFromS3Bucket()
+			licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket()
 			Expect(err).To(Succeed(), "Unable to download license file")
 
 			// Create License Config Map

test/secret/secret_c3_test.go

@@ -54,7 +54,7 @@ var _ = Describe("Secret Test for SVA C3", func() {
 			5. Verify New Secrets via api access (password)*/
 
 			// Download License File
-			licenseFilePath, err := testenv.DownloadFromS3Bucket()
+			licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket()
 			Expect(err).To(Succeed(), "Unable to download license file")
 
 			// Create License Config Map

test/secret/secret_m4_test.go

@@ -54,7 +54,7 @@ var _ = Describe("Secret Test for M4 SVA", func() {
 			5. Verify New Secrets via api access (password)*/
 
 			// Download License File
-			licenseFilePath, err := testenv.DownloadFromS3Bucket()
+			licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket()
 			Expect(err).To(Succeed(), "Unable to download license file")
 
 			// Create License Config Map

test/secret/secret_s1_test.go

@@ -54,7 +54,7 @@ var _ = Describe("Secret Test for SVA S1", func() {
 			5. Verify New Secrets via api access (password)*/
 
 			// Download License File
-			licenseFilePath, err := testenv.DownloadFromS3Bucket()
+			licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket()
 			Expect(err).To(Succeed(), "Unable to download license file")
 
 			// Create License Config Map
@@ -134,7 +134,7 @@ var _ = Describe("Secret Test for SVA S1", func() {
 			5. Verify New Secrets via api access (password)*/
 
 			// Download License File
-			licenseFilePath, err := testenv.DownloadFromS3Bucket()
+			licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket()
 			Expect(err).To(Succeed(), "Unable to download license file")
 
 			// Create License Config Map

test/testenv/s3utils.go

@@ -38,22 +38,9 @@ func GetS3Endpoint() string {
 // CheckPrefixExistsOnS3 lists object in a bucket
 func CheckPrefixExistsOnS3(prefix string) bool {
 	dataBucket := testIndexesS3Bucket
-	sess, err := session.NewSession(&aws.Config{Region: aws.String(s3Region)})
-	if err != nil {
-		logf.Log.Error(err, "Failed to create s3 session")
-	}
-	svc := s3.New(session.Must(sess, err))
-	resp, err := svc.ListObjects(&s3.ListObjectsInput{
-		Bucket: aws.String(dataBucket),
-		Prefix: aws.String(prefix),
-	})
-
-	if err != nil {
-		logf.Log.Error(err, "Failed to list objects on s3 bucket")
-		return false
-	}
 
-	for _, key := range resp.Contents {
+	resp := GetFileListOnS3(dataBucket)
+	for _, key := range resp {
 		logf.Log.Info("CHECKING KEY ", "KEY", *key.Key)
 		if strings.Contains(*key.Key, prefix) {
 			logf.Log.Info("Prefix found on bucket", "Prefix", prefix, "KEY", *key.Key)
@@ -64,27 +51,104 @@ func CheckPrefixExistsOnS3(prefix string) bool {
 	return false
 }
 
-// DownloadFromS3Bucket downloads license file from S3
-func DownloadFromS3Bucket() (string, error) {
-	dataBucket := testS3Bucket
+// DownloadLicenseFromS3Bucket downloads license file from S3
+func DownloadLicenseFromS3Bucket() (string, error) {
 	location := enterpriseLicenseLocation
 	item := "enterprise.lic"
-	file, err := os.Create(item)
+	dataBucket := testS3Bucket
+	filename, err := DownloadFileFromS3(dataBucket, item, location)
+	return filename, err
+}
+
+// S3Session Create session object for S3 bucket connection
+func S3Session() (*session.Session, error) {
+	sess, err := session.NewSession(&aws.Config{Region: aws.String(s3Region)})
 	if err != nil {
-		logf.Log.Error(err, "Failed to create license file")
+		logf.Log.Error(err, "Failed to create session to S3")
+	}
+	return sess, err
+}
+
+// DownloadFileFromS3 downloads file from S3
+func DownloadFileFromS3(dataBucket string, filename string, filepath string) (string, error) {
+	file, err := os.Create(filename)
+	if err != nil {
+		logf.Log.Error(err, "Failed to create file")
 	}
 	defer file.Close()
-	sess, _ := session.NewSession(&aws.Config{Region: aws.String(s3Region)})
-	downloader := s3manager.NewDownloader(sess)
-	numBytes, err := downloader.Download(file,
-		&s3.GetObjectInput{
+	sess, err := S3Session()
+	if err == nil {
+		downloader := s3manager.NewDownloader(sess)
+		numBytes, err := downloader.Download(file,
+			&s3.GetObjectInput{
+				Bucket: aws.String(dataBucket),
+				Key:    aws.String(filepath + "/" + filename),
+			})
+		if err != nil {
+			logf.Log.Error(err, "Failed to download file")
+		}
+		logf.Log.Info("Downloaded", "filename", file.Name(), "bytes", numBytes)
+	}
+	return file.Name(), err
+}
+
+// UploadFileToS3 upload file to S3
+func UploadFileToS3(dataBucket string, filename string, filepath string, file *os.File) (string, error) {
+	sess, err := S3Session()
+	if err == nil {
+		uploader := s3manager.NewUploader(sess)
+		numBytes, err := uploader.Upload(&s3manager.UploadInput{
 			Bucket: aws.String(dataBucket),
-			Key:    aws.String(location + "/" + "enterprise.lic"),
+			Key:    aws.String(filename), // Name of the file to be saved
+			Body:   file,                 // File
 		})
+		if err != nil {
+			logf.Log.Error(err, "Error in file upload")
+		}
+		logf.Log.Info("Uploaded", "filename", file.Name(), "bytes", numBytes)
+	}
+	return file.Name(), err
+}
+
+// GetFileListOnS3 lists object in a bucket
+func GetFileListOnS3(dataBucket string) []*s3.Object {
+	sess, err := S3Session()
+	svc := s3.New(session.Must(sess, err))
+	resp, err := svc.ListObjects(&s3.ListObjectsInput{
+		Bucket: aws.String(dataBucket),
+	})
 	if err != nil {
-		logf.Log.Error(err, "Failed to download license file")
+		logf.Log.Error(err, "Failed to list objects on s3 bucket")
+		return nil
 	}
 
-	logf.Log.Info("Downloaded", "filename", file.Name(), "bytes", numBytes)
-	return file.Name(), err
+	return resp.Contents
+}
+
+// DeleteFilesOnS3 Delete a list of file on S3 Bucket
+func DeleteFilesOnS3(dataBucket string, filenames []string) bool {
+	sess, err := S3Session()
+	if err == nil {
+		svc := s3.New(session.Must(sess, err))
+		objects := make([]*s3.ObjectIdentifier, len(filenames))
+
+		for ind, filename := range filenames {
+			objects[ind].Key = aws.String(filename)
+		}
+		deleteInput := &s3.DeleteObjectsInput{
+			Bucket: aws.String(dataBucket),
+			Delete: &s3.Delete{
+				Objects: objects,
+				Quiet:   aws.Bool(false),
+			},
+		}
+
+		_, err = svc.DeleteObjects(deleteInput)
+		if err != nil {
+			logf.Log.Error(err, "Failed to delete files on S3 bucket")
+			return false
+		}
+		return true
+	}
+	return false
 }