From a20fb7613ad595e48b4552f25e5383a4488724cb Mon Sep 17 00:00:00 2001 From: tpereirasplunk Date: Wed, 12 May 2021 10:06:23 -0700 Subject: [PATCH] CSPL-1027: Utility to download or upload file to s3 --- test/licensemaster/lm_c3_test.go | 2 +- test/licensemaster/lm_m4_test.go | 2 +- test/licensemaster/lm_s1_test.go | 2 +- test/secret/secret_c3_test.go | 2 +- test/secret/secret_m4_test.go | 2 +- test/secret/secret_s1_test.go | 4 +- test/testenv/s3utils.go | 120 +++++++++++++++++++++++-------- 7 files changed, 99 insertions(+), 35 deletions(-) diff --git a/test/licensemaster/lm_c3_test.go b/test/licensemaster/lm_c3_test.go index 42106a859..37c3b95c0 100644 --- a/test/licensemaster/lm_c3_test.go +++ b/test/licensemaster/lm_c3_test.go @@ -46,7 +46,7 @@ var _ = Describe("Licensemaster test", func() { It("licensemaster, integration: Splunk Operator can configure License Master with Indexers and Search Heads in C3 SVA", func() { // Download License File - licenseFilePath, err := testenv.DownloadFromS3Bucket() + licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() Expect(err).To(Succeed(), "Unable to download license file") // Create License Config Map diff --git a/test/licensemaster/lm_m4_test.go b/test/licensemaster/lm_m4_test.go index 93cca6454..326ecca2f 100644 --- a/test/licensemaster/lm_m4_test.go +++ b/test/licensemaster/lm_m4_test.go @@ -46,7 +46,7 @@ var _ = Describe("Licensemaster test", func() { It("licensemaster: Splunk Operator can configure license master with indexers and search head in M4 SVA", func() { // Download License File - licenseFilePath, err := testenv.DownloadFromS3Bucket() + licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() Expect(err).To(Succeed(), "Unable to download license file") // Create License Config Map diff --git a/test/licensemaster/lm_s1_test.go b/test/licensemaster/lm_s1_test.go index f444b3882..51490e842 100644 --- a/test/licensemaster/lm_s1_test.go +++ b/test/licensemaster/lm_s1_test.go @@ -46,7 +46,7 @@ var _ = Describe("Licensemaster test", func() { It("licensemaster: Splunk Operator can configure License Master with Standalone in S1 SVA", func() { // Download License File - licenseFilePath, err := testenv.DownloadFromS3Bucket() + licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() Expect(err).To(Succeed(), "Unable to download license file") // Create License Config Map diff --git a/test/secret/secret_c3_test.go b/test/secret/secret_c3_test.go index e1527cb60..1b9d826d7 100644 --- a/test/secret/secret_c3_test.go +++ b/test/secret/secret_c3_test.go @@ -54,7 +54,7 @@ var _ = Describe("Secret Test for SVA C3", func() { 5. Verify New Secrets via api access (password)*/ // Download License File - licenseFilePath, err := testenv.DownloadFromS3Bucket() + licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() Expect(err).To(Succeed(), "Unable to download license file") // Create License Config Map diff --git a/test/secret/secret_m4_test.go b/test/secret/secret_m4_test.go index 44599f6ef..265740684 100644 --- a/test/secret/secret_m4_test.go +++ b/test/secret/secret_m4_test.go @@ -54,7 +54,7 @@ var _ = Describe("Secret Test for M4 SVA", func() { 5. Verify New Secrets via api access (password)*/ // Download License File - licenseFilePath, err := testenv.DownloadFromS3Bucket() + licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() Expect(err).To(Succeed(), "Unable to download license file") // Create License Config Map diff --git a/test/secret/secret_s1_test.go b/test/secret/secret_s1_test.go index 81741c6b4..dbeb86be2 100644 --- a/test/secret/secret_s1_test.go +++ b/test/secret/secret_s1_test.go @@ -54,7 +54,7 @@ var _ = Describe("Secret Test for SVA S1", func() { 5. Verify New Secrets via api access (password)*/ // Download License File - licenseFilePath, err := testenv.DownloadFromS3Bucket() + licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() Expect(err).To(Succeed(), "Unable to download license file") // Create License Config Map @@ -134,7 +134,7 @@ var _ = Describe("Secret Test for SVA S1", func() { 5. Verify New Secrets via api access (password)*/ // Download License File - licenseFilePath, err := testenv.DownloadFromS3Bucket() + licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() Expect(err).To(Succeed(), "Unable to download license file") // Create License Config Map diff --git a/test/testenv/s3utils.go b/test/testenv/s3utils.go index f051db318..a0dccda14 100644 --- a/test/testenv/s3utils.go +++ b/test/testenv/s3utils.go @@ -38,22 +38,9 @@ func GetS3Endpoint() string { // CheckPrefixExistsOnS3 lists object in a bucket func CheckPrefixExistsOnS3(prefix string) bool { dataBucket := testIndexesS3Bucket - sess, err := session.NewSession(&aws.Config{Region: aws.String(s3Region)}) - if err != nil { - logf.Log.Error(err, "Failed to create s3 session") - } - svc := s3.New(session.Must(sess, err)) - resp, err := svc.ListObjects(&s3.ListObjectsInput{ - Bucket: aws.String(dataBucket), - Prefix: aws.String(prefix), - }) - - if err != nil { - logf.Log.Error(err, "Failed to list objects on s3 bucket") - return false - } - for _, key := range resp.Contents { + resp := GetFileListOnS3(dataBucket) + for _, key := range resp { logf.Log.Info("CHECKING KEY ", "KEY", *key.Key) if strings.Contains(*key.Key, prefix) { logf.Log.Info("Prefix found on bucket", "Prefix", prefix, "KEY", *key.Key) @@ -64,27 +51,104 @@ func CheckPrefixExistsOnS3(prefix string) bool { return false } -// DownloadFromS3Bucket downloads license file from S3 -func DownloadFromS3Bucket() (string, error) { - dataBucket := testS3Bucket +// DownloadLicenseFromS3Bucket downloads license file from S3 +func DownloadLicenseFromS3Bucket() (string, error) { location := enterpriseLicenseLocation item := "enterprise.lic" - file, err := os.Create(item) + dataBucket := testS3Bucket + filename, err := DownloadFileFromS3(dataBucket, item, location) + return filename, err +} + +// S3Session Create session object for S3 bucket connection +func S3Session() (*session.Session, error) { + sess, err := session.NewSession(&aws.Config{Region: aws.String(s3Region)}) if err != nil { - logf.Log.Error(err, "Failed to create license file") + logf.Log.Error(err, "Failed to create session to S3") + } + return sess, err +} + +// DownloadFileFromS3 downloads file from S3 +func DownloadFileFromS3(dataBucket string, filename string, filepath string) (string, error) { + file, err := os.Create(filename) + if err != nil { + logf.Log.Error(err, "Failed to create file") } defer file.Close() - sess, _ := session.NewSession(&aws.Config{Region: aws.String(s3Region)}) - downloader := s3manager.NewDownloader(sess) - numBytes, err := downloader.Download(file, - &s3.GetObjectInput{ + sess, err := S3Session() + if err == nil { + downloader := s3manager.NewDownloader(sess) + numBytes, err := downloader.Download(file, + &s3.GetObjectInput{ + Bucket: aws.String(dataBucket), + Key: aws.String(filepath + "/" + filename), + }) + if err != nil { + logf.Log.Error(err, "Failed to download file") + } + logf.Log.Info("Downloaded", "filename", file.Name(), "bytes", numBytes) + } + return file.Name(), err +} + +// UploadFileToS3 upload file to S3 +func UploadFileToS3(dataBucket string, filename string, filepath string, file *os.File) (string, error) { + sess, err := S3Session() + if err == nil { + uploader := s3manager.NewUploader(sess) + numBytes, err := uploader.Upload(&s3manager.UploadInput{ Bucket: aws.String(dataBucket), - Key: aws.String(location + "/" + "enterprise.lic"), + Key: aws.String(filename), // Name of the file to be saved + Body: file, // File }) + if err != nil { + logf.Log.Error(err, "Error in file upload") + } + logf.Log.Info("Uploaded", "filename", file.Name(), "bytes", numBytes) + } + return file.Name(), err +} + +// GetFileListOnS3 lists object in a bucket +func GetFileListOnS3(dataBucket string) []*s3.Object { + sess, err := S3Session() + svc := s3.New(session.Must(sess, err)) + resp, err := svc.ListObjects(&s3.ListObjectsInput{ + Bucket: aws.String(dataBucket), + }) if err != nil { - logf.Log.Error(err, "Failed to download license file") + logf.Log.Error(err, "Failed to list objects on s3 bucket") + return nil } - logf.Log.Info("Downloaded", "filename", file.Name(), "bytes", numBytes) - return file.Name(), err + return resp.Contents +} + +// DeleteFilesOnS3 Delete a list of file on S3 Bucket +func DeleteFilesOnS3(dataBucket string, filenames []string) bool { + sess, err := S3Session() + if err == nil { + svc := s3.New(session.Must(sess, err)) + objects := make([]*s3.ObjectIdentifier, len(filenames)) + + for ind, filename := range filenames { + objects[ind].Key = aws.String(filename) + } + deleteInput := &s3.DeleteObjectsInput{ + Bucket: aws.String(dataBucket), + Delete: &s3.Delete{ + Objects: objects, + Quiet: aws.Bool(false), + }, + } + + _, err = svc.DeleteObjects(deleteInput) + if err != nil { + logf.Log.Error(err, "Failed to delete files on S3 bucket") + return false + } + return true + } + return false }