feat: s3 support added in pv backup (backport #265)

cmd/config.cluster.yml

@@ -44,6 +44,15 @@ postgresql:
   database: swiftwave
   time_zone: Asia/Kolkata
   ssl_mode: disable # disable or require
+persistent_volume_backup:
+  s3_config:
+    enabled: false # true or false
+    endpoint: ""
+    region: ""
+    bucket: ""
+    access_key_id: ""
+    secret_access_key: ""
+    force_path_style: false
 pubsub:
   mode: remote # local or remote
   buffer_length: 1000

cmd/config.standalone.yml

@@ -44,6 +44,15 @@ postgresql:
   database: swiftwave
   time_zone: Asia/Kolkata
   ssl_mode: disable # disable or require
+persistent_volume_backup:
+  s3_config:
+    enabled: false # true or false
+    endpoint: ""
+    region: ""
+    bucket: ""
+    access_key_id: ""
+    secret_access_key: ""
+    force_path_style: false
 pubsub:
   mode: local # local or remote
   buffer_length: 1000

go.mod

@@ -4,6 +4,7 @@ go 1.21.7
 
 require (
 	github.com/99designs/gqlgen v0.17.43
+	github.com/aws/aws-sdk-go v1.50.18
 	github.com/docker/docker v25.0.3+incompatible
 	github.com/fatih/color v1.16.0
 	github.com/go-git/go-git/v5 v5.11.0
@@ -60,6 +61,7 @@ require (
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/klauspost/compress v1.16.7 // indirect
 	github.com/moby/patternmatcher v0.5.0 // indirect

go.sum

@@ -27,6 +27,8 @@ github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/aws/aws-sdk-go v1.50.18 h1:h+FQjxp5sSDqFKScTUXHVahBlqduKtiR0qM18evcvag=
+github.com/aws/aws-sdk-go v1.50.18/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
@@ -123,6 +125,10 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
 github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
@@ -351,6 +357,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
 gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
 gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

swiftwave_service/core/pv_backup.operations.go

@@ -2,6 +2,8 @@ package core
 
 import (
 	"context"
+	"github.com/swiftwave-org/swiftwave/swiftwave_service/uploader"
+	"github.com/swiftwave-org/swiftwave/system_config"
 	"gorm.io/gorm"
 	"log"
 	"os"
@@ -35,11 +37,19 @@ func (persistentVolumeBackup *PersistentVolumeBackup) Update(ctx context.Context
 	return tx.Error
 }
 
-func (persistentVolumeBackup *PersistentVolumeBackup) Delete(ctx context.Context, db gorm.DB, dataDir string) error {
-	if persistentVolumeBackup.File != "" && persistentVolumeBackup.Type == LocalBackup {
-		err := os.Remove(filepath.Join(dataDir, persistentVolumeBackup.File))
-		if err != nil {
-			log.Println("error deleting file: ", err)
+func (persistentVolumeBackup *PersistentVolumeBackup) Delete(ctx context.Context, db gorm.DB, dataDir string, config system_config.S3Config) error {
+	if persistentVolumeBackup.File != "" {
+		if persistentVolumeBackup.Type == LocalBackup {
+			err := os.Remove(filepath.Join(dataDir, persistentVolumeBackup.File))
+			if err != nil {
+				log.Println("error deleting file: ", err)
+			}
+		}
+		if persistentVolumeBackup.Type == S3Backup {
+			err := uploader.DeleteFileFromS3(persistentVolumeBackup.File, config.Bucket, config)
+			if err != nil {
+				log.Println("error deleting file from s3: ", err)
+			}
 		}
 	}
 	tx := db.Delete(persistentVolumeBackup)
@@ -52,7 +62,7 @@ func FindPersistentVolumeBackupsByPersistentVolumeId(ctx context.Context, db gor
 	return persistentVolumeBackups, tx.Error
 }
 
-func DeletePersistentVolumeBackupsByPersistentVolumeId(ctx context.Context, db gorm.DB, persistentVolumeId uint, dataDir string) error {
+func DeletePersistentVolumeBackupsByPersistentVolumeId(ctx context.Context, db gorm.DB, persistentVolumeId uint, dataDir string, config system_config.S3Config) error {
 	transaction := db.Begin()
 	var persistentVolumeBackups []*PersistentVolumeBackup
 	tx := transaction.Where("persistent_volume_id = ?", persistentVolumeId).Find(&persistentVolumeBackups)
@@ -61,7 +71,7 @@ func DeletePersistentVolumeBackupsByPersistentVolumeId(ctx context.Context, db g
 		return tx.Error
 	}
 	for _, p := range persistentVolumeBackups {
-		err := p.Delete(ctx, *transaction, dataDir)
+		err := p.Delete(ctx, *transaction, dataDir, config)
 		if err != nil {
 			log.Println("error deleting persistentVolumeBackup: ", err)
 		}

swiftwave_service/core/types.go

@@ -124,6 +124,7 @@ type BackupType string
 
 const (
 	LocalBackup BackupType = "local"
+	S3Backup    BackupType = "s3"
 )
 
 // BackupStatus : status of backup

swiftwave_service/graphql/schema/persistent_volume_backup.graphqls

@@ -1,5 +1,6 @@
 enum PersistentVolumeBackupType {
     local
+    s3
 }
 
 enum PersistentVolumeBackupStatus {

swiftwave_service/rest/persistent_volume.go

@@ -2,9 +2,12 @@ package rest
 
 import (
 	"fmt"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/google/uuid"
 	"github.com/labstack/echo/v4"
 	"github.com/swiftwave-org/swiftwave/swiftwave_service/core"
+	"github.com/swiftwave-org/swiftwave/swiftwave_service/uploader"
 	"io"
 	"log"
 	"mime/multipart"
@@ -32,11 +35,50 @@ func (server *Server) downloadPersistentVolumeBackup(c echo.Context) error {
 	if persistentVolumeBackup.Status != core.BackupSuccess {
 		return c.String(400, "Sorry, backup is not available for download")
 	}
-	// send file
-	filePath := filepath.Join(server.SystemConfig.ServiceConfig.DataDir, persistentVolumeBackup.File)
-	// file name
-	fileName := persistentVolumeBackup.File
-	return c.Attachment(filePath, fileName)
+	if persistentVolumeBackup.Type == core.LocalBackup {
+		// send file
+		filePath := filepath.Join(server.SystemConfig.ServiceConfig.DataDir, persistentVolumeBackup.File)
+		// file name
+		fileName := persistentVolumeBackup.File
+		c.Request().Header.Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s", fileName))
+		return c.Attachment(filePath, fileName)
+	} else if persistentVolumeBackup.Type == core.S3Backup {
+		s3config := server.SystemConfig.PersistentVolumeBackupConfig.S3Config
+		if !s3config.Enabled {
+			return c.String(400, "S3 backup is not enabled")
+		}
+		// download file from s3
+		s3Client, err := uploader.GenerateS3Client(s3config)
+		if err != nil {
+			return c.String(500, "Internal server error")
+		}
+		// download file
+		resp, err := s3Client.GetObject(&s3.GetObjectInput{
+			Bucket: aws.String(s3config.Bucket),
+			Key:    aws.String(persistentVolumeBackup.File),
+		})
+		if err != nil {
+			return c.String(500, "Internal server error")
+		}
+		defer func(resp *s3.GetObjectOutput) {
+			err := resp.Body.Close()
+			if err != nil {
+				log.Println(err)
+			}
+		}(resp)
+		// send file
+		if resp.ContentLength != nil {
+			contentLength, err := strconv.ParseInt(strconv.FormatInt(*resp.ContentLength, 10), 10, 64)
+			if err != nil {
+				return c.String(500, "Internal server error")
+			}
+			c.Response().Header().Set("Content-Length", fmt.Sprintf("%d", contentLength))
+		}
+		c.Response().Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s", persistentVolumeBackup.File))
+		return c.Stream(200, "application/octet-stream", resp.Body)
+	} else {
+		return c.String(500, "Internal server error")
+	}
 }
 
 // POST /persistent-volume/restore/:id/upload

swiftwave_service/uploader/s3.go

@@ -0,0 +1,50 @@
+package uploader
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/swiftwave-org/swiftwave/system_config"
+	"io"
+)
+
+func UploadFileToS3(reader io.ReadSeeker, filename, bucket string, config system_config.S3Config) error {
+	s3Client, err := GenerateS3Client(config)
+	if err != nil {
+		return err
+	}
+	_, err = s3Client.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(filename),
+		Body:   reader,
+		ACL:    nil,
+	})
+	return err
+}
+
+func DeleteFileFromS3(filename, bucket string, config system_config.S3Config) error {
+	s3Client, err := GenerateS3Client(config)
+	if err != nil {
+		return err
+	}
+	_, err = s3Client.DeleteObject(&s3.DeleteObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(filename),
+	})
+	return err
+}
+
+func GenerateS3Client(config system_config.S3Config) (*s3.S3, error) {
+	s3Config := &aws.Config{
+		Credentials:      credentials.NewStaticCredentials(config.AccessKeyID, config.SecretAccessKey, ""),
+		Endpoint:         aws.String(config.Endpoint),
+		Region:           aws.String(config.Region),
+		S3ForcePathStyle: aws.Bool(config.ForcePathStyle),
+	}
+	newSession, err := session.NewSession(s3Config)
+	if err != nil {
+		return nil, err
+	}
+	return s3.New(newSession), nil
+}

swiftwave_service/worker/process_pv_backup_request.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"github.com/google/uuid"
 	"github.com/swiftwave-org/swiftwave/swiftwave_service/core"
+	"github.com/swiftwave-org/swiftwave/swiftwave_service/uploader"
 	"gorm.io/gorm"
 	"log"
 	"os"
@@ -31,13 +32,49 @@ func (m Manager) PersistentVolumeBackup(request PersistentVolumeBackupRequest, c
 	dockerManager := m.ServiceManager.DockerManager
 	// generate a random filename
 	backupFileName := "backup_" + persistentVolume.Name + "_" + uuid.NewString() + ".tar.gz"
-	backupFilePath := filepath.Join(m.SystemConfig.ServiceConfig.DataDir, backupFileName)
+	var backupFilePath string
+	if persistentVolumeBackup.Type == core.LocalBackup {
+		backupFilePath = filepath.Join(m.SystemConfig.ServiceConfig.DataDir, backupFileName)
+	} else if persistentVolumeBackup.Type == core.S3Backup {
+		// fetch tmp dir
+		tmpDir := os.TempDir()
+		backupFilePath = filepath.Join(tmpDir, backupFileName)
+		defer func() {
+			err := os.Remove(backupFilePath)
+			if err != nil {
+				log.Println("failed to remove backup file " + err.Error())
+			}
+		}()
+	} else {
+		return nil
+	}
 	// create backup
 	err = dockerManager.BackupVolume(persistentVolume.Name, backupFilePath)
 	if err != nil {
 		markPVBackupRequestAsFailed(dbWithoutTx, persistentVolumeBackup)
 		return nil
 	}
+	// upload to s3
+	if persistentVolumeBackup.Type == core.S3Backup {
+		backupFileReader, err := os.Open(backupFilePath)
+		if err != nil {
+			markPVBackupRequestAsFailed(dbWithoutTx, persistentVolumeBackup)
+			return nil
+		}
+		defer func() {
+			err := backupFileReader.Close()
+			if err != nil {
+				log.Println("failed to close backup file reader " + err.Error())
+			}
+		}()
+		s3Config := m.SystemConfig.PersistentVolumeBackupConfig.S3Config
+		err = uploader.UploadFileToS3(backupFileReader, backupFileName, s3Config.Bucket, s3Config)
+		if err != nil {
+			log.Println("error while uploading backup to s3 > " + err.Error())
+			markPVBackupRequestAsFailed(dbWithoutTx, persistentVolumeBackup)
+			return nil
+		}
+	}
 	// update status
 	persistentVolumeBackup.Status = core.BackupSuccess
 	persistentVolumeBackup.File = backupFileName