diff --git a/cmd/config.cluster.yml b/cmd/config.cluster.yml index 0807fc8960..bd3d3585d9 100644 --- a/cmd/config.cluster.yml +++ b/cmd/config.cluster.yml @@ -44,6 +44,15 @@ postgresql: database: swiftwave time_zone: Asia/Kolkata ssl_mode: disable # disable or require +persistent_volume_backup: + s3_config: + enabled: false # true or false + endpoint: "" + region: "" + bucket: "" + access_key_id: "" + secret_access_key: "" + force_path_style: false pubsub: mode: remote # local or remote buffer_length: 1000 diff --git a/cmd/config.standalone.yml b/cmd/config.standalone.yml index d7c2409b95..ce41171132 100644 --- a/cmd/config.standalone.yml +++ b/cmd/config.standalone.yml @@ -44,6 +44,15 @@ postgresql: database: swiftwave time_zone: Asia/Kolkata ssl_mode: disable # disable or require +persistent_volume_backup: + s3_config: + enabled: false # true or false + endpoint: "" + region: "" + bucket: "" + access_key_id: "" + secret_access_key: "" + force_path_style: false pubsub: mode: local # local or remote buffer_length: 1000 diff --git a/go.mod b/go.mod index c00bd352fc..6f63139d8d 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.21.7 require ( github.com/99designs/gqlgen v0.17.43 + github.com/aws/aws-sdk-go v1.50.18 github.com/docker/docker v25.0.3+incompatible github.com/fatih/color v1.16.0 github.com/go-git/go-git/v5 v5.11.0 @@ -60,6 +61,7 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.16.7 // indirect github.com/moby/patternmatcher v0.5.0 // indirect diff --git a/go.sum b/go.sum index aeb6456df8..51ace6833c 100644 --- a/go.sum +++ b/go.sum @@ -27,6 +27,8 @@ github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/aws/aws-sdk-go v1.50.18 h1:h+FQjxp5sSDqFKScTUXHVahBlqduKtiR0qM18evcvag= +github.com/aws/aws-sdk-go v1.50.18/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= @@ -123,6 +125,10 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -351,6 +357,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/swiftwave_service/core/pv_backup.operations.go b/swiftwave_service/core/pv_backup.operations.go index 86cba51ba1..4652d35c2d 100644 --- a/swiftwave_service/core/pv_backup.operations.go +++ b/swiftwave_service/core/pv_backup.operations.go @@ -2,6 +2,8 @@ package core import ( "context" + "github.com/swiftwave-org/swiftwave/swiftwave_service/uploader" + "github.com/swiftwave-org/swiftwave/system_config" "gorm.io/gorm" "log" "os" @@ -35,11 +37,19 @@ func (persistentVolumeBackup *PersistentVolumeBackup) Update(ctx context.Context return tx.Error } -func (persistentVolumeBackup *PersistentVolumeBackup) Delete(ctx context.Context, db gorm.DB, dataDir string) error { - if persistentVolumeBackup.File != "" && persistentVolumeBackup.Type == LocalBackup { - err := os.Remove(filepath.Join(dataDir, persistentVolumeBackup.File)) - if err != nil { - log.Println("error deleting file: ", err) +func (persistentVolumeBackup *PersistentVolumeBackup) Delete(ctx context.Context, db gorm.DB, dataDir string, config system_config.S3Config) error { + if persistentVolumeBackup.File != "" { + if persistentVolumeBackup.Type == LocalBackup { + err := os.Remove(filepath.Join(dataDir, persistentVolumeBackup.File)) + if err != nil { + log.Println("error deleting file: ", err) + } + } + if persistentVolumeBackup.Type == S3Backup { + err := uploader.DeleteFileFromS3(persistentVolumeBackup.File, config.Bucket, config) + if err != nil { + log.Println("error deleting file from s3: ", err) + } } } tx := db.Delete(persistentVolumeBackup) @@ -52,7 +62,7 @@ func FindPersistentVolumeBackupsByPersistentVolumeId(ctx context.Context, db gor return persistentVolumeBackups, tx.Error } -func DeletePersistentVolumeBackupsByPersistentVolumeId(ctx context.Context, db gorm.DB, persistentVolumeId uint, dataDir string) error { +func DeletePersistentVolumeBackupsByPersistentVolumeId(ctx context.Context, db gorm.DB, persistentVolumeId uint, dataDir string, config system_config.S3Config) error { transaction := db.Begin() var persistentVolumeBackups []*PersistentVolumeBackup tx := transaction.Where("persistent_volume_id = ?", persistentVolumeId).Find(&persistentVolumeBackups) @@ -61,7 +71,7 @@ func DeletePersistentVolumeBackupsByPersistentVolumeId(ctx context.Context, db g return tx.Error } for _, p := range persistentVolumeBackups { - err := p.Delete(ctx, *transaction, dataDir) + err := p.Delete(ctx, *transaction, dataDir, config) if err != nil { log.Println("error deleting persistentVolumeBackup: ", err) } diff --git a/swiftwave_service/core/types.go b/swiftwave_service/core/types.go index 7c4de1c045..f439815d6d 100644 --- a/swiftwave_service/core/types.go +++ b/swiftwave_service/core/types.go @@ -124,6 +124,7 @@ type BackupType string const ( LocalBackup BackupType = "local" + S3Backup BackupType = "s3" ) // BackupStatus : status of backup diff --git a/swiftwave_service/graphql/model/models_gen.go b/swiftwave_service/graphql/model/models_gen.go index 3649627eb9..3ba3af8725 100644 --- a/swiftwave_service/graphql/model/models_gen.go +++ b/swiftwave_service/graphql/model/models_gen.go @@ -630,15 +630,17 @@ type PersistentVolumeBackupType string const ( PersistentVolumeBackupTypeLocal PersistentVolumeBackupType = "local" + PersistentVolumeBackupTypeS3 PersistentVolumeBackupType = "s3" ) var AllPersistentVolumeBackupType = []PersistentVolumeBackupType{ PersistentVolumeBackupTypeLocal, + PersistentVolumeBackupTypeS3, } func (e PersistentVolumeBackupType) IsValid() bool { switch e { - case PersistentVolumeBackupTypeLocal: + case PersistentVolumeBackupTypeLocal, PersistentVolumeBackupTypeS3: return true } return false diff --git a/swiftwave_service/graphql/persistent_volume.resolvers.go b/swiftwave_service/graphql/persistent_volume.resolvers.go index 6395eb62f0..fbb79dfcc7 100644 --- a/swiftwave_service/graphql/persistent_volume.resolvers.go +++ b/swiftwave_service/graphql/persistent_volume.resolvers.go @@ -6,6 +6,7 @@ package graphql import ( "context" + "github.com/swiftwave-org/swiftwave/swiftwave_service/core" "github.com/swiftwave-org/swiftwave/swiftwave_service/graphql/model" ) diff --git a/swiftwave_service/graphql/persistent_volume_backup.resolvers.go b/swiftwave_service/graphql/persistent_volume_backup.resolvers.go index 771089f223..3195a23342 100644 --- a/swiftwave_service/graphql/persistent_volume_backup.resolvers.go +++ b/swiftwave_service/graphql/persistent_volume_backup.resolvers.go @@ -6,6 +6,7 @@ package graphql import ( "context" + "errors" "github.com/swiftwave-org/swiftwave/swiftwave_service/core" "github.com/swiftwave-org/swiftwave/swiftwave_service/graphql/model" @@ -14,6 +15,12 @@ import ( // BackupPersistentVolume is the resolver for the backupPersistentVolume field. func (r *mutationResolver) BackupPersistentVolume(ctx context.Context, input model.PersistentVolumeBackupInput) (*model.PersistentVolumeBackup, error) { record := persistentVolumeBackupInputToDatabaseObject(&input) + if record.Type == core.S3Backup { + // check if s3 enabled + if !r.ServiceConfig.PersistentVolumeBackupConfig.S3Config.Enabled { + return nil, errors.New("s3 backup is not enabled") + } + } err := record.Create(ctx, r.ServiceManager.DbClient) if err != nil { return nil, err @@ -33,7 +40,7 @@ func (r *mutationResolver) DeletePersistentVolumeBackup(ctx context.Context, id if err != nil { return false, err } - err = record.Delete(ctx, r.ServiceManager.DbClient, r.ServiceConfig.ServiceConfig.DataDir) + err = record.Delete(ctx, r.ServiceManager.DbClient, r.ServiceConfig.ServiceConfig.DataDir, r.ServiceConfig.PersistentVolumeBackupConfig.S3Config) if err != nil { return false, err } @@ -42,7 +49,7 @@ func (r *mutationResolver) DeletePersistentVolumeBackup(ctx context.Context, id // DeletePersistentVolumeBackupsByPersistentVolumeID is the resolver for the deletePersistentVolumeBackupsByPersistentVolumeId field. func (r *mutationResolver) DeletePersistentVolumeBackupsByPersistentVolumeID(ctx context.Context, persistentVolumeID uint) (bool, error) { - err := core.DeletePersistentVolumeBackupsByPersistentVolumeId(ctx, r.ServiceManager.DbClient, persistentVolumeID, r.ServiceConfig.ServiceConfig.DataDir) + err := core.DeletePersistentVolumeBackupsByPersistentVolumeId(ctx, r.ServiceManager.DbClient, persistentVolumeID, r.ServiceConfig.ServiceConfig.DataDir, r.ServiceConfig.PersistentVolumeBackupConfig.S3Config) if err != nil { return false, err } diff --git a/swiftwave_service/graphql/schema/persistent_volume_backup.graphqls b/swiftwave_service/graphql/schema/persistent_volume_backup.graphqls index 068549c055..cb50954e74 100644 --- a/swiftwave_service/graphql/schema/persistent_volume_backup.graphqls +++ b/swiftwave_service/graphql/schema/persistent_volume_backup.graphqls @@ -1,5 +1,6 @@ enum PersistentVolumeBackupType { local + s3 } enum PersistentVolumeBackupStatus { diff --git a/swiftwave_service/rest/persistent_volume.go b/swiftwave_service/rest/persistent_volume.go index 877cac7e59..a290351156 100644 --- a/swiftwave_service/rest/persistent_volume.go +++ b/swiftwave_service/rest/persistent_volume.go @@ -2,9 +2,12 @@ package rest import ( "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" "github.com/google/uuid" "github.com/labstack/echo/v4" "github.com/swiftwave-org/swiftwave/swiftwave_service/core" + "github.com/swiftwave-org/swiftwave/swiftwave_service/uploader" "io" "log" "mime/multipart" @@ -32,11 +35,50 @@ func (server *Server) downloadPersistentVolumeBackup(c echo.Context) error { if persistentVolumeBackup.Status != core.BackupSuccess { return c.String(400, "Sorry, backup is not available for download") } - // send file - filePath := filepath.Join(server.SystemConfig.ServiceConfig.DataDir, persistentVolumeBackup.File) - // file name - fileName := persistentVolumeBackup.File - return c.Attachment(filePath, fileName) + if persistentVolumeBackup.Type == core.LocalBackup { + // send file + filePath := filepath.Join(server.SystemConfig.ServiceConfig.DataDir, persistentVolumeBackup.File) + // file name + fileName := persistentVolumeBackup.File + c.Request().Header.Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s", fileName)) + return c.Attachment(filePath, fileName) + } else if persistentVolumeBackup.Type == core.S3Backup { + s3config := server.SystemConfig.PersistentVolumeBackupConfig.S3Config + if !s3config.Enabled { + return c.String(400, "S3 backup is not enabled") + } + // download file from s3 + s3Client, err := uploader.GenerateS3Client(s3config) + if err != nil { + return c.String(500, "Internal server error") + } + // download file + resp, err := s3Client.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(s3config.Bucket), + Key: aws.String(persistentVolumeBackup.File), + }) + if err != nil { + return c.String(500, "Internal server error") + } + defer func(resp *s3.GetObjectOutput) { + err := resp.Body.Close() + if err != nil { + log.Println(err) + } + }(resp) + // send file + if resp.ContentLength != nil { + contentLength, err := strconv.ParseInt(strconv.FormatInt(*resp.ContentLength, 10), 10, 64) + if err != nil { + return c.String(500, "Internal server error") + } + c.Response().Header().Set("Content-Length", fmt.Sprintf("%d", contentLength)) + } + c.Response().Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s", persistentVolumeBackup.File)) + return c.Stream(200, "application/octet-stream", resp.Body) + } else { + return c.String(500, "Internal server error") + } } // POST /persistent-volume/restore/:id/upload diff --git a/swiftwave_service/uploader/s3.go b/swiftwave_service/uploader/s3.go new file mode 100644 index 0000000000..ada8d9a49d --- /dev/null +++ b/swiftwave_service/uploader/s3.go @@ -0,0 +1,50 @@ +package uploader + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/swiftwave-org/swiftwave/system_config" + "io" +) + +func UploadFileToS3(reader io.ReadSeeker, filename, bucket string, config system_config.S3Config) error { + s3Client, err := GenerateS3Client(config) + if err != nil { + return err + } + _, err = s3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(filename), + Body: reader, + ACL: nil, + }) + return err +} + +func DeleteFileFromS3(filename, bucket string, config system_config.S3Config) error { + s3Client, err := GenerateS3Client(config) + if err != nil { + return err + } + _, err = s3Client.DeleteObject(&s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(filename), + }) + return err +} + +func GenerateS3Client(config system_config.S3Config) (*s3.S3, error) { + s3Config := &aws.Config{ + Credentials: credentials.NewStaticCredentials(config.AccessKeyID, config.SecretAccessKey, ""), + Endpoint: aws.String(config.Endpoint), + Region: aws.String(config.Region), + S3ForcePathStyle: aws.Bool(config.ForcePathStyle), + } + newSession, err := session.NewSession(s3Config) + if err != nil { + return nil, err + } + return s3.New(newSession), nil +} diff --git a/swiftwave_service/worker/process_pv_backup_request.go b/swiftwave_service/worker/process_pv_backup_request.go index fe15fe0314..e15da22a92 100644 --- a/swiftwave_service/worker/process_pv_backup_request.go +++ b/swiftwave_service/worker/process_pv_backup_request.go @@ -4,6 +4,7 @@ import ( "context" "github.com/google/uuid" "github.com/swiftwave-org/swiftwave/swiftwave_service/core" + "github.com/swiftwave-org/swiftwave/swiftwave_service/uploader" "gorm.io/gorm" "log" "os" @@ -31,13 +32,49 @@ func (m Manager) PersistentVolumeBackup(request PersistentVolumeBackupRequest, c dockerManager := m.ServiceManager.DockerManager // generate a random filename backupFileName := "backup_" + persistentVolume.Name + "_" + uuid.NewString() + ".tar.gz" - backupFilePath := filepath.Join(m.SystemConfig.ServiceConfig.DataDir, backupFileName) + var backupFilePath string + if persistentVolumeBackup.Type == core.LocalBackup { + backupFilePath = filepath.Join(m.SystemConfig.ServiceConfig.DataDir, backupFileName) + } else if persistentVolumeBackup.Type == core.S3Backup { + // fetch tmp dir + tmpDir := os.TempDir() + backupFilePath = filepath.Join(tmpDir, backupFileName) + defer func() { + err := os.Remove(backupFilePath) + if err != nil { + log.Println("failed to remove backup file " + err.Error()) + } + }() + } else { + return nil + } // create backup err = dockerManager.BackupVolume(persistentVolume.Name, backupFilePath) if err != nil { markPVBackupRequestAsFailed(dbWithoutTx, persistentVolumeBackup) return nil } + // upload to s3 + if persistentVolumeBackup.Type == core.S3Backup { + backupFileReader, err := os.Open(backupFilePath) + if err != nil { + markPVBackupRequestAsFailed(dbWithoutTx, persistentVolumeBackup) + return nil + } + defer func() { + err := backupFileReader.Close() + if err != nil { + log.Println("failed to close backup file reader " + err.Error()) + } + }() + s3Config := m.SystemConfig.PersistentVolumeBackupConfig.S3Config + err = uploader.UploadFileToS3(backupFileReader, backupFileName, s3Config.Bucket, s3Config) + if err != nil { + log.Println("error while uploading backup to s3 > " + err.Error()) + markPVBackupRequestAsFailed(dbWithoutTx, persistentVolumeBackup) + return nil + } + } // update status persistentVolumeBackup.Status = core.BackupSuccess persistentVolumeBackup.File = backupFileName diff --git a/system_config/types.go b/system_config/types.go index 46ac0154a8..3c7ccca013 100644 --- a/system_config/types.go +++ b/system_config/types.go @@ -1,16 +1,17 @@ package system_config type Config struct { - Version string `yaml:"version"` - IsDevelopmentMode bool `yaml:"-"` - Mode Mode `yaml:"mode"` - ServiceConfig ServiceConfig `yaml:"service"` - HAProxyConfig HAProxyConfig `yaml:"haproxy"` - UDPProxyConfig UDPProxyConfig `yaml:"udp_proxy"` - PostgresqlConfig PostgresqlConfig `yaml:"postgresql"` - LetsEncryptConfig LetsEncryptConfig `yaml:"lets_encrypt"` - PubSubConfig PubSubConfig `yaml:"pubsub"` - TaskQueueConfig TaskQueueConfig `yaml:"task_queue"` + Version string `yaml:"version"` + IsDevelopmentMode bool `yaml:"-"` + Mode Mode `yaml:"mode"` + ServiceConfig ServiceConfig `yaml:"service"` + HAProxyConfig HAProxyConfig `yaml:"haproxy"` + UDPProxyConfig UDPProxyConfig `yaml:"udp_proxy"` + PostgresqlConfig PostgresqlConfig `yaml:"postgresql"` + LetsEncryptConfig LetsEncryptConfig `yaml:"lets_encrypt"` + PubSubConfig PubSubConfig `yaml:"pubsub"` + TaskQueueConfig TaskQueueConfig `yaml:"task_queue"` + PersistentVolumeBackupConfig PersistentVolumeBackupConfig `yaml:"persistent_volume_backup"` } type ServiceConfig struct { @@ -91,3 +92,17 @@ type AMQPConfig struct { VHost string `yaml:"vhost"` ClientName string `yaml:"client_name"` } + +type PersistentVolumeBackupConfig struct { + S3Config S3Config `yaml:"s3_config"` +} + +type S3Config struct { + Enabled bool `yaml:"enabled"` + Endpoint string `yaml:"endpoint"` + Region string `yaml:"region"` + Bucket string `yaml:"bucket"` + AccessKeyID string `yaml:"access_key_id"` + SecretAccessKey string `yaml:"secret_access_key"` + ForcePathStyle bool `yaml:"force_path_style"` +}