Skip to content

Commit

Permalink
Add support for backup and restore from different namespace (#966) (#969
Browse files Browse the repository at this point in the history 
)

/cherry-pick
Signed-off-by: hmsayem hmsayem@appscode.com

Co-authored-by: Hossain Mahmud <hmsayem@appscode.com>
  • Loading branch information
@1gtm @hmsayem
1gtm and hmsayem committed Jan 20, 2022
1 parent 8bfeb53 commit d375d3e
Show file tree
Hide file tree
Showing 126 changed files with 27,285 additions and 11,336 deletions.
6 changes: 3 additions & 3 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@ require (
k8s.io/apimachinery v0.21.1
k8s.io/client-go v0.21.1
k8s.io/klog/v2 v2.8.0
kmodules.xyz/client-go v0.0.0-20211122091731-6c471b24a4ea
kmodules.xyz/custom-resources v0.0.0-20211122142737-3bf3dbd8ac52
kmodules.xyz/client-go v0.0.0-20220108081101-27afc2ac4ebe
kmodules.xyz/custom-resources v0.0.0-20220104123914-3c036dd7c1cd
kmodules.xyz/offshoot-api v0.0.0-20211103060642-3e217667cf41
stash.appscode.dev/apimachinery v0.17.0
stash.appscode.dev/apimachinery v0.17.1-0.20220113052814-7da4b19c88a3
)

replace bitbucket.org/ww/goautoneg => gomodules.xyz/goautoneg v0.0.0-20120707110453-a547fc61f48d
Expand Down
22 changes: 11 additions & 11 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -550,10 +550,7 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
github.com/quobyte/api v0.1.8/go.mod h1:jL7lIHrmqQ7yh05OJ+eEEdHr0u/kmT1Ff9iHd+4H6VI=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
github.com/robfig/cron v1.1.0 h1:jk4/Hud3TTdcrJgUOBgsqrZBarcxl6ADIjSC2iniwLY=
github.com/robfig/cron v1.1.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto=
Expand Down Expand Up @@ -848,8 +845,9 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
Expand Down Expand Up @@ -1007,6 +1005,7 @@ k8s.io/cloud-provider v0.21.1/go.mod h1:GgiRu7hOsZh3+VqMMbfLJJS9ZZM9A8k/YiZG8zkW
k8s.io/cluster-bootstrap v0.21.1/go.mod h1:izdXmPTfqI9gkjQLf9PQ1Y9/DSqG54zU2UByEzgdajs=
k8s.io/code-generator v0.18.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc=
k8s.io/code-generator v0.21.1/go.mod h1:hUlps5+9QaTrKx+jiM4rmq7YmH8wPOIko64uZCHDh6Q=
k8s.io/component-base v0.21.1 h1:iLpj2btXbR326s/xNQWmPNGu0gaYSjzn7IN/5i28nQw=
k8s.io/component-base v0.21.1/go.mod h1:NgzFZ2qu4m1juby4TnrmpR8adRk6ka62YdH5DkIIyKA=
k8s.io/component-helpers v0.21.1/go.mod h1:FtC1flbiQlosHQrLrRUulnKxE4ajgWCGy/67fT2GRlQ=
k8s.io/controller-manager v0.21.1/go.mod h1:8ugs8DCcHqybiwdVERhnnyGoS5Ksq/ea1p2B0CosHyc=
Expand Down Expand Up @@ -1041,20 +1040,21 @@ k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/
kmodules.xyz/client-go v0.0.0-20210617233340-13d22e91512b/go.mod h1:A6GAK6xP5zBuWK6A/vUkkjKzcuywkms7fIxRf5wblO4=
kmodules.xyz/client-go v0.0.0-20211013093146-1fbfd52e78c9/go.mod h1:0gkPeALtYjB27OHt4rd6+ZmMgoVTHVLtEJQeU23/gtA=
kmodules.xyz/client-go v0.0.0-20211107190155-5bb4090d2728/go.mod h1:ENUu8pPK19xzBkVpAJHoGCI2QRvb1SqffWRt0K2sV5I=
kmodules.xyz/client-go v0.0.0-20211122091731-6c471b24a4ea h1:z5Li57oxum0018ryWpI5w5HYVFgI2S2cVj27R76IRnU=
kmodules.xyz/client-go v0.0.0-20211122091731-6c471b24a4ea/go.mod h1:ENUu8pPK19xzBkVpAJHoGCI2QRvb1SqffWRt0K2sV5I=
kmodules.xyz/client-go v0.0.0-20220104114408-2a3a05dbe89f/go.mod h1:xxl1ve1Obe4xaW+XjXsNHyLTni4QPIvHn9TfnYEoQRo=
kmodules.xyz/client-go v0.0.0-20220108081101-27afc2ac4ebe h1:EZE/eC9UF5/wcdHiinzQDVshOceIP9uRxT7qM4BnUQg=
kmodules.xyz/client-go v0.0.0-20220108081101-27afc2ac4ebe/go.mod h1:xxl1ve1Obe4xaW+XjXsNHyLTni4QPIvHn9TfnYEoQRo=
kmodules.xyz/constants v0.0.0-20210218100002-2c304bfda278/go.mod h1:DbiFk1bJ1KEO94t1SlAn7tzc+Zz95rSXgyUKa2nzPmY=
kmodules.xyz/crd-schema-fuzz v0.0.0-20210618002152-fae23aef5fb4/go.mod h1:IIkUctlfoptoci0BOrsUf8ya+MOG5uaeh1PE4uzaIbA=
kmodules.xyz/custom-resources v0.0.0-20211122142737-3bf3dbd8ac52 h1:UWVpU7y5znTUusU+JhPB+ojh26f6K2v8sNb37U1DolQ=
kmodules.xyz/custom-resources v0.0.0-20211122142737-3bf3dbd8ac52/go.mod h1:yHLFe4wVYxepTnN00CFUf29xH+jEHDokq6d2fbp9pks=
kmodules.xyz/custom-resources v0.0.0-20220104123914-3c036dd7c1cd h1:EMcK5eA42CO9Cn290VOy2WIt2YSEbQItlMq8lU+831M=
kmodules.xyz/custom-resources v0.0.0-20220104123914-3c036dd7c1cd/go.mod h1:/XjDeILFV2wBota5kHo21DMvOt08nSAk1vm6buCuwt4=
kmodules.xyz/objectstore-api v0.0.0-20211116180107-8720be0c9bf7 h1:JIAEFjN3GDhLEG1Fh1zYpy/QFyyN337mJTM+ODEGosg=
kmodules.xyz/objectstore-api v0.0.0-20211116180107-8720be0c9bf7/go.mod h1:IICnDdPFOEeGXdaPVHOGYfdwD1cyh/p1I/TWMkyNTIE=
kmodules.xyz/offshoot-api v0.0.0-20210829122105-6f4d481b0c61/go.mod h1:3LECbAL3FgbyK80NP3V3Pmiuo/a3hFWg/PR6SPFhTns=
kmodules.xyz/offshoot-api v0.0.0-20211103060642-3e217667cf41 h1:GCDyb1fbgCHUNuFk9MecSFvzMq/WCeGk8pDEtqivziA=
kmodules.xyz/offshoot-api v0.0.0-20211103060642-3e217667cf41/go.mod h1:sJYyxf84ZvbVz4SivxMgSelGRYn19wOLUtObiEncCxk=
kmodules.xyz/prober v0.0.0-20210618020259-5836fb959027 h1:rtSPsruEkoBOLcbJkxMQjnPafpv8/vJnfWlTIj9yRBg=
kmodules.xyz/prober v0.0.0-20210618020259-5836fb959027/go.mod h1:H4NcvS1RQxeXtQONALbU6r9OT3Xa7cMhfamX/k9t9wo=
kmodules.xyz/resource-metrics v0.0.5/go.mod h1:6Dv63HDgp83DhA+lZNB7GIQR6PLjNrYW6ghQKioQzII=
kmodules.xyz/resource-metrics v0.0.6/go.mod h1:M7rWuo2qh3BpHhogiEVPnvGY9Xx4Pfygqn1Rex8YbgM=
modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=
modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
Expand All @@ -1079,5 +1079,5 @@ sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=
stash.appscode.dev/apimachinery v0.17.0 h1:v0razjOko1K0npOwRzvtymdMByL/cWSNQBgaIq7KaoU=
stash.appscode.dev/apimachinery v0.17.0/go.mod h1:hmxBy6Ei6RjBLgXw4A1hE4eyEgsa43H2LCs1yDfI3GM=
stash.appscode.dev/apimachinery v0.17.1-0.20220113052814-7da4b19c88a3 h1:jWOAWMKDc15vVPqq4pToYyRkitlGh6Y2+1V5QgGZsYY=
stash.appscode.dev/apimachinery v0.17.1-0.20220113052814-7da4b19c88a3/go.mod h1:BGM/ztGKtXZrtz/voZzRqop8KbbZ+pFI4YeSLVb2MB0=
59 changes: 27 additions & 32 deletions pkg/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (
"context"
"fmt"
"path/filepath"
"strings"

api_v1beta1 "stash.appscode.dev/apimachinery/apis/stash/v1beta1"
stash "stash.appscode.dev/apimachinery/client/clientset/versioned"
Expand Down Expand Up @@ -60,7 +59,7 @@ func NewCmdBackup() *cobra.Command {
Short: "Takes a backup of Postgres DB",
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
flags.EnsureRequiredFlags(cmd, "appbinding", "provider", "secret-dir")
flags.EnsureRequiredFlags(cmd, "appbinding", "provider", "storage-secret-name", "storage-secret-namespace")

// prepare client
config, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfigPath)
Expand Down Expand Up @@ -114,22 +113,24 @@ func NewCmdBackup() *cobra.Command {
},
}

cmd.Flags().StringVar(&opt.backupCMD, "backup-cmd", opt.pgArgs, "Backup command to take a database dump (can only be pg_dumpall or pg_dump)")
cmd.Flags().StringVar(&opt.backupCMD, "backup-cmd", PgDumpallCMD, "Backup command to take a database dump (can only be pg_dumpall or pg_dump)")
cmd.Flags().StringVar(&opt.pgArgs, "pg-args", opt.pgArgs, "Additional arguments")
cmd.Flags().StringVar(&opt.user, "user", DefaultPostgresUser, "Specifies database user (not applicable for basic authentication)")
cmd.Flags().Int32Var(&opt.waitTimeout, "wait-timeout", opt.waitTimeout, "Time limit to wait for the database to be ready")

cmd.Flags().StringVar(&masterURL, "master", masterURL, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
cmd.Flags().StringVar(&kubeconfigPath, "kubeconfig", kubeconfigPath, "Path to kubeconfig file with authorization information (the master location is set by the master flag).")
cmd.Flags().StringVar(&opt.namespace, "namespace", "default", "Namespace of Backup/Restore Session")
cmd.Flags().StringVar(&opt.backupSessionName, "backupsession", opt.backupSessionName, "Name of the Backup Session")
cmd.Flags().StringVar(&opt.appBindingName, "appbinding", opt.appBindingName, "Name of the app binding")
cmd.Flags().StringVar(&opt.storageSecret.Name, "storage-secret-name", opt.storageSecret.Name, "Name of the storage secret")
cmd.Flags().StringVar(&opt.storageSecret.Namespace, "storage-secret-namespace", opt.storageSecret.Namespace, "Namespace of the storage secret")

cmd.Flags().StringVar(&opt.setupOptions.Provider, "provider", opt.setupOptions.Provider, "Backend provider (i.e. gcs, s3, azure etc)")
cmd.Flags().StringVar(&opt.setupOptions.Bucket, "bucket", opt.setupOptions.Bucket, "Name of the cloud bucket/container (keep empty for local backend)")
cmd.Flags().StringVar(&opt.setupOptions.Endpoint, "endpoint", opt.setupOptions.Endpoint, "Endpoint for s3/s3 compatible backend or REST server URL")
cmd.Flags().StringVar(&opt.setupOptions.Region, "region", opt.setupOptions.Region, "Region for s3/s3 compatible backend")
cmd.Flags().StringVar(&opt.setupOptions.Path, "path", opt.setupOptions.Path, "Directory inside the bucket where backup will be stored")
cmd.Flags().StringVar(&opt.setupOptions.SecretDir, "secret-dir", opt.setupOptions.SecretDir, "Directory where storage secret has been mounted")
cmd.Flags().StringVar(&opt.setupOptions.ScratchDir, "scratch-dir", opt.setupOptions.ScratchDir, "Temporary directory")
cmd.Flags().BoolVar(&opt.setupOptions.EnableCache, "enable-cache", opt.setupOptions.EnableCache, "Specify whether to enable caching for restic")
cmd.Flags().Int64Var(&opt.setupOptions.MaxConnections, "max-connections", opt.setupOptions.MaxConnections, "Specify maximum concurrent connections for GCS, Azure and B2 backend")
Expand All @@ -152,6 +153,11 @@ func NewCmdBackup() *cobra.Command {
}

func (opt *postgresOptions) backupPostgreSQL(targetRef api_v1beta1.TargetRef) (*restic.BackupOutput, error) {
var err error
opt.setupOptions.StorageSecret, err = opt.kubeClient.CoreV1().Secrets(opt.storageSecret.Namespace).Get(context.TODO(), opt.storageSecret.Name, metav1.GetOptions{})
if err != nil {
return nil, err
}
// if any pre-backup actions has been assigned to it, execute them
actionOptions := api_util.ActionOptions{
StashClient: opt.stashClient,
Expand All @@ -160,7 +166,7 @@ func (opt *postgresOptions) backupPostgreSQL(targetRef api_v1beta1.TargetRef) (*
BackupSessionName: opt.backupSessionName,
Namespace: opt.namespace,
}
err := api_util.ExecutePreBackupActions(actionOptions)
err = api_util.ExecutePreBackupActions(actionOptions)
if err != nil {
return nil, err
}
Expand All @@ -179,19 +185,24 @@ func (opt *postgresOptions) backupPostgreSQL(targetRef api_v1beta1.TargetRef) (*
return nil, err
}

// get app binding
appBinding, err := opt.catalogClient.AppcatalogV1alpha1().AppBindings(opt.namespace).Get(context.TODO(), opt.appBindingName, metav1.GetOptions{})
if err != nil {
return nil, err
}
// get secret
appBindingSecret, err := opt.kubeClient.CoreV1().Secrets(opt.namespace).Get(context.TODO(), appBinding.Spec.Secret.Name, metav1.GetOptions{})

session := opt.newSessionWrapper(opt.backupCMD)

err = opt.setDatabaseCredentials(appBinding, session)
if err != nil {
return nil, err
}

err = session.setDatabaseConnectionParameters(appBinding)
if err != nil {
return nil, err
}

// transform secret
err = appBinding.TransformSecret(opt.kubeClient, appBindingSecret.Data)
err = session.setTLSParameters(appBinding, opt.setupOptions.ScratchDir)
if err != nil {
return nil, err
}
Expand All @@ -203,36 +214,20 @@ func (opt *postgresOptions) backupPostgreSQL(targetRef api_v1beta1.TargetRef) (*
return nil, fmt.Errorf("invalid pg backup command: expected %s or %s, but instead got %s", PgDumpCMD, PgDumpallCMD, pgBackupCMD)
}

if appBinding.Spec.ClientConfig.Service.Port == 0 {
appBinding.Spec.ClientConfig.Service.Port = 5432
}

resticWrapper, userName, err := opt.GetResticWrapperWithPGConnectorVariables(appBinding, appBindingSecret)
err = session.waitForDBReady(opt.waitTimeout)
if err != nil {
return nil, err
}
// set env for pg_dump/pg_dumpall
dumpCommand := restic.Command{
Name: pgBackupCMD,
Args: []interface{}{
fmt.Sprintf("--host=%s", appBinding.Spec.ClientConfig.Service.Name),
fmt.Sprintf("--port=%d", appBinding.Spec.ClientConfig.Service.Port),
fmt.Sprintf("--username=%s", userName),
},
}
for _, arg := range strings.Fields(opt.pgArgs) {
dumpCommand.Args = append(dumpCommand.Args, arg)
}
// add the dump command into stdin pipe commands
opt.backupOptions.StdinPipeCommands = append(opt.backupOptions.StdinPipeCommands, dumpCommand)

// wait for DB ready
err = opt.waitForDBReady(appBinding, appBindingSecret, opt.waitTimeout)
session.setUserArgs(opt.pgArgs)

// add the dump command into stdin pipe commands
opt.backupOptions.StdinPipeCommands = append(opt.backupOptions.StdinPipeCommands, *session.cmd)
resticWrapper, err := restic.NewResticWrapperFromShell(opt.setupOptions, session.sh)
if err != nil {
return nil, err
}

// Run backup
return resticWrapper.RunBackup(opt.backupOptions, targetRef)

}
Loading

0 comments on commit d375d3e

Please sign in to comment.