Skip to content

Commit c87ee33

Browse files
1gtmhmsayem
1gtm
and
hmsayem
authored
Add support for backup and restore from different namespace (#966) (#968)
/cherry-pick Signed-off-by: hmsayem hmsayem@appscode.com Co-authored-by: Hossain Mahmud <hmsayem@appscode.com>
1 parent c98dd3f commit c87ee33

File tree

126 files changed

+27285
-11336
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

126 files changed

+27285
-11336
lines changed

‎go.mod

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,10 @@ require (
1515
k8s.io/apimachinery v0.21.1
1616
k8s.io/client-go v0.21.1
1717
k8s.io/klog/v2 v2.8.0
18-
kmodules.xyz/client-go v0.0.0-20211122091731-6c471b24a4ea
19-
kmodules.xyz/custom-resources v0.0.0-20211122142737-3bf3dbd8ac52
18+
kmodules.xyz/client-go v0.0.0-20220108081101-27afc2ac4ebe
19+
kmodules.xyz/custom-resources v0.0.0-20220104123914-3c036dd7c1cd
2020
kmodules.xyz/offshoot-api v0.0.0-20211103060642-3e217667cf41
21-
stash.appscode.dev/apimachinery v0.17.0
21+
stash.appscode.dev/apimachinery v0.17.1-0.20220113052814-7da4b19c88a3
2222
)
2323

2424
replace bitbucket.org/ww/goautoneg => gomodules.xyz/goautoneg v0.0.0-20120707110453-a547fc61f48d

‎go.sum

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -550,10 +550,7 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
550550
github.com/quobyte/api v0.1.8/go.mod h1:jL7lIHrmqQ7yh05OJ+eEEdHr0u/kmT1Ff9iHd+4H6VI=
551551
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
552552
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
553-
github.com/robfig/cron v1.1.0 h1:jk4/Hud3TTdcrJgUOBgsqrZBarcxl6ADIjSC2iniwLY=
554553
github.com/robfig/cron v1.1.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
555-
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
556-
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
557554
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
558555
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
559556
github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto=
@@ -848,8 +845,9 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
848845
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
849846
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
850847
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
851-
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
852848
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
849+
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
850+
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
853851
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
854852
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
855853
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1007,6 +1005,7 @@ k8s.io/cloud-provider v0.21.1/go.mod h1:GgiRu7hOsZh3+VqMMbfLJJS9ZZM9A8k/YiZG8zkW
10071005
k8s.io/cluster-bootstrap v0.21.1/go.mod h1:izdXmPTfqI9gkjQLf9PQ1Y9/DSqG54zU2UByEzgdajs=
10081006
k8s.io/code-generator v0.18.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc=
10091007
k8s.io/code-generator v0.21.1/go.mod h1:hUlps5+9QaTrKx+jiM4rmq7YmH8wPOIko64uZCHDh6Q=
1008+
k8s.io/component-base v0.21.1 h1:iLpj2btXbR326s/xNQWmPNGu0gaYSjzn7IN/5i28nQw=
10101009
k8s.io/component-base v0.21.1/go.mod h1:NgzFZ2qu4m1juby4TnrmpR8adRk6ka62YdH5DkIIyKA=
10111010
k8s.io/component-helpers v0.21.1/go.mod h1:FtC1flbiQlosHQrLrRUulnKxE4ajgWCGy/67fT2GRlQ=
10121011
k8s.io/controller-manager v0.21.1/go.mod h1:8ugs8DCcHqybiwdVERhnnyGoS5Ksq/ea1p2B0CosHyc=
@@ -1041,20 +1040,21 @@ k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/
10411040
kmodules.xyz/client-go v0.0.0-20210617233340-13d22e91512b/go.mod h1:A6GAK6xP5zBuWK6A/vUkkjKzcuywkms7fIxRf5wblO4=
10421041
kmodules.xyz/client-go v0.0.0-20211013093146-1fbfd52e78c9/go.mod h1:0gkPeALtYjB27OHt4rd6+ZmMgoVTHVLtEJQeU23/gtA=
10431042
kmodules.xyz/client-go v0.0.0-20211107190155-5bb4090d2728/go.mod h1:ENUu8pPK19xzBkVpAJHoGCI2QRvb1SqffWRt0K2sV5I=
1044-
kmodules.xyz/client-go v0.0.0-20211122091731-6c471b24a4ea h1:z5Li57oxum0018ryWpI5w5HYVFgI2S2cVj27R76IRnU=
1045-
kmodules.xyz/client-go v0.0.0-20211122091731-6c471b24a4ea/go.mod h1:ENUu8pPK19xzBkVpAJHoGCI2QRvb1SqffWRt0K2sV5I=
1043+
kmodules.xyz/client-go v0.0.0-20220104114408-2a3a05dbe89f/go.mod h1:xxl1ve1Obe4xaW+XjXsNHyLTni4QPIvHn9TfnYEoQRo=
1044+
kmodules.xyz/client-go v0.0.0-20220108081101-27afc2ac4ebe h1:EZE/eC9UF5/wcdHiinzQDVshOceIP9uRxT7qM4BnUQg=
1045+
kmodules.xyz/client-go v0.0.0-20220108081101-27afc2ac4ebe/go.mod h1:xxl1ve1Obe4xaW+XjXsNHyLTni4QPIvHn9TfnYEoQRo=
10461046
kmodules.xyz/constants v0.0.0-20210218100002-2c304bfda278/go.mod h1:DbiFk1bJ1KEO94t1SlAn7tzc+Zz95rSXgyUKa2nzPmY=
10471047
kmodules.xyz/crd-schema-fuzz v0.0.0-20210618002152-fae23aef5fb4/go.mod h1:IIkUctlfoptoci0BOrsUf8ya+MOG5uaeh1PE4uzaIbA=
1048-
kmodules.xyz/custom-resources v0.0.0-20211122142737-3bf3dbd8ac52 h1:UWVpU7y5znTUusU+JhPB+ojh26f6K2v8sNb37U1DolQ=
1049-
kmodules.xyz/custom-resources v0.0.0-20211122142737-3bf3dbd8ac52/go.mod h1:yHLFe4wVYxepTnN00CFUf29xH+jEHDokq6d2fbp9pks=
1048+
kmodules.xyz/custom-resources v0.0.0-20220104123914-3c036dd7c1cd h1:EMcK5eA42CO9Cn290VOy2WIt2YSEbQItlMq8lU+831M=
1049+
kmodules.xyz/custom-resources v0.0.0-20220104123914-3c036dd7c1cd/go.mod h1:/XjDeILFV2wBota5kHo21DMvOt08nSAk1vm6buCuwt4=
10501050
kmodules.xyz/objectstore-api v0.0.0-20211116180107-8720be0c9bf7 h1:JIAEFjN3GDhLEG1Fh1zYpy/QFyyN337mJTM+ODEGosg=
10511051
kmodules.xyz/objectstore-api v0.0.0-20211116180107-8720be0c9bf7/go.mod h1:IICnDdPFOEeGXdaPVHOGYfdwD1cyh/p1I/TWMkyNTIE=
10521052
kmodules.xyz/offshoot-api v0.0.0-20210829122105-6f4d481b0c61/go.mod h1:3LECbAL3FgbyK80NP3V3Pmiuo/a3hFWg/PR6SPFhTns=
10531053
kmodules.xyz/offshoot-api v0.0.0-20211103060642-3e217667cf41 h1:GCDyb1fbgCHUNuFk9MecSFvzMq/WCeGk8pDEtqivziA=
10541054
kmodules.xyz/offshoot-api v0.0.0-20211103060642-3e217667cf41/go.mod h1:sJYyxf84ZvbVz4SivxMgSelGRYn19wOLUtObiEncCxk=
10551055
kmodules.xyz/prober v0.0.0-20210618020259-5836fb959027 h1:rtSPsruEkoBOLcbJkxMQjnPafpv8/vJnfWlTIj9yRBg=
10561056
kmodules.xyz/prober v0.0.0-20210618020259-5836fb959027/go.mod h1:H4NcvS1RQxeXtQONALbU6r9OT3Xa7cMhfamX/k9t9wo=
1057-
kmodules.xyz/resource-metrics v0.0.5/go.mod h1:6Dv63HDgp83DhA+lZNB7GIQR6PLjNrYW6ghQKioQzII=
1057+
kmodules.xyz/resource-metrics v0.0.6/go.mod h1:M7rWuo2qh3BpHhogiEVPnvGY9Xx4Pfygqn1Rex8YbgM=
10581058
modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=
10591059
modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
10601060
modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
@@ -1079,5 +1079,5 @@ sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
10791079
sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
10801080
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
10811081
sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=
1082-
stash.appscode.dev/apimachinery v0.17.0 h1:v0razjOko1K0npOwRzvtymdMByL/cWSNQBgaIq7KaoU=
1083-
stash.appscode.dev/apimachinery v0.17.0/go.mod h1:hmxBy6Ei6RjBLgXw4A1hE4eyEgsa43H2LCs1yDfI3GM=
1082+
stash.appscode.dev/apimachinery v0.17.1-0.20220113052814-7da4b19c88a3 h1:jWOAWMKDc15vVPqq4pToYyRkitlGh6Y2+1V5QgGZsYY=
1083+
stash.appscode.dev/apimachinery v0.17.1-0.20220113052814-7da4b19c88a3/go.mod h1:BGM/ztGKtXZrtz/voZzRqop8KbbZ+pFI4YeSLVb2MB0=

‎pkg/backup.go

Lines changed: 27 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ import (
2020
"context"
2121
"fmt"
2222
"path/filepath"
23-
"strings"
2423

2524
api_v1beta1 "stash.appscode.dev/apimachinery/apis/stash/v1beta1"
2625
stash "stash.appscode.dev/apimachinery/client/clientset/versioned"
@@ -60,7 +59,7 @@ func NewCmdBackup() *cobra.Command {
6059
Short: "Takes a backup of Postgres DB",
6160
DisableAutoGenTag: true,
6261
RunE: func(cmd *cobra.Command, args []string) error {
63-
flags.EnsureRequiredFlags(cmd, "appbinding", "provider", "secret-dir")
62+
flags.EnsureRequiredFlags(cmd, "appbinding", "provider", "storage-secret-name", "storage-secret-namespace")
6463

6564
// prepare client
6665
config, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfigPath)
@@ -114,22 +113,24 @@ func NewCmdBackup() *cobra.Command {
114113
},
115114
}
116115

117-
cmd.Flags().StringVar(&opt.backupCMD, "backup-cmd", opt.pgArgs, "Backup command to take a database dump (can only be pg_dumpall or pg_dump)")
116+
cmd.Flags().StringVar(&opt.backupCMD, "backup-cmd", PgDumpallCMD, "Backup command to take a database dump (can only be pg_dumpall or pg_dump)")
118117
cmd.Flags().StringVar(&opt.pgArgs, "pg-args", opt.pgArgs, "Additional arguments")
118+
cmd.Flags().StringVar(&opt.user, "user", DefaultPostgresUser, "Specifies database user (not applicable for basic authentication)")
119119
cmd.Flags().Int32Var(&opt.waitTimeout, "wait-timeout", opt.waitTimeout, "Time limit to wait for the database to be ready")
120120

121121
cmd.Flags().StringVar(&masterURL, "master", masterURL, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
122122
cmd.Flags().StringVar(&kubeconfigPath, "kubeconfig", kubeconfigPath, "Path to kubeconfig file with authorization information (the master location is set by the master flag).")
123123
cmd.Flags().StringVar(&opt.namespace, "namespace", "default", "Namespace of Backup/Restore Session")
124124
cmd.Flags().StringVar(&opt.backupSessionName, "backupsession", opt.backupSessionName, "Name of the Backup Session")
125125
cmd.Flags().StringVar(&opt.appBindingName, "appbinding", opt.appBindingName, "Name of the app binding")
126+
cmd.Flags().StringVar(&opt.storageSecret.Name, "storage-secret-name", opt.storageSecret.Name, "Name of the storage secret")
127+
cmd.Flags().StringVar(&opt.storageSecret.Namespace, "storage-secret-namespace", opt.storageSecret.Namespace, "Namespace of the storage secret")
126128

127129
cmd.Flags().StringVar(&opt.setupOptions.Provider, "provider", opt.setupOptions.Provider, "Backend provider (i.e. gcs, s3, azure etc)")
128130
cmd.Flags().StringVar(&opt.setupOptions.Bucket, "bucket", opt.setupOptions.Bucket, "Name of the cloud bucket/container (keep empty for local backend)")
129131
cmd.Flags().StringVar(&opt.setupOptions.Endpoint, "endpoint", opt.setupOptions.Endpoint, "Endpoint for s3/s3 compatible backend or REST server URL")
130132
cmd.Flags().StringVar(&opt.setupOptions.Region, "region", opt.setupOptions.Region, "Region for s3/s3 compatible backend")
131133
cmd.Flags().StringVar(&opt.setupOptions.Path, "path", opt.setupOptions.Path, "Directory inside the bucket where backup will be stored")
132-
cmd.Flags().StringVar(&opt.setupOptions.SecretDir, "secret-dir", opt.setupOptions.SecretDir, "Directory where storage secret has been mounted")
133134
cmd.Flags().StringVar(&opt.setupOptions.ScratchDir, "scratch-dir", opt.setupOptions.ScratchDir, "Temporary directory")
134135
cmd.Flags().BoolVar(&opt.setupOptions.EnableCache, "enable-cache", opt.setupOptions.EnableCache, "Specify whether to enable caching for restic")
135136
cmd.Flags().Int64Var(&opt.setupOptions.MaxConnections, "max-connections", opt.setupOptions.MaxConnections, "Specify maximum concurrent connections for GCS, Azure and B2 backend")
@@ -152,6 +153,11 @@ func NewCmdBackup() *cobra.Command {
152153
}
153154

154155
func (opt *postgresOptions) backupPostgreSQL(targetRef api_v1beta1.TargetRef) (*restic.BackupOutput, error) {
156+
var err error
157+
opt.setupOptions.StorageSecret, err = opt.kubeClient.CoreV1().Secrets(opt.storageSecret.Namespace).Get(context.TODO(), opt.storageSecret.Name, metav1.GetOptions{})
158+
if err != nil {
159+
return nil, err
160+
}
155161
// if any pre-backup actions has been assigned to it, execute them
156162
actionOptions := api_util.ActionOptions{
157163
StashClient: opt.stashClient,
@@ -160,7 +166,7 @@ func (opt *postgresOptions) backupPostgreSQL(targetRef api_v1beta1.TargetRef) (*
160166
BackupSessionName: opt.backupSessionName,
161167
Namespace: opt.namespace,
162168
}
163-
err := api_util.ExecutePreBackupActions(actionOptions)
169+
err = api_util.ExecutePreBackupActions(actionOptions)
164170
if err != nil {
165171
return nil, err
166172
}
@@ -179,19 +185,24 @@ func (opt *postgresOptions) backupPostgreSQL(targetRef api_v1beta1.TargetRef) (*
179185
return nil, err
180186
}
181187

182-
// get app binding
183188
appBinding, err := opt.catalogClient.AppcatalogV1alpha1().AppBindings(opt.namespace).Get(context.TODO(), opt.appBindingName, metav1.GetOptions{})
184189
if err != nil {
185190
return nil, err
186191
}
187-
// get secret
188-
appBindingSecret, err := opt.kubeClient.CoreV1().Secrets(opt.namespace).Get(context.TODO(), appBinding.Spec.Secret.Name, metav1.GetOptions{})
192+
193+
session := opt.newSessionWrapper(opt.backupCMD)
194+
195+
err = opt.setDatabaseCredentials(appBinding, session)
196+
if err != nil {
197+
return nil, err
198+
}
199+
200+
err = session.setDatabaseConnectionParameters(appBinding)
189201
if err != nil {
190202
return nil, err
191203
}
192204

193-
// transform secret
194-
err = appBinding.TransformSecret(opt.kubeClient, appBindingSecret.Data)
205+
err = session.setTLSParameters(appBinding, opt.setupOptions.ScratchDir)
195206
if err != nil {
196207
return nil, err
197208
}
@@ -203,36 +214,20 @@ func (opt *postgresOptions) backupPostgreSQL(targetRef api_v1beta1.TargetRef) (*
203214
return nil, fmt.Errorf("invalid pg backup command: expected %s or %s, but instead got %s", PgDumpCMD, PgDumpallCMD, pgBackupCMD)
204215
}
205216

206-
if appBinding.Spec.ClientConfig.Service.Port == 0 {
207-
appBinding.Spec.ClientConfig.Service.Port = 5432
208-
}
209-
210-
resticWrapper, userName, err := opt.GetResticWrapperWithPGConnectorVariables(appBinding, appBindingSecret)
217+
err = session.waitForDBReady(opt.waitTimeout)
211218
if err != nil {
212219
return nil, err
213220
}
214-
// set env for pg_dump/pg_dumpall
215-
dumpCommand := restic.Command{
216-
Name: pgBackupCMD,
217-
Args: []interface{}{
218-
fmt.Sprintf("--host=%s", appBinding.Spec.ClientConfig.Service.Name),
219-
fmt.Sprintf("--port=%d", appBinding.Spec.ClientConfig.Service.Port),
220-
fmt.Sprintf("--username=%s", userName),
221-
},
222-
}
223-
for _, arg := range strings.Fields(opt.pgArgs) {
224-
dumpCommand.Args = append(dumpCommand.Args, arg)
225-
}
226-
// add the dump command into stdin pipe commands
227-
opt.backupOptions.StdinPipeCommands = append(opt.backupOptions.StdinPipeCommands, dumpCommand)
228221

229-
// wait for DB ready
230-
err = opt.waitForDBReady(appBinding, appBindingSecret, opt.waitTimeout)
222+
session.setUserArgs(opt.pgArgs)
223+
224+
// add the dump command into stdin pipe commands
225+
opt.backupOptions.StdinPipeCommands = append(opt.backupOptions.StdinPipeCommands, *session.cmd)
226+
resticWrapper, err := restic.NewResticWrapperFromShell(opt.setupOptions, session.sh)
231227
if err != nil {
232228
return nil, err
233229
}
234230

235-
// Run backup
236231
return resticWrapper.RunBackup(opt.backupOptions, targetRef)
237232

238233
}

0 commit comments

Comments
 (0)