Skip to content

Commit afad5db

Browse files
hossainemruztamalsaha
hossainemruz
authored and
tamalsaha
committed
Refactor error + metric handling (#7)
1 parent 248d0ea commit afad5db

File tree

15 files changed

+629
-389
lines changed

15 files changed

+629
-389
lines changed

‎chart/stash-postgres/templates/postgres-backup-function.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ spec:
2121
- --pg-args=${pgArgs:={{ .Values.backup.pgArgs }}} # optional arguments pass to pgdump command
2222
# target information
2323
- --namespace=${NAMESPACE:=default}
24-
- --app-binding=${TARGET_NAME:=}
24+
- --appbinding=${TARGET_NAME:=}
2525
# cleanup information
2626
- --retention-keep-last=${RETENTION_KEEP_LAST:=0}
2727
- --retention-keep-hourly=${RETENTION_KEEP_HOURLY:=0}

‎chart/stash-postgres/templates/postgres-restore-function.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ spec:
2222
- --pg-args=${pgArgs:={{ .Values.restore.pgArgs }}} # optional arguments pass to pgdump command
2323
# target information
2424
- --namespace=${NAMESPACE:=default}
25-
- --app-binding=${TARGET_NAME:=}
25+
- --appbinding=${TARGET_NAME:=}
2626
- --snapshot=${RESTORE_SNAPSHOTS:=}
2727
# output & metric information
2828
- --output-dir=${outputDir:=}

‎go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ require (
99
k8s.io/client-go v11.0.0+incompatible
1010
kmodules.xyz/client-go v0.0.0-20190808141354-bbb9e14f60ab
1111
kmodules.xyz/custom-resources v0.0.0-20190808144301-114abf10dfe2
12-
stash.appscode.dev/stash v0.9.0-rc.0.0.20190828102811-1b3de9eacd2c
12+
stash.appscode.dev/stash v0.9.0-rc.0.0.20190910104640-5f49f5d214aa
1313
)
1414

1515
replace (

‎go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -444,5 +444,5 @@ sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5
444444
sigs.k8s.io/structured-merge-diff v0.0.0-20190302045857-e85c7b244fd2/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
445445
sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
446446
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
447-
stash.appscode.dev/stash v0.9.0-rc.0.0.20190828102811-1b3de9eacd2c h1:1/v6yHdS1H4p5IRBwO/xUe0rXftlk6wcBu67Qe+kw4E=
448-
stash.appscode.dev/stash v0.9.0-rc.0.0.20190828102811-1b3de9eacd2c/go.mod h1:6WrRpjkI1av7305B60BLRx+PZzwMAoiGmOd5ewYAAyw=
447+
stash.appscode.dev/stash v0.9.0-rc.0.0.20190910104640-5f49f5d214aa h1:/cDpVS+SY8jr0P9AmPiIkj6EaPrVJzsmGDX2RgUYtMU=
448+
stash.appscode.dev/stash v0.9.0-rc.0.0.20190910104640-5f49f5d214aa/go.mod h1:6WrRpjkI1av7305B60BLRx+PZzwMAoiGmOd5ewYAAyw=

‎pkg/backup.go

Lines changed: 102 additions & 115 deletions
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,32 @@
11
package pkg
22

33
import (
4-
"fmt"
5-
"os/exec"
64
"path/filepath"
7-
"time"
85

96
"github.com/appscode/go/flags"
10-
"github.com/appscode/go/log"
117
"github.com/spf13/cobra"
128
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
139
"k8s.io/client-go/kubernetes"
1410
"k8s.io/client-go/tools/clientcmd"
1511
appcatalog_cs "kmodules.xyz/custom-resources/client/clientset/versioned"
12+
api_v1beta1 "stash.appscode.dev/stash/apis/stash/v1beta1"
1613
"stash.appscode.dev/stash/pkg/restic"
1714
"stash.appscode.dev/stash/pkg/util"
1815
)
1916

20-
const (
21-
JobPGBackup = "stash-pg-backup"
22-
PostgresUser = "POSTGRES_USER"
23-
PostgresPassword = "POSTGRES_PASSWORD"
24-
EnvPgPassword = "PGPASSWORD"
25-
PgDumpFile = "dumpfile.sql"
26-
PgDumpCMD = "pg_dumpall"
27-
PgRestoreCMD = "psql"
28-
)
29-
3017
func NewCmdBackup() *cobra.Command {
3118
var (
3219
masterURL string
3320
kubeconfigPath string
34-
namespace string
35-
appBindingName string
36-
pgArgs string
37-
outputDir string
38-
setupOpt = restic.SetupOptions{
39-
ScratchDir: restic.DefaultScratchDir,
40-
EnableCache: false,
41-
}
42-
backupOpt = restic.BackupOptions{
43-
Host: restic.DefaultHost,
44-
StdinFileName: PgDumpFile,
21+
opt = postgresOptions{
22+
backupOptions: restic.BackupOptions{
23+
Host: restic.DefaultHost,
24+
StdinFileName: PgDumpFile,
25+
},
26+
setupOptions: restic.SetupOptions{
27+
ScratchDir: restic.DefaultScratchDir,
28+
EnableCache: false,
29+
},
4530
}
4631
)
4732

@@ -50,122 +35,124 @@ func NewCmdBackup() *cobra.Command {
5035
Short: "Takes a backup of Postgres DB",
5136
DisableAutoGenTag: true,
5237
RunE: func(cmd *cobra.Command, args []string) error {
53-
flags.EnsureRequiredFlags(cmd, "app-binding", "provider", "secret-dir")
54-
55-
// apply nice, ionice settings from env
56-
var err error
57-
setupOpt.Nice, err = util.NiceSettingsFromEnv()
58-
if err != nil {
59-
return util.HandleResticError(outputDir, restic.DefaultOutputFileName, err)
60-
}
61-
setupOpt.IONice, err = util.IONiceSettingsFromEnv()
62-
if err != nil {
63-
return util.HandleResticError(outputDir, restic.DefaultOutputFileName, err)
64-
}
38+
flags.EnsureRequiredFlags(cmd, "appbinding", "provider", "secret-dir")
6539

6640
// prepare client
6741
config, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfigPath)
6842
if err != nil {
6943
return err
7044
}
71-
kubeClient, err := kubernetes.NewForConfig(config)
72-
if err != nil {
73-
return err
74-
}
75-
appCatalogClient, err := appcatalog_cs.NewForConfig(config)
76-
if err != nil {
77-
return err
78-
}
79-
80-
// get app binding
81-
appBinding, err := appCatalogClient.AppcatalogV1alpha1().AppBindings(namespace).Get(appBindingName, metav1.GetOptions{})
45+
opt.kubeClient, err = kubernetes.NewForConfig(config)
8246
if err != nil {
8347
return err
8448
}
85-
// get secret
86-
appBindingSecret, err := kubeClient.CoreV1().Secrets(namespace).Get(appBinding.Spec.Secret.Name, metav1.GetOptions{})
49+
opt.catalogClient, err = appcatalog_cs.NewForConfig(config)
8750
if err != nil {
8851
return err
8952
}
9053

91-
// init restic wrapper
92-
resticWrapper, err := restic.NewResticWrapper(setupOpt)
54+
var backupOutput *restic.BackupOutput
55+
backupOutput, err = opt.backupPostgreSQL()
9356
if err != nil {
94-
return err
95-
}
96-
97-
// set env for pg_dump
98-
resticWrapper.SetEnv(EnvPgPassword, string(appBindingSecret.Data[PostgresPassword]))
99-
// setup pipe command
100-
backupOpt.StdinPipeCommand = restic.Command{
101-
Name: PgDumpCMD,
102-
Args: []interface{}{
103-
"-U", string(appBindingSecret.Data[PostgresUser]),
104-
"-h", appBinding.Spec.ClientConfig.Service.Name,
105-
},
106-
}
107-
if pgArgs != "" {
108-
backupOpt.StdinPipeCommand.Args = append(backupOpt.StdinPipeCommand.Args, pgArgs)
57+
backupOutput = &restic.BackupOutput{
58+
HostBackupStats: []api_v1beta1.HostBackupStats{
59+
{
60+
Hostname: opt.backupOptions.Host,
61+
Phase: api_v1beta1.HostBackupFailed,
62+
Error: err.Error(),
63+
},
64+
},
65+
}
10966
}
110-
111-
// wait for DB ready
112-
waitForDBReady(appBinding.Spec.ClientConfig.Service.Name, appBinding.Spec.ClientConfig.Service.Port)
113-
114-
// Run backup
115-
backupOutput, backupErr := resticWrapper.RunBackup(backupOpt)
116-
11767
// If output directory specified, then write the output in "output.json" file in the specified directory
118-
if backupErr == nil && outputDir != "" {
119-
err := backupOutput.WriteOutput(filepath.Join(outputDir, restic.DefaultOutputFileName))
120-
if err != nil {
121-
return err
122-
}
68+
if opt.outputDir != "" {
69+
return backupOutput.WriteOutput(filepath.Join(opt.outputDir, restic.DefaultOutputFileName))
12370
}
124-
return backupErr
71+
72+
return nil
12573
},
12674
}
12775

128-
cmd.Flags().StringVar(&pgArgs, "pg-args", pgArgs, "Additional arguments")
76+
cmd.Flags().StringVar(&opt.pgArgs, "pg-args", opt.pgArgs, "Additional arguments")
12977

13078
cmd.Flags().StringVar(&masterURL, "master", masterURL, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
13179
cmd.Flags().StringVar(&kubeconfigPath, "kubeconfig", kubeconfigPath, "Path to kubeconfig file with authorization information (the master location is set by the master flag).")
132-
cmd.Flags().StringVar(&namespace, "namespace", "default", "Namespace of Backup/Restore Session")
133-
cmd.Flags().StringVar(&appBindingName, "app-binding", appBindingName, "Name of the app binding")
134-
135-
cmd.Flags().StringVar(&setupOpt.Provider, "provider", setupOpt.Provider, "Backend provider (i.e. gcs, s3, azure etc)")
136-
cmd.Flags().StringVar(&setupOpt.Bucket, "bucket", setupOpt.Bucket, "Name of the cloud bucket/container (keep empty for local backend)")
137-
cmd.Flags().StringVar(&setupOpt.Endpoint, "endpoint", setupOpt.Endpoint, "Endpoint for s3/s3 compatible backend or REST server URL")
138-
cmd.Flags().StringVar(&setupOpt.Path, "path", setupOpt.Path, "Directory inside the bucket where backup will be stored")
139-
cmd.Flags().StringVar(&setupOpt.SecretDir, "secret-dir", setupOpt.SecretDir, "Directory where storage secret has been mounted")
140-
cmd.Flags().StringVar(&setupOpt.ScratchDir, "scratch-dir", setupOpt.ScratchDir, "Temporary directory")
141-
cmd.Flags().BoolVar(&setupOpt.EnableCache, "enable-cache", setupOpt.EnableCache, "Specify whether to enable caching for restic")
142-
cmd.Flags().IntVar(&setupOpt.MaxConnections, "max-connections", setupOpt.MaxConnections, "Specify maximum concurrent connections for GCS, Azure and B2 backend")
143-
144-
cmd.Flags().StringVar(&backupOpt.Host, "hostname", backupOpt.Host, "Name of the host machine")
145-
146-
cmd.Flags().IntVar(&backupOpt.RetentionPolicy.KeepLast, "retention-keep-last", backupOpt.RetentionPolicy.KeepLast, "Specify value for retention strategy")
147-
cmd.Flags().IntVar(&backupOpt.RetentionPolicy.KeepHourly, "retention-keep-hourly", backupOpt.RetentionPolicy.KeepHourly, "Specify value for retention strategy")
148-
cmd.Flags().IntVar(&backupOpt.RetentionPolicy.KeepDaily, "retention-keep-daily", backupOpt.RetentionPolicy.KeepDaily, "Specify value for retention strategy")
149-
cmd.Flags().IntVar(&backupOpt.RetentionPolicy.KeepWeekly, "retention-keep-weekly", backupOpt.RetentionPolicy.KeepWeekly, "Specify value for retention strategy")
150-
cmd.Flags().IntVar(&backupOpt.RetentionPolicy.KeepMonthly, "retention-keep-monthly", backupOpt.RetentionPolicy.KeepMonthly, "Specify value for retention strategy")
151-
cmd.Flags().IntVar(&backupOpt.RetentionPolicy.KeepYearly, "retention-keep-yearly", backupOpt.RetentionPolicy.KeepYearly, "Specify value for retention strategy")
152-
cmd.Flags().StringSliceVar(&backupOpt.RetentionPolicy.KeepTags, "retention-keep-tags", backupOpt.RetentionPolicy.KeepTags, "Specify value for retention strategy")
153-
cmd.Flags().BoolVar(&backupOpt.RetentionPolicy.Prune, "retention-prune", backupOpt.RetentionPolicy.Prune, "Specify whether to prune old snapshot data")
154-
cmd.Flags().BoolVar(&backupOpt.RetentionPolicy.DryRun, "retention-dry-run", backupOpt.RetentionPolicy.DryRun, "Specify whether to test retention policy without deleting actual data")
155-
156-
cmd.Flags().StringVar(&outputDir, "output-dir", outputDir, "Directory where output.json file will be written (keep empty if you don't need to write output in file)")
80+
cmd.Flags().StringVar(&opt.namespace, "namespace", "default", "Namespace of Backup/Restore Session")
81+
cmd.Flags().StringVar(&opt.appBindingName, "appbinding", opt.appBindingName, "Name of the app binding")
82+
83+
cmd.Flags().StringVar(&opt.setupOptions.Provider, "provider", opt.setupOptions.Provider, "Backend provider (i.e. gcs, s3, azure etc)")
84+
cmd.Flags().StringVar(&opt.setupOptions.Bucket, "bucket", opt.setupOptions.Bucket, "Name of the cloud bucket/container (keep empty for local backend)")
85+
cmd.Flags().StringVar(&opt.setupOptions.Endpoint, "endpoint", opt.setupOptions.Endpoint, "Endpoint for s3/s3 compatible backend or REST server URL")
86+
cmd.Flags().StringVar(&opt.setupOptions.Path, "path", opt.setupOptions.Path, "Directory inside the bucket where backup will be stored")
87+
cmd.Flags().StringVar(&opt.setupOptions.SecretDir, "secret-dir", opt.setupOptions.SecretDir, "Directory where storage secret has been mounted")
88+
cmd.Flags().StringVar(&opt.setupOptions.ScratchDir, "scratch-dir", opt.setupOptions.ScratchDir, "Temporary directory")
89+
cmd.Flags().BoolVar(&opt.setupOptions.EnableCache, "enable-cache", opt.setupOptions.EnableCache, "Specify whether to enable caching for restic")
90+
cmd.Flags().IntVar(&opt.setupOptions.MaxConnections, "max-connections", opt.setupOptions.MaxConnections, "Specify maximum concurrent connections for GCS, Azure and B2 backend")
91+
92+
cmd.Flags().StringVar(&opt.backupOptions.Host, "hostname", opt.backupOptions.Host, "Name of the host machine")
93+
94+
cmd.Flags().IntVar(&opt.backupOptions.RetentionPolicy.KeepLast, "retention-keep-last", opt.backupOptions.RetentionPolicy.KeepLast, "Specify value for retention strategy")
95+
cmd.Flags().IntVar(&opt.backupOptions.RetentionPolicy.KeepHourly, "retention-keep-hourly", opt.backupOptions.RetentionPolicy.KeepHourly, "Specify value for retention strategy")
96+
cmd.Flags().IntVar(&opt.backupOptions.RetentionPolicy.KeepDaily, "retention-keep-daily", opt.backupOptions.RetentionPolicy.KeepDaily, "Specify value for retention strategy")
97+
cmd.Flags().IntVar(&opt.backupOptions.RetentionPolicy.KeepWeekly, "retention-keep-weekly", opt.backupOptions.RetentionPolicy.KeepWeekly, "Specify value for retention strategy")
98+
cmd.Flags().IntVar(&opt.backupOptions.RetentionPolicy.KeepMonthly, "retention-keep-monthly", opt.backupOptions.RetentionPolicy.KeepMonthly, "Specify value for retention strategy")
99+
cmd.Flags().IntVar(&opt.backupOptions.RetentionPolicy.KeepYearly, "retention-keep-yearly", opt.backupOptions.RetentionPolicy.KeepYearly, "Specify value for retention strategy")
100+
cmd.Flags().StringSliceVar(&opt.backupOptions.RetentionPolicy.KeepTags, "retention-keep-tags", opt.backupOptions.RetentionPolicy.KeepTags, "Specify value for retention strategy")
101+
cmd.Flags().BoolVar(&opt.backupOptions.RetentionPolicy.Prune, "retention-prune", opt.backupOptions.RetentionPolicy.Prune, "Specify whether to prune old snapshot data")
102+
cmd.Flags().BoolVar(&opt.backupOptions.RetentionPolicy.DryRun, "retention-dry-run", opt.backupOptions.RetentionPolicy.DryRun, "Specify whether to test retention policy without deleting actual data")
103+
104+
cmd.Flags().StringVar(&opt.outputDir, "output-dir", opt.outputDir, "Directory where output.json file will be written (keep empty if you don't need to write output in file)")
157105

158106
return cmd
159107
}
160108

161-
func waitForDBReady(host string, port int32) {
162-
log.Infoln("Checking database connection")
163-
cmd := fmt.Sprintf(`nc "%s" "%d" -w 30`, host, port)
164-
for {
165-
if err := exec.Command(cmd).Run(); err != nil {
166-
break
167-
}
168-
log.Infoln("Waiting... database is not ready yet")
169-
time.Sleep(5 * time.Second)
109+
func (opt *postgresOptions) backupPostgreSQL() (*restic.BackupOutput, error) {
110+
// apply nice, ionice settings from env
111+
var err error
112+
opt.setupOptions.Nice, err = util.NiceSettingsFromEnv()
113+
if err != nil {
114+
return nil, err
115+
}
116+
opt.setupOptions.IONice, err = util.IONiceSettingsFromEnv()
117+
if err != nil {
118+
return nil, err
119+
}
120+
121+
// get app binding
122+
appBinding, err := opt.catalogClient.AppcatalogV1alpha1().AppBindings(opt.namespace).Get(opt.appBindingName, metav1.GetOptions{})
123+
if err != nil {
124+
return nil, err
125+
}
126+
// get secret
127+
appBindingSecret, err := opt.kubeClient.CoreV1().Secrets(opt.namespace).Get(appBinding.Spec.Secret.Name, metav1.GetOptions{})
128+
if err != nil {
129+
return nil, err
170130
}
131+
132+
// init restic wrapper
133+
resticWrapper, err := restic.NewResticWrapper(opt.setupOptions)
134+
if err != nil {
135+
return nil, err
136+
}
137+
138+
// set env for pg_dump
139+
resticWrapper.SetEnv(EnvPgPassword, string(appBindingSecret.Data[PostgresPassword]))
140+
// setup pipe command
141+
opt.backupOptions.StdinPipeCommand = restic.Command{
142+
Name: PgDumpCMD,
143+
Args: []interface{}{
144+
"-U", string(appBindingSecret.Data[PostgresUser]),
145+
"-h", appBinding.Spec.ClientConfig.Service.Name,
146+
},
147+
}
148+
if opt.pgArgs != "" {
149+
opt.backupOptions.StdinPipeCommand.Args = append(opt.backupOptions.StdinPipeCommand.Args, opt.pgArgs)
150+
}
151+
152+
// wait for DB ready
153+
waitForDBReady(appBinding.Spec.ClientConfig.Service.Name, appBinding.Spec.ClientConfig.Service.Port)
154+
155+
// Run backup
156+
return resticWrapper.RunBackup(opt.backupOptions)
157+
171158
}

0 commit comments

Comments
 (0)