Skip to content

Commit 3fa7bdb

Browse files
Dipta Dastamalsaha
Dipta Das
authored and
tamalsaha
committed
Backup and restore Mongo DB (#699)
Ref: https://github.com/kubedb/mongodb/tree/master/hack/docker/mongo-tools/4.1.7
1 parent 4c39235 commit 3fa7bdb

File tree

3 files changed

+304
-0
lines changed

3 files changed

+304
-0
lines changed

‎backup_mongo.go

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
package cmds
2+
3+
import (
4+
"fmt"
5+
"path/filepath"
6+
7+
"github.com/appscode/go/flags"
8+
"github.com/appscode/stash/pkg/restic"
9+
"github.com/spf13/cobra"
10+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
11+
"k8s.io/apimachinery/pkg/util/errors"
12+
"k8s.io/client-go/kubernetes"
13+
"k8s.io/client-go/tools/clientcmd"
14+
appcatalog_cs "kmodules.xyz/custom-resources/client/clientset/versioned"
15+
)
16+
17+
const (
18+
JobMongoBackup = "stash-mongo-backup"
19+
MongoUser = "username"
20+
MongoPassword = "password"
21+
MongoDumpFile = "dumpfile.sql"
22+
MongoDumpCMD = "mongodump"
23+
MongoRestoreCMD = "mongorestore"
24+
)
25+
26+
func NewCmdBackupMongo() *cobra.Command {
27+
var (
28+
masterURL string
29+
kubeconfigPath string
30+
namespace string
31+
appBindingName string
32+
mongoArgs string
33+
outputDir string
34+
setupOpt = restic.SetupOptions{
35+
ScratchDir: restic.DefaultScratchDir,
36+
EnableCache: false,
37+
}
38+
backupOpt = restic.BackupOptions{
39+
StdinFileName: MongoDumpFile,
40+
}
41+
metrics = restic.MetricsOptions{
42+
JobName: JobMongoBackup,
43+
}
44+
)
45+
46+
cmd := &cobra.Command{
47+
Use: "backup-mongo",
48+
Short: "Takes a backup of Mongo DB",
49+
DisableAutoGenTag: true,
50+
RunE: func(cmd *cobra.Command, args []string) error {
51+
flags.EnsureRequiredFlags(cmd, "app-binding", "provider", "secret-dir")
52+
53+
// prepare client
54+
config, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfigPath)
55+
if err != nil {
56+
return err
57+
}
58+
kubeClient, err := kubernetes.NewForConfig(config)
59+
if err != nil {
60+
return err
61+
}
62+
appCatalogClient, err := appcatalog_cs.NewForConfig(config)
63+
if err != nil {
64+
return err
65+
}
66+
67+
// get app binding
68+
appBinding, err := appCatalogClient.AppcatalogV1alpha1().AppBindings(namespace).Get(appBindingName, metav1.GetOptions{})
69+
if err != nil {
70+
return err
71+
}
72+
// get secret
73+
appBindingSecret, err := kubeClient.CoreV1().Secrets(namespace).Get(appBinding.Spec.Secret.Name, metav1.GetOptions{})
74+
if err != nil {
75+
return err
76+
}
77+
78+
// init restic wrapper
79+
resticWrapper, err := restic.NewResticWrapper(setupOpt)
80+
if err != nil {
81+
return err
82+
}
83+
// hide password, don't print cmd
84+
resticWrapper.HideCMD()
85+
86+
// setup pipe command
87+
backupOpt.StdinPipeCommand = restic.Command{
88+
Name: MongoDumpCMD,
89+
Args: []interface{}{
90+
"--host", appBinding.Spec.ClientConfig.Service.Name,
91+
"--port", fmt.Sprint(appBinding.Spec.ClientConfig.Service.Port),
92+
"--username", string(appBindingSecret.Data[MongoUser]),
93+
"--password=" + string(appBindingSecret.Data[MongoPassword]),
94+
"--archive",
95+
},
96+
}
97+
if mongoArgs != "" {
98+
backupOpt.StdinPipeCommand.Args = append(backupOpt.StdinPipeCommand.Args, mongoArgs)
99+
}
100+
101+
// wait for DB ready
102+
waitForDBReady(appBinding.Spec.ClientConfig.Service.Name, appBinding.Spec.ClientConfig.Service.Port)
103+
104+
// Run backup
105+
backupOutput, backupErr := resticWrapper.RunBackup(backupOpt)
106+
// If metrics are enabled then generate metrics
107+
if metrics.Enabled {
108+
err := backupOutput.HandleMetrics(&metrics, backupErr)
109+
if err != nil {
110+
return errors.NewAggregate([]error{backupErr, err})
111+
}
112+
}
113+
// If output directory specified, then write the output in "output.json" file in the specified directory
114+
if backupErr == nil && outputDir != "" {
115+
err := backupOutput.WriteOutput(filepath.Join(outputDir, restic.DefaultOutputFileName))
116+
if err != nil {
117+
return err
118+
}
119+
}
120+
return backupErr
121+
},
122+
}
123+
124+
cmd.Flags().StringVar(&mongoArgs, "mongo-args", mongoArgs, "Additional arguments")
125+
126+
cmd.Flags().StringVar(&masterURL, "master", masterURL, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
127+
cmd.Flags().StringVar(&kubeconfigPath, "kubeconfig", kubeconfigPath, "Path to kubeconfig file with authorization information (the master location is set by the master flag).")
128+
cmd.Flags().StringVar(&namespace, "namespace", "default", "Namespace of Backup/Restore Session")
129+
cmd.Flags().StringVar(&appBindingName, "app-binding", appBindingName, "Name of the app binding")
130+
131+
cmd.Flags().StringVar(&setupOpt.Provider, "provider", setupOpt.Provider, "Backend provider (i.e. gcs, s3, azure etc)")
132+
cmd.Flags().StringVar(&setupOpt.Bucket, "bucket", setupOpt.Bucket, "Name of the cloud bucket/container (keep empty for local backend)")
133+
cmd.Flags().StringVar(&setupOpt.Endpoint, "endpoint", setupOpt.Endpoint, "Endpoint for s3/s3 compatible backend")
134+
cmd.Flags().StringVar(&setupOpt.Path, "path", setupOpt.Path, "Directory inside the bucket where backup will be stored")
135+
cmd.Flags().StringVar(&setupOpt.SecretDir, "secret-dir", setupOpt.SecretDir, "Directory where storage secret has been mounted")
136+
cmd.Flags().StringVar(&setupOpt.ScratchDir, "scratch-dir", setupOpt.ScratchDir, "Temporary directory")
137+
cmd.Flags().BoolVar(&setupOpt.EnableCache, "enable-cache", setupOpt.EnableCache, "Specify weather to enable caching for restic")
138+
139+
cmd.Flags().StringVar(&backupOpt.Host, "hostname", backupOpt.Host, "Name of the host machine")
140+
141+
cmd.Flags().IntVar(&backupOpt.RetentionPolicy.KeepLast, "retention-keep-last", backupOpt.RetentionPolicy.KeepLast, "Specify value for retention strategy")
142+
cmd.Flags().IntVar(&backupOpt.RetentionPolicy.KeepHourly, "retention-keep-hourly", backupOpt.RetentionPolicy.KeepHourly, "Specify value for retention strategy")
143+
cmd.Flags().IntVar(&backupOpt.RetentionPolicy.KeepDaily, "retention-keep-daily", backupOpt.RetentionPolicy.KeepDaily, "Specify value for retention strategy")
144+
cmd.Flags().IntVar(&backupOpt.RetentionPolicy.KeepWeekly, "retention-keep-weekly", backupOpt.RetentionPolicy.KeepWeekly, "Specify value for retention strategy")
145+
cmd.Flags().IntVar(&backupOpt.RetentionPolicy.KeepMonthly, "retention-keep-monthly", backupOpt.RetentionPolicy.KeepMonthly, "Specify value for retention strategy")
146+
cmd.Flags().IntVar(&backupOpt.RetentionPolicy.KeepYearly, "retention-keep-yearly", backupOpt.RetentionPolicy.KeepYearly, "Specify value for retention strategy")
147+
cmd.Flags().StringSliceVar(&backupOpt.RetentionPolicy.KeepTags, "retention-keep-tags", backupOpt.RetentionPolicy.KeepTags, "Specify value for retention strategy")
148+
cmd.Flags().BoolVar(&backupOpt.RetentionPolicy.Prune, "retention-prune", backupOpt.RetentionPolicy.Prune, "Specify weather to prune old snapshot data")
149+
cmd.Flags().BoolVar(&backupOpt.RetentionPolicy.DryRun, "retention-dry-run", backupOpt.RetentionPolicy.DryRun, "Specify weather to test retention policy without deleting actual data")
150+
151+
cmd.Flags().StringVar(&outputDir, "output-dir", outputDir, "Directory where output.json file will be written (keep empty if you don't need to write output in file)")
152+
153+
cmd.Flags().BoolVar(&metrics.Enabled, "metrics-enabled", metrics.Enabled, "Specify weather to export Prometheus metrics")
154+
cmd.Flags().StringVar(&metrics.PushgatewayURL, "metrics-pushgateway-url", metrics.PushgatewayURL, "Pushgateway URL where the metrics will be pushed")
155+
cmd.Flags().StringVar(&metrics.MetricFileDir, "metrics-dir", metrics.MetricFileDir, "Directory where to write metric.prom file (keep empty if you don't want to write metric in a text file)")
156+
cmd.Flags().StringSliceVar(&metrics.Labels, "metrics-labels", metrics.Labels, "Labels to apply in exported metrics")
157+
158+
return cmd
159+
}

‎restore_mongo.go

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
package cmds
2+
3+
import (
4+
"fmt"
5+
"path/filepath"
6+
7+
"github.com/appscode/go/flags"
8+
"github.com/appscode/stash/pkg/restic"
9+
"github.com/spf13/cobra"
10+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
11+
"k8s.io/apimachinery/pkg/util/errors"
12+
"k8s.io/client-go/kubernetes"
13+
"k8s.io/client-go/tools/clientcmd"
14+
appcatalog_cs "kmodules.xyz/custom-resources/client/clientset/versioned"
15+
)
16+
17+
func NewCmdRestoreMongo() *cobra.Command {
18+
var (
19+
masterURL string
20+
kubeconfigPath string
21+
namespace string
22+
appBindingName string
23+
outputDir string
24+
mongoArgs string
25+
setupOpt = restic.SetupOptions{
26+
ScratchDir: restic.DefaultScratchDir,
27+
EnableCache: false,
28+
}
29+
dumpOpt = restic.DumpOptions{
30+
FileName: MongoDumpFile,
31+
}
32+
metrics = restic.MetricsOptions{
33+
JobName: JobMongoBackup,
34+
}
35+
)
36+
37+
cmd := &cobra.Command{
38+
Use: "restore-mongo",
39+
Short: "Restores Mongo DB Backup",
40+
DisableAutoGenTag: true,
41+
RunE: func(cmd *cobra.Command, args []string) error {
42+
flags.EnsureRequiredFlags(cmd, "app-binding", "provider", "secret-dir")
43+
44+
// prepare client
45+
config, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfigPath)
46+
if err != nil {
47+
return err
48+
}
49+
kubeClient, err := kubernetes.NewForConfig(config)
50+
if err != nil {
51+
return err
52+
}
53+
appCatalogClient, err := appcatalog_cs.NewForConfig(config)
54+
if err != nil {
55+
return err
56+
}
57+
58+
// get app binding
59+
appBinding, err := appCatalogClient.AppcatalogV1alpha1().AppBindings(namespace).Get(appBindingName, metav1.GetOptions{})
60+
if err != nil {
61+
return err
62+
}
63+
// get secret
64+
appBindingSecret, err := kubeClient.CoreV1().Secrets(namespace).Get(appBinding.Spec.Secret.Name, metav1.GetOptions{})
65+
if err != nil {
66+
return err
67+
}
68+
69+
// init restic wrapper
70+
resticWrapper, err := restic.NewResticWrapper(setupOpt)
71+
if err != nil {
72+
return err
73+
}
74+
// hide password, don't print cmd
75+
resticWrapper.HideCMD()
76+
77+
// setup pipe command
78+
dumpOpt.StdoutPipeCommand = restic.Command{
79+
Name: MongoRestoreCMD,
80+
Args: []interface{}{
81+
"--host", appBinding.Spec.ClientConfig.Service.Name,
82+
"--port", fmt.Sprint(appBinding.Spec.ClientConfig.Service.Port),
83+
"--username", string(appBindingSecret.Data[MongoUser]),
84+
"--password=" + string(appBindingSecret.Data[MongoPassword]),
85+
"--archive",
86+
},
87+
}
88+
if mongoArgs != "" {
89+
dumpOpt.StdoutPipeCommand.Args = append(dumpOpt.StdoutPipeCommand.Args, mongoArgs)
90+
}
91+
92+
// wait for DB ready
93+
waitForDBReady(appBinding.Spec.ClientConfig.Service.Name, appBinding.Spec.ClientConfig.Service.Port)
94+
95+
// Run dump
96+
dumpOutput, backupErr := resticWrapper.Dump(dumpOpt)
97+
// If metrics are enabled then generate metrics
98+
if metrics.Enabled {
99+
err := dumpOutput.HandleMetrics(&metrics, backupErr)
100+
if err != nil {
101+
return errors.NewAggregate([]error{backupErr, err})
102+
}
103+
}
104+
// If output directory specified, then write the output in "output.json" file in the specified directory
105+
if backupErr == nil && outputDir != "" {
106+
err := dumpOutput.WriteOutput(filepath.Join(outputDir, restic.DefaultOutputFileName))
107+
if err != nil {
108+
return err
109+
}
110+
}
111+
return backupErr
112+
},
113+
}
114+
115+
cmd.Flags().StringVar(&mongoArgs, "mongo-args", mongoArgs, "Additional arguments")
116+
117+
cmd.Flags().StringVar(&masterURL, "master", masterURL, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
118+
cmd.Flags().StringVar(&kubeconfigPath, "kubeconfig", kubeconfigPath, "Path to kubeconfig file with authorization information (the master location is set by the master flag).")
119+
cmd.Flags().StringVar(&namespace, "namespace", "default", "Namespace of Backup/Restore Session")
120+
cmd.Flags().StringVar(&appBindingName, "app-binding", appBindingName, "Name of the app binding")
121+
122+
cmd.Flags().StringVar(&setupOpt.Provider, "provider", setupOpt.Provider, "Backend provider (i.e. gcs, s3, azure etc)")
123+
cmd.Flags().StringVar(&setupOpt.Bucket, "bucket", setupOpt.Bucket, "Name of the cloud bucket/container (keep empty for local backend)")
124+
cmd.Flags().StringVar(&setupOpt.Endpoint, "endpoint", setupOpt.Endpoint, "Endpoint for s3/s3 compatible backend")
125+
cmd.Flags().StringVar(&setupOpt.Path, "path", setupOpt.Path, "Directory inside the bucket where backup will be stored")
126+
cmd.Flags().StringVar(&setupOpt.SecretDir, "secret-dir", setupOpt.SecretDir, "Directory where storage secret has been mounted")
127+
cmd.Flags().StringVar(&setupOpt.ScratchDir, "scratch-dir", setupOpt.ScratchDir, "Temporary directory")
128+
cmd.Flags().BoolVar(&setupOpt.EnableCache, "enable-cache", setupOpt.EnableCache, "Specify weather to enable caching for restic")
129+
130+
cmd.Flags().StringVar(&dumpOpt.Host, "hostname", dumpOpt.Host, "Name of the host machine")
131+
// TODO: sliceVar
132+
cmd.Flags().StringVar(&dumpOpt.Snapshot, "snapshot", dumpOpt.Snapshot, "Snapshot to dump")
133+
134+
cmd.Flags().StringVar(&outputDir, "output-dir", outputDir, "Directory where output.json file will be written (keep empty if you don't need to write output in file)")
135+
136+
cmd.Flags().BoolVar(&metrics.Enabled, "metrics-enabled", metrics.Enabled, "Specify weather to export Prometheus metrics")
137+
cmd.Flags().StringVar(&metrics.PushgatewayURL, "metrics-pushgateway-url", metrics.PushgatewayURL, "Pushgateway URL where the metrics will be pushed")
138+
cmd.Flags().StringVar(&metrics.MetricFileDir, "metrics-dir", metrics.MetricFileDir, "Directory where to write metric.prom file (keep empty if you don't want to write metric in a text file)")
139+
cmd.Flags().StringSliceVar(&metrics.Labels, "metrics-labels", metrics.Labels, "Labels to apply in exported metrics")
140+
141+
return cmd
142+
}

‎root.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,9 @@ func NewRootCmd() *cobra.Command {
6262
rootCmd.AddCommand(NewCmdBackupMySql())
6363
rootCmd.AddCommand(NewCmdRestoreMySql())
6464

65+
rootCmd.AddCommand(NewCmdBackupMongo())
66+
rootCmd.AddCommand(NewCmdRestoreMongo())
67+
6568
rootCmd.AddCommand(NewCmdUpdateStatus())
6669

6770
return rootCmd

0 commit comments

Comments
 (0)