[cherry-pick] Support multiple commands in backup/restore pipeline (#335) (#341)

/cherry-pick
Signed-off-by: Emruz Hossain <emruz@appscode.com>

Co-authored-by: Emruz Hossain <emruz@appscode.com>

Author: 1gtm

.github/workflows/ci.yml

@@ -78,6 +78,7 @@ jobs:
           helm repo add appscode https://charts.appscode.com/stable/
           helm repo update
           helm install stash-crds appscode/stash-crds
+          helm install kubedb-crds appscode/kubedb-crds
           helm install kmodules-crds appscode/kmodules-crds
           kubectl wait --for=condition=NamesAccepted crds --all --timeout=5m
 

docs/examples/restore/restored-mysql.yaml

@@ -15,4 +15,6 @@ spec:
     resources:
       requests:
         storage: 50Mi
+  init:
+    waitForInitialRestore: true
   terminationPolicy: WipeOut

docs/examples/restore/restoresession.yaml

@@ -4,7 +4,7 @@ metadata:
   name: restore-sample-mysql
   namespace: demo
   labels:
-    kubedb.com/kind: MySQL # this label is mandatory if you are using KubeDB to deploy the database.
+    app.kubernetes.io/name: mysqls.kubedb.com # this label is mandatory if you are using KubeDB to deploy the database.
 spec:
   task:
     name: mysql-restore-8.0.14-v6

docs/mysql.md

@@ -130,10 +130,7 @@ metadata:
     app.kubernetes.io/component: database
     app.kubernetes.io/instance: sample-mysql
     app.kubernetes.io/managed-by: kubedb.com
-    app.kubernetes.io/name: mysql
-    app.kubernetes.io/version: 8.0.14
-    kubedb.com/kind: MySQL
-    kubedb.com/name: sample-mysql
+    app.kubernetes.io/name: mysqls.kubedb.com
   name: sample-mysql
   namespace: demo
 spec:
@@ -456,12 +453,14 @@ spec:
     resources:
       requests:
         storage: 50Mi
+  init:
+    waitForInitialRestore: true
   terminationPolicy: WipeOut
 ```
 
 Here,
 
-- `spec.init.stashRestoreSession.name` specifies the `RestoreSession` CRD name that we will use later to restore the database.
+- `spec.init.waitForInitialRestore` tells KubeDB to wait for the first restore to complete before marking this database as ready to use.
 
 Let's create the above database,
 
@@ -501,7 +500,7 @@ metadata:
   name: sample-mysql-restore
   namespace: demo
   labels:
-    kubedb.com/kind: MySQL # this label is mandatory if you are using KubeDB to deploy the database.
+    app.kubernetes.io/name: mysqls.kubedb.com # this label is mandatory if you are using KubeDB to deploy the database.
 spec:
   task:
     name: mysql-restore-{{< param "info.subproject_version" >}}
@@ -518,13 +517,13 @@ spec:
 
 Here,
 
-- `.metadata.labels` specifies a `kubedb.com/kind: MySQL` label that is used by KubeDB to watch this RestoreSession object.
+- `.metadata.labels` specifies a `app.kubernetes.io/name: mysqls.kubedb.com` label that is used by KubeDB to watch this RestoreSession object.
 - `.spec.task.name` specifies the name of the Task CRD that specifies the necessary Functions and their execution order to restore a MySQL database.
 - `.spec.repository.name` specifies the Repository CRD that holds the backend information where our backed up data has been stored.
 - `.spec.target.ref` refers to the newly created AppBinding object for the `restored-mysql` MySQL object.
 - `.spec.rules` specifies that we are restoring data from the latest backup snapshot of the database.
 
-> **Warning:** Label `kubedb.com/kind: MySQL` is mandatory if you are using KubeDB to deploy the database. Otherwise, the database will be stuck in **`Initializing`** state.
+> **Warning:** Label `app.kubernetes.io/name: mysqls.kubedb.com` is mandatory if you are using KubeDB to deploy the database. Otherwise, the database will be stuck in **`Provisioning`** state.
 
 Let's create the RestoreSession CRD object we have shown above,
 

go.mod

@@ -11,11 +11,11 @@ require (
 	k8s.io/api v0.18.9
 	k8s.io/apimachinery v0.18.9
 	k8s.io/client-go v0.18.9
-	kmodules.xyz/client-go v0.0.0-20210118094617-273ba20ad7ca
-	kmodules.xyz/custom-resources v0.0.0-20201124062543-bd8d35c21b0c
-	kmodules.xyz/offshoot-api v0.0.0-20201105074700-8675f5f686f2
+	kmodules.xyz/client-go v0.0.0-20210218100652-8f345ddfe801
+	kmodules.xyz/custom-resources v0.0.0-20210222100152-7426e3678989
+	kmodules.xyz/offshoot-api v0.0.0-20210218144050-e1d794cb382a
 	kmodules.xyz/schema-checker v0.1.0
-	stash.appscode.dev/apimachinery v0.11.9
+	stash.appscode.dev/apimachinery v0.11.10-0.20210223054644-c403645d9bc1
 )
 
 replace bitbucket.org/ww/goautoneg => gomodules.xyz/goautoneg v0.0.0-20120707110453-a547fc61f48d

go.sum

@@ -211,7 +211,7 @@ func (opt *mysqlOptions) backupMySQL(targetRef api_v1beta1.TargetRef) (*restic.B
 	// set env for mysqldump
 	resticWrapper.SetEnv(EnvMySqlPassword, string(appBindingSecret.Data[MySqlPassword]))
 	// setup pipe command
-	opt.backupOptions.StdinPipeCommand = restic.Command{
+	backupCmd := restic.Command{
 		Name: MySqlDumpCMD,
 		Args: []interface{}{
 			"-u", string(appBindingSecret.Data[MySqlUser]),
@@ -220,10 +220,10 @@ func (opt *mysqlOptions) backupMySQL(targetRef api_v1beta1.TargetRef) (*restic.B
 	}
 	// if port is specified, append port in the arguments
 	if appBinding.Spec.ClientConfig.Service.Port != 0 {
-		opt.backupOptions.StdinPipeCommand.Args = append(opt.backupOptions.StdinPipeCommand.Args, fmt.Sprintf("--port=%d", appBinding.Spec.ClientConfig.Service.Port))
+		backupCmd.Args = append(backupCmd.Args, fmt.Sprintf("--port=%d", appBinding.Spec.ClientConfig.Service.Port))
 	}
 	for _, arg := range strings.Fields(opt.myArgs) {
-		opt.backupOptions.StdinPipeCommand.Args = append(opt.backupOptions.StdinPipeCommand.Args, arg)
+		backupCmd.Args = append(backupCmd.Args, arg)
 	}
 
 	// if ssl enabled, add ca.crt in the arguments
@@ -235,7 +235,7 @@ func (opt *mysqlOptions) backupMySQL(targetRef api_v1beta1.TargetRef) (*restic.B
 			fmt.Sprintf("--ssl-ca=%v", filepath.Join(opt.setupOptions.ScratchDir, MySQLTLSRootCA)),
 		}
 
-		opt.backupOptions.StdinPipeCommand.Args = append(opt.backupOptions.StdinPipeCommand.Args, tlsCreds...)
+		backupCmd.Args = append(backupCmd.Args, tlsCreds...)
 	}
 
 	// wait for DB ready
@@ -244,6 +244,9 @@ func (opt *mysqlOptions) backupMySQL(targetRef api_v1beta1.TargetRef) (*restic.B
 		return nil, err
 	}
 
+	// add backup command in the pipeline
+	opt.backupOptions.StdinPipeCommands = append(opt.backupOptions.StdinPipeCommands, backupCmd)
+
 	// Run backup
 	return resticWrapper.RunBackup(opt.backupOptions, targetRef)
 }

pkg/backup.go

@@ -177,7 +177,7 @@ func (opt *mysqlOptions) restoreMySQL(targetRef api_v1beta1.TargetRef) (*restic.
 	// set env for mysql
 	resticWrapper.SetEnv(EnvMySqlPassword, string(appBindingSecret.Data[MySqlPassword]))
 	// setup pipe command
-	opt.dumpOptions.StdoutPipeCommand = restic.Command{
+	restoreCmd := restic.Command{
 		Name: MySqlRestoreCMD,
 		Args: []interface{}{
 			"-u", string(appBindingSecret.Data[MySqlUser]),
@@ -186,10 +186,10 @@ func (opt *mysqlOptions) restoreMySQL(targetRef api_v1beta1.TargetRef) (*restic.
 	}
 	// if port is specified, append port in the arguments
 	if appBinding.Spec.ClientConfig.Service.Port != 0 {
-		opt.dumpOptions.StdoutPipeCommand.Args = append(opt.dumpOptions.StdoutPipeCommand.Args, fmt.Sprintf("--port=%d", appBinding.Spec.ClientConfig.Service.Port))
+		restoreCmd.Args = append(restoreCmd.Args, fmt.Sprintf("--port=%d", appBinding.Spec.ClientConfig.Service.Port))
 	}
 	for _, arg := range strings.Fields(opt.myArgs) {
-		opt.dumpOptions.StdoutPipeCommand.Args = append(opt.dumpOptions.StdoutPipeCommand.Args, arg)
+		restoreCmd.Args = append(restoreCmd.Args, arg)
 	}
 
 	// if ssl enabled, add ca.crt in the arguments
@@ -201,7 +201,7 @@ func (opt *mysqlOptions) restoreMySQL(targetRef api_v1beta1.TargetRef) (*restic.
 			fmt.Sprintf("--ssl-ca=%v", filepath.Join(opt.setupOptions.ScratchDir, MySQLTLSRootCA)),
 		}
 
-		opt.dumpOptions.StdoutPipeCommand.Args = append(opt.dumpOptions.StdoutPipeCommand.Args, tlsCreds...)
+		restoreCmd.Args = append(restoreCmd.Args, tlsCreds...)
 	}
 
 	// wait for DB ready
@@ -210,6 +210,9 @@ func (opt *mysqlOptions) restoreMySQL(targetRef api_v1beta1.TargetRef) (*restic.
 		return nil, err
 	}
 
+	// append the restore command to the pipeline
+	opt.dumpOptions.StdoutPipeCommands = append(opt.dumpOptions.StdoutPipeCommands, restoreCmd)
+
 	// Run dump
 	return resticWrapper.Dump(opt.dumpOptions, targetRef)
 }