Fix waitForDBReady() logic + Make timeout configurable (#23)

Signed-off-by: hossainemruz <emruz@appscode.com>

Makefile

@@ -77,7 +77,7 @@ TAG              := $(VERSION)_$(OS)_$(ARCH)
 TAG_PROD         := $(TAG)
 TAG_DBG          := $(VERSION)-dbg_$(OS)_$(ARCH)
 
-GO_VERSION       ?= 1.14.2
+GO_VERSION       ?= 1.14
 BUILD_IMAGE      ?= appscode/golang-dev:$(GO_VERSION)
 CHART_TEST_IMAGE ?= quay.io/helmpack/chart-testing:v3.0.0-rc.1
 

charts/stash-mysql/README.md

@@ -53,6 +53,7 @@ The following table lists the configurable parameters of the `stash-mysql` chart
 | image.tag        | Tag of the image that is used to backup/restore MySQL database. This is usually same as the database version it can backup. | `"5.7.25"`          |
 | backup.args      | Arguments to pass to `mysqldump` command  during bakcup process                                                             | `"--all-databases"` |
 | restore.args     | Arguments to pass to `mysql` command during restore process                                                                 | `""`                |
+| waitTimeout      | Time limit to wait for the database to be ready before backup/restore process.                                              | `300`               |
 
 
 Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example:

charts/stash-mysql/templates/mysql-backup-function.yaml

@@ -20,6 +20,7 @@ spec:
   - --max-connections=${MAX_CONNECTIONS:=0} # 0 indicates use default connection limit
   - --hostname=${HOSTNAME:=}
   - --mysql-args=${args:={{ .Values.backup.args }}} # optional arguments pass to mysqldump command
+  - --wait-timeout=${waitTimeout:={{ .Values.waitTimeout}}}
   # target information
   - --namespace=${NAMESPACE:=default}
   - --appbinding=${TARGET_NAME:=}

charts/stash-mysql/templates/mysql-restore-function.yaml

@@ -21,6 +21,7 @@ spec:
   - --hostname=${HOSTNAME:=}
   - --source-hostname=${SOURCE_HOSTNAME:=}
   - --mysql-args=${args:={{ .Values.restore.args }}} # optional arguments pass to mysql command
+  - --wait-timeout=${waitTimeout:={{ .Values.waitTimeout}}}
   # target information
   - --namespace=${NAMESPACE:=default}
   - --appbinding=${TARGET_NAME:=}

charts/stash-mysql/values.yaml

@@ -23,3 +23,6 @@ backup:
 restore:
   # Arguments to pass to `mysql` command during restore process
   args: ""
+
+# Time limit to wait for the database to be ready before backup/restore process.
+waitTimeout: 300

go.mod

@@ -4,12 +4,14 @@ go 1.12
 
 require (
 	github.com/appscode/go v0.0.0-20200323182826-54e98e09185a
+	github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27
 	github.com/go-openapi/spec v0.19.3
 	github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/google/gofuzz v1.0.0
 	github.com/spf13/cobra v0.0.5
 	github.com/yudai/gojsondiff v1.0.0
+	k8s.io/api v0.0.0-20191122220107-b5267f2975e0
 	k8s.io/apiextensions-apiserver v0.0.0-20191114105449-027877536833
 	k8s.io/apimachinery v0.16.5-beta.1
 	k8s.io/client-go v12.0.0+incompatible

pkg/backup.go

@@ -38,7 +38,8 @@ func NewCmdBackup() *cobra.Command {
 		kubeconfigPath string
 		opt            = mysqlOptions{
 
-			myArgs: "--all-databases",
+			myArgs:      "--all-databases",
+			waitTimeout: 300,
 			setupOptions: restic.SetupOptions{
 				ScratchDir:  restic.DefaultScratchDir,
 				EnableCache: false,
@@ -94,6 +95,7 @@ func NewCmdBackup() *cobra.Command {
 	}
 
 	cmd.Flags().StringVar(&opt.myArgs, "mysql-args", opt.myArgs, "Additional arguments")
+	cmd.Flags().Int32Var(&opt.waitTimeout, "wait-timeout", opt.waitTimeout, "Time limit to wait for the database to be ready")
 
 	cmd.Flags().StringVar(&masterURL, "master", masterURL, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
 	cmd.Flags().StringVar(&kubeconfigPath, "kubeconfig", kubeconfigPath, "Path to kubeconfig file with authorization information (the master location is set by the master flag).")
@@ -171,7 +173,10 @@ func (opt *mysqlOptions) backupMySQL() (*restic.BackupOutput, error) {
 	}
 
 	// wait for DB ready
-	waitForDBReady(appBinding.Spec.ClientConfig.Service.Name, appBinding.Spec.ClientConfig.Service.Port)
+	err = waitForDBReady(appBinding, appBindingSecret, opt.waitTimeout)
+	if err != nil {
+		return nil, err
+	}
 
 	// Run backup
 	return resticWrapper.RunBackup(opt.backupOptions)

pkg/restore.go

@@ -41,6 +41,7 @@ func NewCmdRestore() *cobra.Command {
 				ScratchDir:  restic.DefaultScratchDir,
 				EnableCache: false,
 			},
+			waitTimeout: 300,
 			dumpOptions: restic.DumpOptions{
 				Host:     restic.DefaultHost,
 				FileName: MySqlDumpFile,
@@ -92,6 +93,7 @@ func NewCmdRestore() *cobra.Command {
 	}
 
 	cmd.Flags().StringVar(&opt.myArgs, "mysql-args", opt.myArgs, "Additional arguments")
+	cmd.Flags().Int32Var(&opt.waitTimeout, "wait-timeout", opt.waitTimeout, "Time limit to wait for the database to be ready")
 
 	cmd.Flags().StringVar(&masterURL, "master", masterURL, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
 	cmd.Flags().StringVar(&kubeconfigPath, "kubeconfig", kubeconfigPath, "Path to kubeconfig file with authorization information (the master location is set by the master flag).")
@@ -162,7 +164,10 @@ func (opt *mysqlOptions) restoreMySQL() (*restic.RestoreOutput, error) {
 	}
 
 	// wait for DB ready
-	waitForDBReady(appBinding.Spec.ClientConfig.Service.Name, appBinding.Spec.ClientConfig.Service.Port)
+	err = waitForDBReady(appBinding, appBindingSecret, opt.waitTimeout)
+	if err != nil {
+		return nil, err
+	}
 
 	// Run dump
 	return resticWrapper.Dump(opt.dumpOptions)

pkg/utils.go

@@ -18,13 +18,14 @@ package pkg
 
 import (
 	"fmt"
-	"os/exec"
-	"time"
 
 	"stash.appscode.dev/apimachinery/pkg/restic"
 
 	"github.com/appscode/go/log"
+	"github.com/codeskyblue/go-sh"
+	core "k8s.io/api/core/v1"
 	"k8s.io/client-go/kubernetes"
+	"kmodules.xyz/custom-resources/apis/appcatalog/v1alpha1"
 	appcatalog_cs "kmodules.xyz/custom-resources/client/clientset/versioned"
 )
 
@@ -44,21 +45,23 @@ type mysqlOptions struct {
 	namespace      string
 	appBindingName string
 	myArgs         string
+	waitTimeout    int32
 	outputDir      string
 
 	setupOptions  restic.SetupOptions
 	backupOptions restic.BackupOptions
 	dumpOptions   restic.DumpOptions
 }
 
-func waitForDBReady(host string, port int32) {
-	log.Infoln("Checking database connection")
-	cmd := fmt.Sprintf(`nc "%s" "%d" -w 30`, host, port)
-	for {
-		if err := exec.Command(cmd).Run(); err != nil {
-			break
-		}
-		log.Infoln("Waiting... database is not ready yet")
-		time.Sleep(5 * time.Second)
+func waitForDBReady(appBinding *v1alpha1.AppBinding, secret *core.Secret, waitTimeout int32) error {
+	log.Infoln("Waiting for the database to be ready.....")
+	shell := sh.NewSession()
+	shell.SetEnv(EnvMySqlPassword, string(secret.Data[MySqlPassword]))
+	args := []interface{}{
+		"ping",
+		"--host", appBinding.Spec.ClientConfig.Service.Name,
+		"--user=root",
+		fmt.Sprintf("--wait=%d", waitTimeout),
 	}
+	return shell.Command("mysqladmin", args...).Run()
 }