Use auth-file for authentication (#615) (#665)

/cherry-pick

Signed-off-by: Emruz Hossain <emruz@appscode.com>

Tasks:
- [x] Update `multielasticdump` to `6.65.3`  (this fixes backup failure when template is empty)
- [x] Use `--httpAuthFile` for authentication
- [x] Show output of` multielasticdump` in the log

Author: 1gtm

.github/workflows/ci.yml

@@ -78,6 +78,7 @@ jobs:
           helm repo add appscode https://charts.appscode.com/stable/
           helm repo update
           helm install stash-crds appscode/stash-crds
+          helm install kubedb-crds appscode/kubedb-crds
           helm install kmodules-crds appscode/kmodules-crds
           kubectl wait --for=condition=NamesAccepted crds --all --timeout=5m
 

Dockerfile.dbg

@@ -36,7 +36,7 @@ RUN set -x \
   && apk add --update --no-cache bash ca-certificates curl
 
 RUN npm config set unsafe-perm true \
-  && npm install elasticdump@6.62.1 -g
+  && npm install elasticdump@6.65.3 -g
 
 COPY --from=0 restic /bin/restic
 COPY bin/{ARG_OS}_{ARG_ARCH}/{ARG_BIN} /{ARG_BIN}

Dockerfile.in

@@ -36,7 +36,7 @@ RUN set -x \
   && apk add --update --no-cache bash ca-certificates curl
 
 RUN npm config set unsafe-perm true \
-  && npm install elasticdump@6.62.1 -g
+  && npm install elasticdump@6.65.3 -g
 
 COPY --from=0 /restic /bin/restic
 COPY bin/{ARG_OS}_{ARG_ARCH}/{ARG_BIN} /{ARG_BIN}

docs/elasticsearch.md

@@ -56,7 +56,7 @@ Let's deploy a sample Elasticsearch database and insert some data into it.
 Below is the YAML of a sample Elasticsearch crd that we are going to create for this tutorial:
 
 ```yaml
-apiVersion: kubedb.com/v1alpha1
+apiVersion: kubedb.com/v1alpha2
 kind: Elasticsearch
 metadata:
   name: sample-elasticsearch
@@ -131,10 +131,7 @@ metadata:
     app.kubernetes.io/component: database
     app.kubernetes.io/instance: sample-elasticsearch
     app.kubernetes.io/managed-by: kubedb.com
-    app.kubernetes.io/name: elasticsearch
-    app.kubernetes.io/version: 7.3.2
-    kubedb.com/kind: Elasticsearch
-    kubedb.com/name: sample-elasticsearch
+    app.kubernetes.io/name: elasticsearches.kubedb.com
   name: sample-elasticsearch
   namespace: demo
 spec:
@@ -421,16 +418,16 @@ Now, we have to deploy the restored database similarly as we have deployed the o
 Below is the YAML for `Elasticsearch` crd we are going deploy to initialize from backup,
 
 ```yaml
-apiVersion: kubedb.com/v1alpha1
+apiVersion: kubedb.com/v1alpha2
 kind: Elasticsearch
 metadata:
   name: restored-elasticsearch
   namespace: demo
 spec:
   version: "7.3.2"
   storageType: Durable
-  databaseSecret:
-    secretName: sample-elasticsearch-auth # use same secret as original the database
+  authSecret:
+    name: sample-elasticsearch-auth # use same secret as original the database
   storage:
     storageClassName: "standard"
     accessModes:
@@ -439,14 +436,13 @@ spec:
       requests:
         storage: 1Gi
   init:
-    stashRestoreSession:
-      name: sample-elasticsearch-restore
+    waitForInitialRestore: true
   terminationPolicy: Delete
 ```
 
 Here,
 
-- `spec.init.stashRestoreSession.name` specifies the `RestoreSession` crd name that we are going to use to restore this database.
+- `spec.init.waitForInitialRestore` tells KubeDB to wait for the first restore to complete before marking this database as ready to use.
 
 Let's create the above database,
 
@@ -486,7 +482,7 @@ metadata:
   name: sample-elasticsearch-restore
   namespace: demo
   labels:
-    kubedb.com/kind: Elasticsearch # this label is mandatory if you are using KubeDB to deploy the database. Otherwise, Elasticsearch crd will be stuck in `Initializing` phase.
+    app.kubernetes.io/name: elasticsearches.kubedb.com # this label is mandatory if you are using KubeDB to deploy the database. Otherwise, Elasticsearch crd will be stuck in `Provisioning` phase.
 spec:
   task:
     name: elasticsearch-restore-{{< param "info.subproject_version" >}}
@@ -512,14 +508,14 @@ spec:
 
 Here,
 
-- `metadata.labels` specifies a `kubedb.com/kind: Elasticsearch` label that is used by KubeDB to watch this `RestoreSession`.
+- `metadata.labels` specifies a `app.kubernetes.io/name: elasticsearches.kubedb.com` label that is used by KubeDB to watch this `RestoreSession`.
 - `spec.task.name` specifies the name of the `Task` crd that specifies the Functions and their execution order to restore an Elasticsearch database.
 - `spec.repository.name` specifies the `Repository` crd that holds the backend information where our backed up data has been stored.
 - `spec.target.ref` refers to the AppBinding crd for the `restored-elasticsearch` database.
 - `spec.interimVolumeTemplate` specifies a PVC template to store the restored data temporarily before inserting into the targeted Elasticsearch database.
 - `spec.rules` specifies that we are restoring from the latest backup snapshot of the database.
 
-> **Warning:** Label `kubedb.com/kind: Elasticsearch` is mandatory if you are using KubeDB to deploy the database. Otherwise, the database will be stuck in `Initializing` state.
+> **Warning:** Label `app.kubernetes.io/name: elasticsearches.kubedb.com` is mandatory if you are using KubeDB to deploy the database. Otherwise, the database will be stuck in `Provisioning` state.
 
 Let's create the `RestoreSession` crd we have shown above,
 

docs/examples/backup/appbinding.yaml

@@ -5,10 +5,7 @@ metadata:
     app.kubernetes.io/component: database
     app.kubernetes.io/instance: sample-elasticsearch
     app.kubernetes.io/managed-by: kubedb.com
-    app.kubernetes.io/name: elasticsearch
-    app.kubernetes.io/version: 7.3.2
-    kubedb.com/kind: Elasticsearch
-    kubedb.com/name: sample-elasticsearch
+    app.kubernetes.io/name: elasticsearches.kubedb.com
   name: sample-elasticsearch
   namespace: demo
 spec:

docs/examples/backup/elasticsearch.yaml

@@ -1,4 +1,4 @@
-apiVersion: kubedb.com/v1alpha1
+apiVersion: kubedb.com/v1alpha2
 kind: Elasticsearch
 metadata:
   name: sample-elasticsearch

docs/examples/restore/restored-elasticsearch.yaml

@@ -1,13 +1,13 @@
-apiVersion: kubedb.com/v1alpha1
+apiVersion: kubedb.com/v1alpha2
 kind: Elasticsearch
 metadata:
   name: restored-elasticsearch
   namespace: demo
 spec:
   version: "7.3.2"
   storageType: Durable
-  databaseSecret:
-    secretName: sample-elasticsearch-auth # use same secret as original the database
+  authSecret:
+    name: sample-elasticsearch-auth # use same secret as original the database
   storage:
     storageClassName: "standard"
     accessModes:
@@ -16,6 +16,5 @@ spec:
       requests:
         storage: 1Gi
   init:
-    stashRestoreSession:
-      name: sample-elasticsearch-restore
+    waitForInitialRestore: true
   terminationPolicy: Delete

docs/examples/restore/restoresession.yaml

@@ -4,7 +4,7 @@ metadata:
   name: sample-elasticsearch-restore
   namespace: demo
   labels:
-    kubedb.com/kind: Elasticsearch # this label is mandatory if you are using KubeDB to deploy the database. Otherwise, Elasticsearch crd will be stuck in `Initializing` phase.
+    app.kubernetes.io/name: elasticsearches.kubedb.com # this label is mandatory if you are using KubeDB to deploy the database. Otherwise, Elasticsearch crd will be stuck in `Provisioning` phase.
 spec:
   task:
     name: elasticsearch-restore-7.3.2-v6

pkg/backup.go

@@ -35,11 +35,9 @@ import (
 	license "go.bytebuilders.dev/license-verifier/kubernetes"
 	"gomodules.xyz/x/flags"
 	"gomodules.xyz/x/log"
-	core "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
-	meta_util "kmodules.xyz/client-go/meta"
 	appcatalog "kmodules.xyz/custom-resources/apis/appcatalog/v1alpha1"
 	appcatalog_cs "kmodules.xyz/custom-resources/client/clientset/versioned"
 	v1 "kmodules.xyz/offshoot-api/api/v1"
@@ -208,6 +206,14 @@ func (opt *esOptions) backupElasticsearch(targetRef api_v1beta1.TargetRef) (*res
 		return nil, err
 	}
 
+	// write the credential ifo into a file
+	// TODO: support backup without authentication
+	httpAuthFilePath := filepath.Join(opt.setupOptions.ScratchDir, ESAuthFile)
+	err = writeAuthFile(httpAuthFilePath, appBindingSecret)
+	if err != nil {
+		return nil, err
+	}
+
 	var tlsArgs string
 	if appBinding.Spec.ClientConfig.CABundle != nil {
 		if err := ioutil.WriteFile(filepath.Join(opt.setupOptions.ScratchDir, ESCACertFile), appBinding.Spec.ClientConfig.CABundle, os.ModePerm); err != nil {
@@ -217,13 +223,7 @@ func (opt *esOptions) backupElasticsearch(targetRef api_v1beta1.TargetRef) (*res
 	}
 
 	appSVC := appBinding.Spec.ClientConfig.Service
-	esURL := fmt.Sprintf("%v://%s:%s@%s:%d",
-		appSVC.Scheme,
-		must(meta_util.GetBytesForKeys(appBindingSecret.Data, core.BasicAuthUsernameKey, ESUser)),
-		must(meta_util.GetBytesForKeys(appBindingSecret.Data, core.BasicAuthPasswordKey, ESPassword)),
-		appSVC.Name,
-		appSVC.Port,
-	) // TODO: support backup without authentication
+	esURL := fmt.Sprintf("%v://%s:%d", appSVC.Scheme, appSVC.Name, appSVC.Port)
 
 	// wait for DB ready
 	waitForDBReady(appBinding.Spec.ClientConfig.Service.Name, appBinding.Spec.ClientConfig.Service.Port, opt.waitTimeout)
@@ -232,12 +232,12 @@ func (opt *esOptions) backupElasticsearch(targetRef api_v1beta1.TargetRef) (*res
 	log.Infoln("Performing multielasticdump on ", appSVC.Name)
 	esShell := sh.NewSession()
 	esShell.ShowCMD = false
-	esShell.Stdout = ioutil.Discard
 	esShell.SetEnv("NODE_TLS_REJECT_UNAUTHORIZED", "0") //xref: https://github.com/taskrabbit/elasticsearch-dump#bypassing-self-sign-certificate-errors
 
 	args := []interface{}{
 		fmt.Sprintf(`--input=%v`, esURL),
 		fmt.Sprintf(`--output=%v`, opt.interimDataDir),
+		fmt.Sprintf("--httpAuthFile=%s", httpAuthFilePath),
 		tlsArgs,
 	}
 	for _, arg := range strings.Fields(opt.esArgs) {

pkg/restore.go

@@ -32,11 +32,9 @@ import (
 	license "go.bytebuilders.dev/license-verifier/kubernetes"
 	"gomodules.xyz/x/flags"
 	"gomodules.xyz/x/log"
-	core "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
-	meta_util "kmodules.xyz/client-go/meta"
 	appcatalog "kmodules.xyz/custom-resources/apis/appcatalog/v1alpha1"
 	appcatalog_cs "kmodules.xyz/custom-resources/client/clientset/versioned"
 	v1 "kmodules.xyz/offshoot-api/api/v1"
@@ -175,6 +173,14 @@ func (opt *esOptions) restoreElasticsearch(targetRef api_v1beta1.TargetRef) (*re
 		return nil, err
 	}
 
+	// write the credential ifo into a file
+	// TODO: support backup without authentication
+	httpAuthFilePath := filepath.Join(opt.setupOptions.ScratchDir, ESAuthFile)
+	err = writeAuthFile(httpAuthFilePath, appBindingSecret)
+	if err != nil {
+		return nil, err
+	}
+
 	var tlsArgs string
 	if appBinding.Spec.ClientConfig.CABundle != nil {
 		if err := ioutil.WriteFile(filepath.Join(opt.setupOptions.ScratchDir, ESCACertFile), appBinding.Spec.ClientConfig.CABundle, os.ModePerm); err != nil {
@@ -184,13 +190,7 @@ func (opt *esOptions) restoreElasticsearch(targetRef api_v1beta1.TargetRef) (*re
 	}
 
 	appSVC := appBinding.Spec.ClientConfig.Service
-	esURL := fmt.Sprintf("%v://%s:%s@%s:%d",
-		appSVC.Scheme,
-		must(meta_util.GetBytesForKeys(appBindingSecret.Data, core.BasicAuthUsernameKey, ESUser)),
-		must(meta_util.GetBytesForKeys(appBindingSecret.Data, core.BasicAuthPasswordKey, ESPassword)),
-		appSVC.Name,
-		appSVC.Port,
-	) // TODO: support backup without authentication
+	esURL := fmt.Sprintf("%v://%s:%d", appSVC.Scheme, appSVC.Name, appSVC.Port)
 
 	// wait for DB ready
 	waitForDBReady(appBinding.Spec.ClientConfig.Service.Name, appBinding.Spec.ClientConfig.Service.Port, opt.waitTimeout)
@@ -214,13 +214,13 @@ func (opt *esOptions) restoreElasticsearch(targetRef api_v1beta1.TargetRef) (*re
 	log.Infoln("Performing multielasticdump on ", appSVC.Name)
 	esShell := sh.NewSession()
 	esShell.ShowCMD = false
-	esShell.Stdout = ioutil.Discard
 	esShell.SetEnv("NODE_TLS_REJECT_UNAUTHORIZED", "0") //xref: https://github.com/taskrabbit/elasticsearch-dump#bypassing-self-sign-certificate-errors
 
 	args := []interface{}{
 		"--direction=load",
 		fmt.Sprintf(`--input=%v`, opt.interimDataDir),
 		fmt.Sprintf(`--output=%v`, esURL),
+		fmt.Sprintf("--httpAuthFile=%s", httpAuthFilePath),
 		tlsArgs,
 	}
 	for _, arg := range strings.Fields(opt.esArgs) {