Skip to content

Commit f49353f

Browse files
1gtm1gtm
authored
Use auth-file for authentication (#615) (#658)
/cherry-pick Signed-off-by: Emruz Hossain <emruz@appscode.com> Tasks: - [x] Update `multielasticdump` to `6.65.3` (this fixes backup failure when template is empty) - [x] Use `--httpAuthFile` for authentication - [x] Show output of` multielasticdump` in the log
1 parent eb8e7ac commit f49353f

File tree

11 files changed

+52
-48
lines changed

11 files changed

+52
-48
lines changed

‎.github/workflows/ci.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ jobs:
7878
helm repo add appscode https://charts.appscode.com/stable/
7979
helm repo update
8080
helm install stash-crds appscode/stash-crds
81+
helm install kubedb-crds appscode/kubedb-crds
8182
helm install kmodules-crds appscode/kmodules-crds
8283
kubectl wait --for=condition=NamesAccepted crds --all --timeout=5m
8384

‎Dockerfile.dbg

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ RUN set -x \
3636
&& apk add --update --no-cache bash ca-certificates curl
3737

3838
RUN npm config set unsafe-perm true \
39-
&& npm install elasticdump@6.62.1 -g
39+
&& npm install elasticdump@6.65.3 -g
4040

4141
COPY --from=0 restic /bin/restic
4242
COPY bin/{ARG_OS}_{ARG_ARCH}/{ARG_BIN} /{ARG_BIN}

‎Dockerfile.in

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ RUN set -x \
3636
&& apk add --update --no-cache bash ca-certificates curl
3737

3838
RUN npm config set unsafe-perm true \
39-
&& npm install elasticdump@6.62.1 -g
39+
&& npm install elasticdump@6.65.3 -g
4040

4141
COPY --from=0 /restic /bin/restic
4242
COPY bin/{ARG_OS}_{ARG_ARCH}/{ARG_BIN} /{ARG_BIN}

‎docs/elasticsearch.md

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ Let's deploy a sample Elasticsearch database and insert some data into it.
5656
Below is the YAML of a sample Elasticsearch crd that we are going to create for this tutorial:
5757

5858
```yaml
59-
apiVersion: kubedb.com/v1alpha1
59+
apiVersion: kubedb.com/v1alpha2
6060
kind: Elasticsearch
6161
metadata:
6262
name: sample-elasticsearch
@@ -131,10 +131,7 @@ metadata:
131131
app.kubernetes.io/component: database
132132
app.kubernetes.io/instance: sample-elasticsearch
133133
app.kubernetes.io/managed-by: kubedb.com
134-
app.kubernetes.io/name: elasticsearch
135-
app.kubernetes.io/version: 5.6.4-v1
136-
kubedb.com/kind: Elasticsearch
137-
kubedb.com/name: sample-elasticsearch
134+
app.kubernetes.io/name: elasticsearches.kubedb.com
138135
name: sample-elasticsearch
139136
namespace: demo
140137
spec:
@@ -421,16 +418,16 @@ Now, we have to deploy the restored database similarly as we have deployed the o
421418
Below is the YAML for `Elasticsearch` crd we are going deploy to initialize from backup,
422419

423420
```yaml
424-
apiVersion: kubedb.com/v1alpha1
421+
apiVersion: kubedb.com/v1alpha2
425422
kind: Elasticsearch
426423
metadata:
427424
name: restored-elasticsearch
428425
namespace: demo
429426
spec:
430427
version: "5.6.4-v1"
431428
storageType: Durable
432-
databaseSecret:
433-
secretName: sample-elasticsearch-auth # use same secret as original the database
429+
authSecret:
430+
name: sample-elasticsearch-auth # use same secret as original the database
434431
storage:
435432
storageClassName: "standard"
436433
accessModes:
@@ -439,14 +436,13 @@ spec:
439436
requests:
440437
storage: 1Gi
441438
init:
442-
stashRestoreSession:
443-
name: sample-elasticsearch-restore
439+
waitForInitialRestore: true
444440
terminationPolicy: Delete
445441
```
446442

447443
Here,
448444

449-
- `spec.init.stashRestoreSession.name` specifies the `RestoreSession` crd name that we are going to use to restore this database.
445+
- `spec.init.waitForInitialRestore` tells KubeDB to wait for the first restore to complete before marking this database as ready to use.
450446

451447
Let's create the above database,
452448

@@ -486,7 +482,7 @@ metadata:
486482
name: sample-elasticsearch-restore
487483
namespace: demo
488484
labels:
489-
kubedb.com/kind: Elasticsearch # this label is mandatory if you are using KubeDB to deploy the database. Otherwise, Elasticsearch crd will be stuck in `Initializing` phase.
485+
app.kubernetes.io/name: elasticsearches.kubedb.com # this label is mandatory if you are using KubeDB to deploy the database. Otherwise, Elasticsearch crd will be stuck in `Provisioning` phase.
490486
spec:
491487
task:
492488
name: elasticsearch-restore-{{< param "info.subproject_version" >}}
@@ -512,14 +508,14 @@ spec:
512508
513509
Here,
514510
515-
- `metadata.labels` specifies a `kubedb.com/kind: Elasticsearch` label that is used by KubeDB to watch this `RestoreSession`.
511+
- `metadata.labels` specifies a `app.kubernetes.io/name: elasticsearches.kubedb.com` label that is used by KubeDB to watch this `RestoreSession`.
516512
- `spec.task.name` specifies the name of the `Task` crd that specifies the Functions and their execution order to restore an Elasticsearch database.
517513
- `spec.repository.name` specifies the `Repository` crd that holds the backend information where our backed up data has been stored.
518514
- `spec.target.ref` refers to the AppBinding crd for the `restored-elasticsearch` database.
519515
- `spec.interimVolumeTemplate` specifies a PVC template to store the restored data temporarily before inserting into the targeted Elasticsearch database.
520516
- `spec.rules` specifies that we are restoring from the latest backup snapshot of the database.
521517

522-
> **Warning:** Label `kubedb.com/kind: Elasticsearch` is mandatory if you are using KubeDB to deploy the database. Otherwise, the database will be stuck in `Initializing` state.
518+
> **Warning:** Label `app.kubernetes.io/name: elasticsearches.kubedb.com` is mandatory if you are using KubeDB to deploy the database. Otherwise, the database will be stuck in `Provisioning` state.
523519

524520
Let's create the `RestoreSession` crd we have shown above,
525521

‎docs/examples/backup/appbinding.yaml

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,7 @@ metadata:
55
app.kubernetes.io/component: database
66
app.kubernetes.io/instance: sample-elasticsearch
77
app.kubernetes.io/managed-by: kubedb.com
8-
app.kubernetes.io/name: elasticsearch
9-
app.kubernetes.io/version: 5.6.4-v1
10-
kubedb.com/kind: Elasticsearch
11-
kubedb.com/name: sample-elasticsearch
8+
app.kubernetes.io/name: elasticsearches.kubedb.com
129
name: sample-elasticsearch
1310
namespace: demo
1411
spec:

‎docs/examples/backup/elasticsearch.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
apiVersion: kubedb.com/v1alpha1
1+
apiVersion: kubedb.com/v1alpha2
22
kind: Elasticsearch
33
metadata:
44
name: sample-elasticsearch

‎docs/examples/restore/restored-elasticsearch.yaml

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
1-
apiVersion: kubedb.com/v1alpha1
1+
apiVersion: kubedb.com/v1alpha2
22
kind: Elasticsearch
33
metadata:
44
name: restored-elasticsearch
55
namespace: demo
66
spec:
77
version: "5.6.4-v1"
88
storageType: Durable
9-
databaseSecret:
10-
secretName: sample-elasticsearch-auth # use same secret as original the database
9+
authSecret:
10+
name: sample-elasticsearch-auth # use same secret as original the database
1111
storage:
1212
storageClassName: "standard"
1313
accessModes:
@@ -16,6 +16,5 @@ spec:
1616
requests:
1717
storage: 1Gi
1818
init:
19-
stashRestoreSession:
20-
name: sample-elasticsearch-restore
19+
waitForInitialRestore: true
2120
terminationPolicy: Delete

‎docs/examples/restore/restoresession.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ metadata:
44
name: sample-elasticsearch-restore
55
namespace: demo
66
labels:
7-
kubedb.com/kind: Elasticsearch # this label is mandatory if you are using KubeDB to deploy the database. Otherwise, Elasticsearch crd will be stuck in `Initializing` phase.
7+
app.kubernetes.io/name: elasticsearches.kubedb.com # this label is mandatory if you are using KubeDB to deploy the database. Otherwise, Elasticsearch crd will be stuck in `Provisioning` phase.
88
spec:
99
task:
1010
name: elasticsearch-restore-5.6.4-v6

‎pkg/backup.go

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,9 @@ import (
3535
license "go.bytebuilders.dev/license-verifier/kubernetes"
3636
"gomodules.xyz/x/flags"
3737
"gomodules.xyz/x/log"
38-
core "k8s.io/api/core/v1"
3938
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
4039
"k8s.io/client-go/kubernetes"
4140
"k8s.io/client-go/tools/clientcmd"
42-
meta_util "kmodules.xyz/client-go/meta"
4341
appcatalog "kmodules.xyz/custom-resources/apis/appcatalog/v1alpha1"
4442
appcatalog_cs "kmodules.xyz/custom-resources/client/clientset/versioned"
4543
v1 "kmodules.xyz/offshoot-api/api/v1"
@@ -208,6 +206,14 @@ func (opt *esOptions) backupElasticsearch(targetRef api_v1beta1.TargetRef) (*res
208206
return nil, err
209207
}
210208

209+
// write the credential ifo into a file
210+
// TODO: support backup without authentication
211+
httpAuthFilePath := filepath.Join(opt.setupOptions.ScratchDir, ESAuthFile)
212+
err = writeAuthFile(httpAuthFilePath, appBindingSecret)
213+
if err != nil {
214+
return nil, err
215+
}
216+
211217
var tlsArgs string
212218
if appBinding.Spec.ClientConfig.CABundle != nil {
213219
if err := ioutil.WriteFile(filepath.Join(opt.setupOptions.ScratchDir, ESCACertFile), appBinding.Spec.ClientConfig.CABundle, os.ModePerm); err != nil {
@@ -217,13 +223,7 @@ func (opt *esOptions) backupElasticsearch(targetRef api_v1beta1.TargetRef) (*res
217223
}
218224

219225
appSVC := appBinding.Spec.ClientConfig.Service
220-
esURL := fmt.Sprintf("%v://%s:%s@%s:%d",
221-
appSVC.Scheme,
222-
must(meta_util.GetBytesForKeys(appBindingSecret.Data, core.BasicAuthUsernameKey, ESUser)),
223-
must(meta_util.GetBytesForKeys(appBindingSecret.Data, core.BasicAuthPasswordKey, ESPassword)),
224-
appSVC.Name,
225-
appSVC.Port,
226-
) // TODO: support backup without authentication
226+
esURL := fmt.Sprintf("%v://%s:%d", appSVC.Scheme, appSVC.Name, appSVC.Port)
227227

228228
// wait for DB ready
229229
waitForDBReady(appBinding.Spec.ClientConfig.Service.Name, appBinding.Spec.ClientConfig.Service.Port, opt.waitTimeout)
@@ -232,13 +232,12 @@ func (opt *esOptions) backupElasticsearch(targetRef api_v1beta1.TargetRef) (*res
232232
log.Infoln("Performing multielasticdump on ", appSVC.Name)
233233
esShell := sh.NewSession()
234234
esShell.ShowCMD = false
235-
esShell.Stdout = ioutil.Discard
236235
esShell.SetEnv("NODE_TLS_REJECT_UNAUTHORIZED", "0") //xref: https://github.com/taskrabbit/elasticsearch-dump#bypassing-self-sign-certificate-errors
237236

238237
args := []interface{}{
239238
fmt.Sprintf(`--input=%v`, esURL),
240239
fmt.Sprintf(`--output=%v`, opt.interimDataDir),
241-
"--ignoreType=alias,settings,template", // ref: https://github.com/taskrabbit/elasticsearch-dump#multielasticdump
240+
fmt.Sprintf("--httpAuthFile=%s", httpAuthFilePath),
242241
tlsArgs,
243242
}
244243
for _, arg := range strings.Fields(opt.esArgs) {

‎pkg/restore.go

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,9 @@ import (
3232
license "go.bytebuilders.dev/license-verifier/kubernetes"
3333
"gomodules.xyz/x/flags"
3434
"gomodules.xyz/x/log"
35-
core "k8s.io/api/core/v1"
3635
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3736
"k8s.io/client-go/kubernetes"
3837
"k8s.io/client-go/tools/clientcmd"
39-
meta_util "kmodules.xyz/client-go/meta"
4038
appcatalog "kmodules.xyz/custom-resources/apis/appcatalog/v1alpha1"
4139
appcatalog_cs "kmodules.xyz/custom-resources/client/clientset/versioned"
4240
v1 "kmodules.xyz/offshoot-api/api/v1"
@@ -175,6 +173,14 @@ func (opt *esOptions) restoreElasticsearch(targetRef api_v1beta1.TargetRef) (*re
175173
return nil, err
176174
}
177175

176+
// write the credential ifo into a file
177+
// TODO: support backup without authentication
178+
httpAuthFilePath := filepath.Join(opt.setupOptions.ScratchDir, ESAuthFile)
179+
err = writeAuthFile(httpAuthFilePath, appBindingSecret)
180+
if err != nil {
181+
return nil, err
182+
}
183+
178184
var tlsArgs string
179185
if appBinding.Spec.ClientConfig.CABundle != nil {
180186
if err := ioutil.WriteFile(filepath.Join(opt.setupOptions.ScratchDir, ESCACertFile), appBinding.Spec.ClientConfig.CABundle, os.ModePerm); err != nil {
@@ -184,13 +190,7 @@ func (opt *esOptions) restoreElasticsearch(targetRef api_v1beta1.TargetRef) (*re
184190
}
185191

186192
appSVC := appBinding.Spec.ClientConfig.Service
187-
esURL := fmt.Sprintf("%v://%s:%s@%s:%d",
188-
appSVC.Scheme,
189-
must(meta_util.GetBytesForKeys(appBindingSecret.Data, core.BasicAuthUsernameKey, ESUser)),
190-
must(meta_util.GetBytesForKeys(appBindingSecret.Data, core.BasicAuthPasswordKey, ESPassword)),
191-
appSVC.Name,
192-
appSVC.Port,
193-
) // TODO: support backup without authentication
193+
esURL := fmt.Sprintf("%v://%s:%d", appSVC.Scheme, appSVC.Name, appSVC.Port)
194194

195195
// wait for DB ready
196196
waitForDBReady(appBinding.Spec.ClientConfig.Service.Name, appBinding.Spec.ClientConfig.Service.Port, opt.waitTimeout)
@@ -214,13 +214,13 @@ func (opt *esOptions) restoreElasticsearch(targetRef api_v1beta1.TargetRef) (*re
214214
log.Infoln("Performing multielasticdump on ", appSVC.Name)
215215
esShell := sh.NewSession()
216216
esShell.ShowCMD = false
217-
esShell.Stdout = ioutil.Discard
218217
esShell.SetEnv("NODE_TLS_REJECT_UNAUTHORIZED", "0") //xref: https://github.com/taskrabbit/elasticsearch-dump#bypassing-self-sign-certificate-errors
219218

220219
args := []interface{}{
221220
"--direction=load",
222221
fmt.Sprintf(`--input=%v`, opt.interimDataDir),
223222
fmt.Sprintf(`--output=%v`, esURL),
223+
fmt.Sprintf("--httpAuthFile=%s", httpAuthFilePath),
224224
tlsArgs,
225225
}
226226
for _, arg := range strings.Fields(opt.esArgs) {

0 commit comments

Comments
 (0)