[cherry-pick] Fix output format + Add PreBackupActions logic (#113) (#…

…115)

Signed-off-by: hossainemruz <emruz@appscode.com>

charts/stash-postgres/templates/postgres-backup-function.yaml

@@ -25,6 +25,7 @@ spec:
   # target information
   - --namespace=${NAMESPACE:=default}
   - --appbinding=${TARGET_NAME:=}
+  - --backupsession=${BACKUP_SESSION:=}
   # cleanup information
   - --retention-keep-last=${RETENTION_KEEP_LAST:=0}
   - --retention-keep-hourly=${RETENTION_KEEP_HOURLY:=0}

charts/stash-postgres/templates/postgres-backup-task.yaml

@@ -16,6 +16,8 @@ spec:
     params:
     - name: outputDir
       value: /tmp/output
+    - name: secretVolume
+      value: secret-volume
   volumes:
   - name: secret-volume
     secret:

charts/stash-postgres/templates/postgres-restore-task.yaml

@@ -16,6 +16,8 @@ spec:
     params:
     - name: outputDir
       value: /tmp/output
+    - name: secretVolume
+      value: secret-volume
   volumes:
   - name: secret-volume
     secret:

go.mod

@@ -16,12 +16,12 @@ require (
 	k8s.io/apimachinery v0.18.3
 	k8s.io/client-go v12.0.0+incompatible
 	k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6
-	kmodules.xyz/client-go v0.0.0-20200630053911-20d035822d35
+	kmodules.xyz/client-go v0.0.0-20200714060506-0b57b4060595
 	kmodules.xyz/crd-schema-fuzz v0.0.0-20200521005638-2433a187de95
 	kmodules.xyz/custom-resources v0.0.0-20200604135349-9e9f5c4fdba9
 	kmodules.xyz/offshoot-api v0.0.0-20200521035628-e135bf07b226
 	sigs.k8s.io/yaml v1.2.0
-	stash.appscode.dev/apimachinery v0.10.0-beta.1
+	stash.appscode.dev/apimachinery v0.10.0-beta.1.0.20200717091944-a8f3bb469467
 )
 
 replace bitbucket.org/ww/goautoneg => gomodules.xyz/goautoneg v0.0.0-20120707110453-a547fc61f48d

go.sum

@@ -913,8 +913,8 @@ kmodules.xyz/client-go v0.0.0-20200521005126-35ce6bd4ed46/go.mod h1:sY/eoe4ktxZE
 kmodules.xyz/client-go v0.0.0-20200521065424-173e32c78a20/go.mod h1:sY/eoe4ktxZEoHpr5NpAQ5s22VSwTE8psJtKVeVgLRY=
 kmodules.xyz/client-go v0.0.0-20200525195850-2fd180961371 h1:PPawDOMyDHGeDPN8j1epNozaIB/Z7MlJsXpwm/r4jgk=
 kmodules.xyz/client-go v0.0.0-20200525195850-2fd180961371/go.mod h1:sY/eoe4ktxZEoHpr5NpAQ5s22VSwTE8psJtKVeVgLRY=
-kmodules.xyz/client-go v0.0.0-20200630053911-20d035822d35 h1:gDzZWVvgAaEBzo4lxMGhPUWqySgFyFDkcqw3NskZiwQ=
-kmodules.xyz/client-go v0.0.0-20200630053911-20d035822d35/go.mod h1:sY/eoe4ktxZEoHpr5NpAQ5s22VSwTE8psJtKVeVgLRY=
+kmodules.xyz/client-go v0.0.0-20200714060506-0b57b4060595 h1:KgvN4M8VbHU6JV4rgYzsUKYhUtF1uCt//dLOyEdG330=
+kmodules.xyz/client-go v0.0.0-20200714060506-0b57b4060595/go.mod h1:sY/eoe4ktxZEoHpr5NpAQ5s22VSwTE8psJtKVeVgLRY=
 kmodules.xyz/constants v0.0.0-20200506032633-a21e58ceec72/go.mod h1:DbiFk1bJ1KEO94t1SlAn7tzc+Zz95rSXgyUKa2nzPmY=
 kmodules.xyz/crd-schema-fuzz v0.0.0-20200521005638-2433a187de95 h1:v0S/+ftzL6Xrs9XevgchAOJyPKlRQXPiZf87xotj3X4=
 kmodules.xyz/crd-schema-fuzz v0.0.0-20200521005638-2433a187de95/go.mod h1:jpu8xFsDKd6kAWUAKk8oTu/GQGBWqhrcaDeOJdaCJnk=
@@ -947,6 +947,6 @@ sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
 sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
-stash.appscode.dev/apimachinery v0.10.0-beta.1 h1:R1IhAhZ/iUbs2M+v9LYJacDCOmc4lk00JzSjVBrxOG0=
-stash.appscode.dev/apimachinery v0.10.0-beta.1/go.mod h1:v3rBovazmdCv/9rAA5U7LpDRFhVqcLsagnLlbPfD2Eg=
+stash.appscode.dev/apimachinery v0.10.0-beta.1.0.20200717091944-a8f3bb469467 h1:ghoHaD+TRpDGB1sjA6F+4wT4uKNF/hhbulzARzO7AXE=
+stash.appscode.dev/apimachinery v0.10.0-beta.1.0.20200717091944-a8f3bb469467/go.mod h1:lyIgBXoTj0cugPfVQC5IjOsMiQEx0YQuvr3KycGltyU=
 vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI=

pkg/backup.go

@@ -23,13 +23,16 @@ import (
 	"strings"
 
 	api_v1beta1 "stash.appscode.dev/apimachinery/apis/stash/v1beta1"
+	stash "stash.appscode.dev/apimachinery/client/clientset/versioned"
 	"stash.appscode.dev/apimachinery/pkg/restic"
+	api_util "stash.appscode.dev/apimachinery/pkg/util"
 
 	"github.com/appscode/go/flags"
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
+	appcatalog "kmodules.xyz/custom-resources/apis/appcatalog/v1alpha1"
 	appcatalog_cs "kmodules.xyz/custom-resources/client/clientset/versioned"
 	v1 "kmodules.xyz/offshoot-api/api/v1"
 )
@@ -67,20 +70,32 @@ func NewCmdBackup() *cobra.Command {
 			if err != nil {
 				return err
 			}
+			opt.stashClient, err = stash.NewForConfig(config)
+			if err != nil {
+				return err
+			}
 			opt.catalogClient, err = appcatalog_cs.NewForConfig(config)
 			if err != nil {
 				return err
 			}
+			targetRef := api_v1beta1.TargetRef{
+				APIVersion: appcatalog.SchemeGroupVersion.String(),
+				Kind:       appcatalog.ResourceKindApp,
+				Name:       opt.appBindingName,
+			}
 
 			var backupOutput *restic.BackupOutput
-			backupOutput, err = opt.backupPostgreSQL()
+			backupOutput, err = opt.backupPostgreSQL(targetRef)
 			if err != nil {
 				backupOutput = &restic.BackupOutput{
-					HostBackupStats: []api_v1beta1.HostBackupStats{
-						{
-							Hostname: opt.backupOptions.Host,
-							Phase:    api_v1beta1.HostBackupFailed,
-							Error:    err.Error(),
+					BackupTargetStatus: api_v1beta1.BackupTargetStatus{
+						Ref: targetRef,
+						Stats: []api_v1beta1.HostBackupStats{
+							{
+								Hostname: opt.backupOptions.Host,
+								Phase:    api_v1beta1.HostBackupFailed,
+								Error:    err.Error(),
+							},
 						},
 					},
 				}
@@ -101,6 +116,7 @@ func NewCmdBackup() *cobra.Command {
 	cmd.Flags().StringVar(&masterURL, "master", masterURL, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
 	cmd.Flags().StringVar(&kubeconfigPath, "kubeconfig", kubeconfigPath, "Path to kubeconfig file with authorization information (the master location is set by the master flag).")
 	cmd.Flags().StringVar(&opt.namespace, "namespace", "default", "Namespace of Backup/Restore Session")
+	cmd.Flags().StringVar(&opt.backupSessionName, "backupsession", opt.backupSessionName, "Name of the Backup Session")
 	cmd.Flags().StringVar(&opt.appBindingName, "appbinding", opt.appBindingName, "Name of the app binding")
 
 	cmd.Flags().StringVar(&opt.setupOptions.Provider, "provider", opt.setupOptions.Provider, "Backend provider (i.e. gcs, s3, azure etc)")
@@ -130,10 +146,25 @@ func NewCmdBackup() *cobra.Command {
 	return cmd
 }
 
-func (opt *postgresOptions) backupPostgreSQL() (*restic.BackupOutput, error) {
+func (opt *postgresOptions) backupPostgreSQL(targetRef api_v1beta1.TargetRef) (*restic.BackupOutput, error) {
+	// if any pre-backup actions has been assigned to it, execute them
+	actionOptions := api_util.ActionOptions{
+		StashClient:       opt.stashClient,
+		TargetRef:         targetRef,
+		SetupOptions:      opt.setupOptions,
+		BackupSessionName: opt.backupSessionName,
+		Namespace:         opt.namespace,
+	}
+	err := api_util.ExecutePreBackupActions(actionOptions)
+	if err != nil {
+		return nil, err
+	}
+	// wait until the backend repository has been initialized.
+	err = api_util.WaitForBackendRepository(actionOptions)
+	if err != nil {
+		return nil, err
+	}
 	// apply nice, ionice settings from env
-	var err error
-
 	opt.setupOptions.Nice, err = v1.NiceSettingsFromEnv()
 	if err != nil {
 		return nil, err
@@ -194,6 +225,6 @@ func (opt *postgresOptions) backupPostgreSQL() (*restic.BackupOutput, error) {
 	}
 
 	// Run backup
-	return resticWrapper.RunBackup(opt.backupOptions)
+	return resticWrapper.RunBackup(opt.backupOptions, targetRef)
 
 }

pkg/restore.go

@@ -29,6 +29,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
+	appcatalog "kmodules.xyz/custom-resources/apis/appcatalog/v1alpha1"
 	appcatalog_cs "kmodules.xyz/custom-resources/client/clientset/versioned"
 	v1 "kmodules.xyz/offshoot-api/api/v1"
 )
@@ -71,15 +72,23 @@ func NewCmdRestore() *cobra.Command {
 				return err
 			}
 
+			targetRef := api_v1beta1.TargetRef{
+				APIVersion: appcatalog.SchemeGroupVersion.String(),
+				Kind:       appcatalog.ResourceKindApp,
+				Name:       opt.appBindingName,
+			}
 			var restoreOutput *restic.RestoreOutput
-			restoreOutput, err = opt.restorePostgreSQL()
+			restoreOutput, err = opt.restorePostgreSQL(targetRef)
 			if err != nil {
 				restoreOutput = &restic.RestoreOutput{
-					HostRestoreStats: []api_v1beta1.HostRestoreStats{
-						{
-							Hostname: opt.dumpOptions.Host,
-							Phase:    api_v1beta1.HostRestoreFailed,
-							Error:    err.Error(),
+					RestoreTargetStatus: api_v1beta1.RestoreMemberStatus{
+						Ref: targetRef,
+						Stats: []api_v1beta1.HostRestoreStats{
+							{
+								Hostname: opt.dumpOptions.Host,
+								Phase:    api_v1beta1.HostRestoreFailed,
+								Error:    err.Error(),
+							},
 						},
 					},
 				}
@@ -120,7 +129,7 @@ func NewCmdRestore() *cobra.Command {
 	return cmd
 }
 
-func (opt *postgresOptions) restorePostgreSQL() (*restic.RestoreOutput, error) {
+func (opt *postgresOptions) restorePostgreSQL(targetRef api_v1beta1.TargetRef) (*restic.RestoreOutput, error) {
 	// apply nice, ionice settings from env
 	var err error
 
@@ -177,5 +186,5 @@ func (opt *postgresOptions) restorePostgreSQL() (*restic.RestoreOutput, error) {
 	}
 
 	// Run dump
-	return resticWrapper.Dump(opt.dumpOptions)
+	return resticWrapper.Dump(opt.dumpOptions, targetRef)
 }

pkg/util.go

@@ -19,6 +19,7 @@ package pkg
 import (
 	"fmt"
 
+	stash "stash.appscode.dev/apimachinery/client/clientset/versioned"
 	"stash.appscode.dev/apimachinery/pkg/restic"
 
 	"github.com/appscode/go/log"
@@ -41,14 +42,16 @@ const (
 
 type postgresOptions struct {
 	kubeClient    kubernetes.Interface
+	stashClient   stash.Interface
 	catalogClient appcatalog_cs.Interface
 
-	namespace      string
-	appBindingName string
-	backupCMD      string
-	pgArgs         string
-	outputDir      string
-	waitTimeout    int32
+	namespace         string
+	backupSessionName string
+	appBindingName    string
+	backupCMD         string
+	pgArgs            string
+	outputDir         string
+	waitTimeout       int32
 
 	setupOptions  restic.SetupOptions
 	backupOptions restic.BackupOptions

vendor/kmodules.xyz/client-go/Makefile

@@ -1,5 +1,4 @@
-# Copyright 2019 AppsCode Inc.
-# Copyright 2016 The Kubernetes Authors.
+# Copyright AppsCode Inc. and Contributors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

vendor/kmodules.xyz/client-go/api/v1/conditions.go

@@ -1,5 +1,5 @@
 /*
-Copyright The Kmodules Authors.
+Copyright AppsCode Inc. and Contributors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

vendor/kmodules.xyz/client-go/api/v1/doc.go

@@ -1,5 +1,5 @@
 /*
-Copyright The Kmodules Authors.
+Copyright AppsCode Inc. and Contributors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

vendor/kmodules.xyz/client-go/apiextensions/kubernetes.go

@@ -1,5 +1,5 @@
 /*
-Copyright The Kmodules Authors.
+Copyright AppsCode Inc. and Contributors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

vendor/kmodules.xyz/client-go/apiextensions/types.go

@@ -1,3 +1,19 @@
+/*
+Copyright AppsCode Inc. and Contributors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package apiextensions
 
 import (

vendor/kmodules.xyz/client-go/apiextensions/v1/crd.go

@@ -1,5 +1,5 @@
 /*
-Copyright The Kmodules Authors.
+Copyright AppsCode Inc. and Contributors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

vendor/kmodules.xyz/client-go/apiextensions/v1beta1/crd.go

@@ -1,5 +1,5 @@
 /*
-Copyright The Kmodules Authors.
+Copyright AppsCode Inc. and Contributors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

vendor/kmodules.xyz/client-go/core/v1/configmap.go

@@ -1,5 +1,5 @@
 /*
-Copyright The Kmodules Authors.
+Copyright AppsCode Inc. and Contributors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

vendor/kmodules.xyz/client-go/core/v1/endpoints.go

@@ -1,5 +1,5 @@
 /*
-Copyright The Kmodules Authors.
+Copyright AppsCode Inc. and Contributors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

vendor/kmodules.xyz/client-go/core/v1/kubernetes.go

@@ -1,5 +1,5 @@
 /*
-Copyright The Kmodules Authors.
+Copyright AppsCode Inc. and Contributors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

vendor/kmodules.xyz/client-go/core/v1/node.go

@@ -1,5 +1,5 @@
 /*
-Copyright The Kmodules Authors.
+Copyright AppsCode Inc. and Contributors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

vendor/kmodules.xyz/client-go/core/v1/pod.go

@@ -1,5 +1,5 @@
 /*
-Copyright The Kmodules Authors.
+Copyright AppsCode Inc. and Contributors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

vendor/kmodules.xyz/client-go/core/v1/pv.go

@@ -1,5 +1,5 @@
 /*
-Copyright The Kmodules Authors.
+Copyright AppsCode Inc. and Contributors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

vendor/kmodules.xyz/client-go/core/v1/pvc.go

@@ -1,5 +1,5 @@
 /*
-Copyright The Kmodules Authors.
+Copyright AppsCode Inc. and Contributors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

vendor/kmodules.xyz/client-go/core/v1/rc.go

@@ -1,5 +1,5 @@
 /*
-Copyright The Kmodules Authors.
+Copyright AppsCode Inc. and Contributors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.