Remove metrics options from function (#5)

Author: hossainemruz

chart/stash-postgres/README.md

@@ -61,19 +61,11 @@ The following table lists the configurable parameters of the `stash-postgres` ch
 | `docker.tag`      | Tag of the image that is used to backup/restore PostgreSQL database. This is usually same as the database version it can backup. | `11.2`           |
 | `backup.pgArgs`   | Optional arguments to pass to `pgdump` command  during bakcup process                                                            |                  |
 | `restore.pgArgs`  | Optional arguments to pass to `psql` command during restore process                                                              |                  |
-| `metrics.enabled` | Specifies whether to send Prometheus metrics                                                                                     | `true`           |
-| `metrics.labels`  | Optional comma separated labels to add to the Prometheus metrics                                                                 |                  |
 
 Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`.
 
 For example:
 
 ```console
-helm install --name stash-postgres-11.2 --set metrics.enabled=false appscode/stash-postgres
-```
-
-**Tips:** Use escape character (`\`) while providing multiple comma-separated labels for `metrics.labels`.
-
-```console
- helm install chart/stash-postgres --set metrics.labels="k1=v1\,k2=v2"
+helm install --name stash-postgres-11.2 --set docker.registry=my-registry appscode/stash-postgres
 ```

chart/stash-postgres/templates/postgres-backup-function.yaml

@@ -35,11 +35,6 @@ spec:
   - --retention-dry-run=${RETENTION_DRY_RUN:=false}
   # output & metric information
   - --output-dir=${outputDir:=}
-  - --metrics-enabled={{ .Values.metrics.enabled }}
-  - --metrics-pushgateway-url=${PROMETHEUS_PUSHGATEWAY_URL:=}
-  {{- if .Values.metrics.labels }}
-  - --metrics-labels={{ .Values.metrics.labels | quote }}
-  {{- end }}
   volumeMounts:
   - name: ${secretVolume}
     mountPath: /etc/repository/secret

chart/stash-postgres/templates/postgres-restore-function.yaml

@@ -27,11 +27,6 @@ spec:
   - --snapshot=${RESTORE_SNAPSHOTS:=}
   # output & metric information
   - --output-dir=${outputDir:=}
-  - --metrics-enabled={{ .Values.metrics.enabled }}
-  - --metrics-pushgateway-url=${PROMETHEUS_PUSHGATEWAY_URL:=}
-  {{- if .Values.metrics.labels }}
-  - --metrics-labels={{ .Values.metrics.labels | quote }}
-  {{- end }}
   volumeMounts:
   - name: ${secretVolume}
     mountPath: /etc/repository/secret

chart/stash-postgres/values.yaml

@@ -13,8 +13,3 @@ backup:
   pgArgs: ""
 restore:
   pgArgs: ""
-
-# default values for prometheus metrics
-metrics:
-  enabled: true
-  labels: []

docs/postgres.md

@@ -683,6 +683,7 @@ So, from the above output, we can see the table `company` that we had created in
 To cleanup the Kubernetes resources created by this tutorial, run:
 
 ```console
+kubectl delete backupconfiguration -n demo sample-postgres-backup
 kubectl delete restoresession -n demo sample-postgres-restore
 kubectl delete pg -n demo restored-postgres
 kubectl delete pg -n demo sample-postgres

go.mod

@@ -9,15 +9,15 @@ require (
 	k8s.io/client-go v11.0.0+incompatible
 	kmodules.xyz/client-go v0.0.0-20190808141354-bbb9e14f60ab
 	kmodules.xyz/custom-resources v0.0.0-20190808144301-114abf10dfe2
-	stash.appscode.dev/stash v0.9.0-rc.0
+	stash.appscode.dev/stash v0.9.0-rc.0.0.20190826131715-3be27ba2689e
 )
 
 replace (
 	github.com/Azure/go-autorest => github.com/Azure/go-autorest v12.4.2+incompatible
 	k8s.io/api => k8s.io/api v0.0.0-20190313235455-40a48860b5ab
 	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.0.0-20190315093550-53c4693659ed
 	k8s.io/apimachinery => github.com/kmodules/apimachinery v0.0.0-20190508045248-a52a97a7a2bf
-	k8s.io/apiserver => github.com/kmodules/apiserver v0.0.0-20190508082252-8397d761d4b5
+	k8s.io/apiserver => github.com/kmodules/apiserver v0.0.0-20190811223248-5a95b2df4348
 	k8s.io/cli-runtime => k8s.io/cli-runtime v0.0.0-20190314001948-2899ed30580f
 	k8s.io/cloud-provider => k8s.io/cloud-provider v0.0.0-20190314002645-c892ea32361a
 	k8s.io/component-base => k8s.io/component-base v0.0.0-20190314000054-4a91899592f4

go.sum

@@ -177,7 +177,7 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kmodules/apimachinery v0.0.0-20190508045248-a52a97a7a2bf h1:XUigLZjjxvtR+TD7mncq0wUOaaG2H/zslqM42crk0SY=
 github.com/kmodules/apimachinery v0.0.0-20190508045248-a52a97a7a2bf/go.mod h1:1nbDY1cBTGna53FTjqnehJWker1BB75r+ElWRqNcYo8=
-github.com/kmodules/apiserver v0.0.0-20190508082252-8397d761d4b5/go.mod h1:rMEImsX8dFD0jGmDQXq3zbWddcIU8vlcrMWPzpXEm9g=
+github.com/kmodules/apiserver v0.0.0-20190811223248-5a95b2df4348/go.mod h1:rMEImsX8dFD0jGmDQXq3zbWddcIU8vlcrMWPzpXEm9g=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -431,8 +431,8 @@ kmodules.xyz/client-go v0.0.0-20190808141354-bbb9e14f60ab h1:rnKcFaDLtqpdooase2B
 kmodules.xyz/client-go v0.0.0-20190808141354-bbb9e14f60ab/go.mod h1:1GI6h5D31op/2D+Hrn9DhXW6iUd5wsHBQRoLeY7fndM=
 kmodules.xyz/custom-resources v0.0.0-20190808144301-114abf10dfe2 h1:u0zEFCvAttk09O64UzY1DVXNke9uAANmf4vllnVquS8=
 kmodules.xyz/custom-resources v0.0.0-20190808144301-114abf10dfe2/go.mod h1:vlKyFcCXC+2Kfn3Fa5Z7RnBWyp4t46FSeEutNqpqMm8=
-kmodules.xyz/objectstore-api v0.0.0-20190808153322-733e8798e8de h1:Z/z3SSS+mvRGzLT5HXLazogjnOYIfbYTYWrKBsiLtp8=
-kmodules.xyz/objectstore-api v0.0.0-20190808153322-733e8798e8de/go.mod h1:Wx+Ffc7weI6zv7NQvq0GWXMGGQFn6LqVb0KcXQFwRFs=
+kmodules.xyz/objectstore-api v0.0.0-20190824212210-196174aa0fc0 h1:ELq3ybK3noQP9SyZx34QsnZgU2As9r8hPAGEnuIy7Co=
+kmodules.xyz/objectstore-api v0.0.0-20190824212210-196174aa0fc0/go.mod h1:vFvmrMvRlkcUANfU4Ko3dZR3HUhWMUQc1sEHcwufims=
 kmodules.xyz/offshoot-api v0.0.0-20190808152534-e3dc715f844b h1:VZUtg4L9Qm2C7PBwa4sp/SvHckKidZSpkA33GoB24SA=
 kmodules.xyz/offshoot-api v0.0.0-20190808152534-e3dc715f844b/go.mod h1:AzvHBjJvhFwUEiWrEKfdsgoqt5Ulio44Beo1vPoq7ww=
 kmodules.xyz/openshift v0.0.0-20190808144841-c8f9a927f1d1 h1:NmVj5+kPpUgoxRans2XZt5SXm9cVy8E3b4yHaeVzqak=
@@ -444,5 +444,5 @@ sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5
 sigs.k8s.io/structured-merge-diff v0.0.0-20190302045857-e85c7b244fd2/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
-stash.appscode.dev/stash v0.9.0-rc.0 h1:GhzBnqHYBgq+2bcuwsDK3nhq2PPEhGkBH0EFAXp2OmY=
-stash.appscode.dev/stash v0.9.0-rc.0/go.mod h1:/YjAPx/9E4ScxWea1mbJC6qx9TomhqS83AUy2LufVVk=
+stash.appscode.dev/stash v0.9.0-rc.0.0.20190826131715-3be27ba2689e h1:AYVMg2yTN7lxe7IThGH0fc4iaBS4H8eBjCcu7w0OLmo=
+stash.appscode.dev/stash v0.9.0-rc.0.0.20190826131715-3be27ba2689e/go.mod h1:6WrRpjkI1av7305B60BLRx+PZzwMAoiGmOd5ewYAAyw=

pkg/backup.go

@@ -10,7 +10,6 @@ import (
 	"github.com/appscode/go/log"
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
 	appcatalog_cs "kmodules.xyz/custom-resources/client/clientset/versioned"
@@ -44,9 +43,6 @@ func NewCmdBackup() *cobra.Command {
 			Host:          restic.DefaultHost,
 			StdinFileName: PgDumpFile,
 		}
-		metrics = restic.MetricsOptions{
-			JobName: JobPGBackup,
-		}
 	)
 
 	cmd := &cobra.Command{
@@ -117,13 +113,7 @@ func NewCmdBackup() *cobra.Command {
 
 			// Run backup
 			backupOutput, backupErr := resticWrapper.RunBackup(backupOpt)
-			// If metrics are enabled then generate metrics
-			if metrics.Enabled {
-				err := backupOutput.HandleMetrics(&metrics, backupErr)
-				if err != nil {
-					return errors.NewAggregate([]error{backupErr, err})
-				}
-			}
+
 			// If output directory specified, then write the output in "output.json" file in the specified directory
 			if backupErr == nil && outputDir != "" {
 				err := backupOutput.WriteOutput(filepath.Join(outputDir, restic.DefaultOutputFileName))
@@ -166,11 +156,6 @@ func NewCmdBackup() *cobra.Command {
 
 	cmd.Flags().StringVar(&outputDir, "output-dir", outputDir, "Directory where output.json file will be written (keep empty if you don't need to write output in file)")
 
-	cmd.Flags().BoolVar(&metrics.Enabled, "metrics-enabled", metrics.Enabled, "Specify whether to export Prometheus metrics")
-	cmd.Flags().StringVar(&metrics.PushgatewayURL, "metrics-pushgateway-url", metrics.PushgatewayURL, "Pushgateway URL where the metrics will be pushed")
-	cmd.Flags().StringVar(&metrics.MetricFileDir, "metrics-dir", metrics.MetricFileDir, "Directory where to write metric.prom file (keep empty if you don't want to write metric in a text file)")
-	cmd.Flags().StringSliceVar(&metrics.Labels, "metrics-labels", metrics.Labels, "Labels to apply in exported metrics")
-
 	return cmd
 }
 

pkg/restore.go

@@ -6,7 +6,6 @@ import (
 	"github.com/appscode/go/flags"
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
 	appcatalog_cs "kmodules.xyz/custom-resources/client/clientset/versioned"
@@ -30,9 +29,6 @@ func NewCmdRestore() *cobra.Command {
 			Host:     restic.DefaultHost,
 			FileName: PgDumpFile,
 		}
-		metrics = restic.MetricsOptions{
-			JobName: JobPGBackup,
-		}
 	)
 
 	cmd := &cobra.Command{
@@ -103,13 +99,7 @@ func NewCmdRestore() *cobra.Command {
 
 			// Run dump
 			dumpOutput, backupErr := resticWrapper.Dump(dumpOpt)
-			// If metrics are enabled then generate metrics
-			if metrics.Enabled {
-				err := dumpOutput.HandleMetrics(&metrics, backupErr)
-				if err != nil {
-					return errors.NewAggregate([]error{backupErr, err})
-				}
-			}
+
 			// If output directory specified, then write the output in "output.json" file in the specified directory
 			if backupErr == nil && outputDir != "" {
 				err := dumpOutput.WriteOutput(filepath.Join(outputDir, restic.DefaultOutputFileName))
@@ -144,11 +134,5 @@ func NewCmdRestore() *cobra.Command {
 	cmd.Flags().StringVar(&dumpOpt.Snapshot, "snapshot", dumpOpt.Snapshot, "Snapshot to dump")
 
 	cmd.Flags().StringVar(&outputDir, "output-dir", outputDir, "Directory where output.json file will be written (keep empty if you don't need to write output in file)")
-
-	cmd.Flags().BoolVar(&metrics.Enabled, "metrics-enabled", metrics.Enabled, "Specify whether to export Prometheus metrics")
-	cmd.Flags().StringVar(&metrics.PushgatewayURL, "metrics-pushgateway-url", metrics.PushgatewayURL, "Pushgateway URL where the metrics will be pushed")
-	cmd.Flags().StringVar(&metrics.MetricFileDir, "metrics-dir", metrics.MetricFileDir, "Directory where to write metric.prom file (keep empty if you don't want to write metric in a text file)")
-	cmd.Flags().StringSliceVar(&metrics.Labels, "metrics-labels", metrics.Labels, "Labels to apply in exported metrics")
-
 	return cmd
 }

vendor/kmodules.xyz/objectstore-api/api/v1/helpers.go

@@ -1,40 +1,61 @@
 package v1
 
 import (
+	"net/url"
+
 	"github.com/pkg/errors"
 	core "k8s.io/api/core/v1"
 )
 
-func (s Backend) Container() (string, error) {
-	if s.S3 != nil {
-		return s.S3.Bucket, nil
-	} else if s.GCS != nil {
-		return s.GCS.Bucket, nil
-	} else if s.Azure != nil {
-		return s.Azure.Container, nil
-	} else if s.Local != nil {
-		return s.Local.MountPath, nil
-	} else if s.Swift != nil {
-		return s.Swift.Container, nil
+const (
+	ProviderLocal = "local"
+	ProviderS3    = "s3"
+	ProviderGCS   = "gcs"
+	ProviderAzure = "azure"
+	ProviderSwift = "swift"
+	ProviderB2    = "b2"
+	ProviderRest  = "rest"
+)
+
+// Container returns name of the bucket
+func (backend Backend) Container() (string, error) {
+	if backend.Local != nil {
+		return backend.Local.MountPath, nil
+	} else if backend.S3 != nil {
+		return backend.S3.Bucket, nil
+	} else if backend.GCS != nil {
+		return backend.GCS.Bucket, nil
+	} else if backend.Azure != nil {
+		return backend.Azure.Container, nil
+	} else if backend.Swift != nil {
+		return backend.Swift.Container, nil
+	} else if backend.Rest != nil {
+		u, err := url.Parse(backend.Rest.URL)
+		if err != nil {
+			return "", err
+		}
+		return u.Host, nil
 	}
-	return "", errors.New("no storage provider is configured")
+	return "", errors.New("failed to get container. Reason: Unknown backend type.")
 }
 
-func (s Backend) Location() (string, error) {
-	if s.S3 != nil {
-		return "s3:" + s.S3.Bucket, nil
-	} else if s.GCS != nil {
-		return "gs:" + s.GCS.Bucket, nil
-	} else if s.Azure != nil {
-		return "azure:" + s.Azure.Container, nil
-	} else if s.Local != nil {
-		return "local:" + s.Local.MountPath, nil
-	} else if s.Swift != nil {
-		return "swift:" + s.Swift.Container, nil
+// Location returns the location of backend (<provider>:<bucket name>)
+func (backend Backend) Location() (string, error) {
+	if backend.S3 != nil {
+		return "s3:" + backend.S3.Bucket, nil
+	} else if backend.GCS != nil {
+		return "gs:" + backend.GCS.Bucket, nil
+	} else if backend.Azure != nil {
+		return "azure:" + backend.Azure.Container, nil
+	} else if backend.Local != nil {
+		return "local:" + backend.Local.MountPath, nil
+	} else if backend.Swift != nil {
+		return "swift:" + backend.Swift.Container, nil
 	}
 	return "", errors.New("no storage provider is configured")
 }
 
+// ToVolumeAndMount returns volumes and mounts for local backend
 func (l LocalSpec) ToVolumeAndMount(volName string) (core.Volume, core.VolumeMount) {
 	vol := core.Volume{
 		Name:         volName,
@@ -47,3 +68,68 @@ func (l LocalSpec) ToVolumeAndMount(volName string) (core.Volume, core.VolumeMou
 	}
 	return vol, mnt
 }
+
+// Prefix returns the prefix used in the backend
+func (backend Backend) Prefix() (string, error) {
+	if backend.Local != nil {
+		return "", nil
+	} else if backend.S3 != nil {
+		return backend.S3.Prefix, nil
+	} else if backend.GCS != nil {
+		return backend.GCS.Prefix, nil
+	} else if backend.Azure != nil {
+		return backend.Azure.Prefix, nil
+	} else if backend.Swift != nil {
+		return backend.Swift.Prefix, nil
+	} else if backend.Rest != nil {
+		u, err := url.Parse(backend.Rest.URL)
+		if err != nil {
+			return "", err
+		}
+		return u.Path, nil
+	}
+	return "", errors.New("failed to get prefix. Reason: Unknown backend type.")
+}
+
+// Provider returns the provider of the backend
+func (backend Backend) Provider() (string, error) {
+	if backend.Local != nil {
+		return ProviderLocal, nil
+	} else if backend.S3 != nil {
+		return ProviderS3, nil
+	} else if backend.GCS != nil {
+		return ProviderGCS, nil
+	} else if backend.Azure != nil {
+		return ProviderAzure, nil
+	} else if backend.Swift != nil {
+		return ProviderSwift, nil
+	} else if backend.B2 != nil {
+		return ProviderB2, nil
+	} else if backend.Rest != nil {
+		return ProviderRest, nil
+	}
+	return "", errors.New("unknown provider.")
+}
+
+// MaxConnections returns maximum parallel connection to use to connect with the backend
+// returns 0 if not specified
+func (backend Backend) MaxConnections() int {
+	if backend.GCS != nil {
+		return backend.GCS.MaxConnections
+	} else if backend.Azure != nil {
+		return backend.Azure.MaxConnections
+	} else if backend.B2 != nil {
+		return backend.B2.MaxConnections
+	}
+	return 0
+}
+
+// Endpoint returns endpoint of Restic rest server and S3/S3 compatible backend
+func (backend Backend) Endpoint() (string, bool) {
+	if backend.S3 != nil {
+		return backend.S3.Endpoint, true
+	} else if backend.Rest != nil {
+		return backend.Rest.URL, true
+	}
+	return "", false
+}