Add ability to upload logical backup to gcs

docker/logical-backup/Dockerfile

@@ -13,7 +13,10 @@ RUN apt-get update     \
         curl \
         jq \
         gnupg \
+        gcc \
+        libffi-dev \
     && pip3 install --no-cache-dir awscli --upgrade \
+    && pip3 install --no-cache-dir gsutil --upgrade \
     && echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
     && cat /etc/apt/sources.list.d/pgdg.list \
     && curl --silent https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \

docker/logical-backup/dump.sh

@@ -37,7 +37,7 @@ function aws_upload {
     PATH_TO_BACKUP=s3://$LOGICAL_BACKUP_S3_BUCKET"/spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz
 
     args=()
-
+    
     [[ ! -z "$EXPECTED_SIZE" ]] && args+=("--expected-size=$EXPECTED_SIZE")
     [[ ! -z "$LOGICAL_BACKUP_S3_ENDPOINT" ]] && args+=("--endpoint-url=$LOGICAL_BACKUP_S3_ENDPOINT")
     [[ ! -z "$LOGICAL_BACKUP_S3_REGION" ]] && args+=("--region=$LOGICAL_BACKUP_S3_REGION")
@@ -46,6 +46,23 @@ function aws_upload {
     aws s3 cp - "$PATH_TO_BACKUP" "${args[@]//\'/}"
 }
 
+function gcs_upload {
+    PATH_TO_BACKUP=gs://$LOGICAL_BACKUP_S3_BUCKET"/spilo/"$SCOPE$LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX"/logical_backups/"$(date +%s).sql.gz
+
+    gsutil -o Credentials:gs_service_key_file=$LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS cp - "$PATH_TO_BACKUP"
+}
+
+function upload {
+    case $LOGICAL_BACKUP_PROVIDER in
+        "gcs")
+            gcs_upload
+            ;;
+        *)
+            aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF))
+            ;;
+    esac
+}
+
 function get_pods {
     declare -r SELECTOR="$1"
 
@@ -93,7 +110,7 @@ for search in "${search_strategy[@]}"; do
 done
 
 set -x
-dump | compress | aws_upload $(($(estimate_size) / DUMP_SIZE_COEFF))
+dump | compress | upload
 [[ ${PIPESTATUS[0]} != 0 || ${PIPESTATUS[1]} != 0 || ${PIPESTATUS[2]} != 0 ]] && (( ERRORCOUNT += 1 ))
 set +x
 

docs/reference/operator_parameters.md

@@ -461,6 +461,10 @@ grouped under the `logical_backup` key.
   The default image is the same image built with the Zalando-internal CI
   pipeline. Default: "registry.opensource.zalan.do/acid/logical-backup"
 
+* **logical_backup_provider**
+  Specifies the storage provider to which the backup should be uploaded (`s3` or `gcs`).
+  Default: "s3"
+
 * **logical_backup_s3_bucket**
   S3 bucket to store backup results. The bucket has to be present and
   accessible by Postgres pods. Default: empty.
@@ -481,6 +485,8 @@ grouped under the `logical_backup` key.
 * **logical_backup_s3_secret_access_key**
   When set, value will be in AWS_SECRET_ACCESS_KEY env variable. The Default is empty.
 
+* **logical_backup_google_application_credentials**
+  Specifies the path of the google cloud service account json file. Default is empty.
 ## Debugging the operator
 
 Options to aid debugging of the operator itself. Grouped under the `debug` key.

pkg/apis/acid.zalan.do/v1/operator_configuration_type.go

@@ -65,12 +65,12 @@ type KubernetesMetaConfiguration struct {
 	// TODO: use a proper toleration structure?
 	PodToleration map[string]string `json:"toleration,omitempty"`
 	// TODO: use namespacedname
-	PodEnvironmentConfigMap    string        `json:"pod_environment_configmap,omitempty"`
-	PodPriorityClassName       string        `json:"pod_priority_class_name,omitempty"`
-	MasterPodMoveTimeout       Duration      `json:"master_pod_move_timeout,omitempty"`
-	EnablePodAntiAffinity      bool          `json:"enable_pod_antiaffinity,omitempty"`
-	PodAntiAffinityTopologyKey string        `json:"pod_antiaffinity_topology_key,omitempty"`
-	PodManagementPolicy        string        `json:"pod_management_policy,omitempty"`
+	PodEnvironmentConfigMap    string   `json:"pod_environment_configmap,omitempty"`
+	PodPriorityClassName       string   `json:"pod_priority_class_name,omitempty"`
+	MasterPodMoveTimeout       Duration `json:"master_pod_move_timeout,omitempty"`
+	EnablePodAntiAffinity      bool     `json:"enable_pod_antiaffinity,omitempty"`
+	PodAntiAffinityTopologyKey string   `json:"pod_antiaffinity_topology_key,omitempty"`
+	PodManagementPolicy        string   `json:"pod_management_policy,omitempty"`
 }
 
 // PostgresPodResourcesDefaults defines the spec of default resources
@@ -154,14 +154,16 @@ type ScalyrConfiguration struct {
 
 // OperatorLogicalBackupConfiguration defines configuration for logical backup
 type OperatorLogicalBackupConfiguration struct {
-	Schedule          string `json:"logical_backup_schedule,omitempty"`
-	DockerImage       string `json:"logical_backup_docker_image,omitempty"`
-	S3Bucket          string `json:"logical_backup_s3_bucket,omitempty"`
-	S3Region          string `json:"logical_backup_s3_region,omitempty"`
-	S3Endpoint        string `json:"logical_backup_s3_endpoint,omitempty"`
-	S3AccessKeyID     string `json:"logical_backup_s3_access_key_id,omitempty"`
-	S3SecretAccessKey string `json:"logical_backup_s3_secret_access_key,omitempty"`
-	S3SSE             string `json:"logical_backup_s3_sse,omitempty"`
+	Schedule                     string `json:"logical_backup_schedule,omitempty"`
+	DockerImage                  string `json:"logical_backup_docker_image,omitempty"`
+	BackupProvider               string `json:"logical_backup_provider,omitempty"`
+	S3Bucket                     string `json:"logical_backup_s3_bucket,omitempty"`
+	S3Region                     string `json:"logical_backup_s3_region,omitempty"`
+	S3Endpoint                   string `json:"logical_backup_s3_endpoint,omitempty"`
+	S3AccessKeyID                string `json:"logical_backup_s3_access_key_id,omitempty"`
+	S3SecretAccessKey            string `json:"logical_backup_s3_secret_access_key,omitempty"`
+	S3SSE                        string `json:"logical_backup_s3_sse,omitempty"`
+	GoogleApplicationCredentials string `json:"logical_backup_google_application_credentials,omitempty"`
 }
 
 // OperatorConfigurationData defines the operation config

pkg/cluster/k8sres.go

@@ -1701,6 +1701,10 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar {
 			},
 		},
 		// Bucket env vars
+		{
+			Name:  "LOGICAL_BACKUP_PROVIDER",
+			Value: c.OpConfig.LogicalBackup.LogicalBackupProvider,
+		},
 		{
 			Name:  "LOGICAL_BACKUP_S3_BUCKET",
 			Value: c.OpConfig.LogicalBackup.LogicalBackupS3Bucket,
@@ -1721,6 +1725,10 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar {
 			Name:  "LOGICAL_BACKUP_S3_BUCKET_SCOPE_SUFFIX",
 			Value: getBucketScopeSuffix(string(c.Postgresql.GetUID())),
 		},
+		{
+			Name:  "LOGICAL_BACKUP_GOOGLE_APPLICATION_CREDENTIALS",
+			Value: c.OpConfig.LogicalBackup.LogicalBackupGoogleApplicationCredentials,
+		},
 		// Postgres env vars
 		{
 			Name:  "PG_VERSION",

pkg/controller/operator_config.go

@@ -106,12 +106,14 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur
 	// logical backup config
 	result.LogicalBackupSchedule = fromCRD.LogicalBackup.Schedule
 	result.LogicalBackupDockerImage = fromCRD.LogicalBackup.DockerImage
+	result.LogicalBackupProvider = fromCRD.LogicalBackup.BackupProvider
 	result.LogicalBackupS3Bucket = fromCRD.LogicalBackup.S3Bucket
 	result.LogicalBackupS3Region = fromCRD.LogicalBackup.S3Region
 	result.LogicalBackupS3Endpoint = fromCRD.LogicalBackup.S3Endpoint
 	result.LogicalBackupS3AccessKeyID = fromCRD.LogicalBackup.S3AccessKeyID
 	result.LogicalBackupS3SecretAccessKey = fromCRD.LogicalBackup.S3SecretAccessKey
 	result.LogicalBackupS3SSE = fromCRD.LogicalBackup.S3SSE
+	result.LogicalBackupGoogleApplicationCredentials = fromCRD.LogicalBackup.GoogleApplicationCredentials
 
 	// debug config
 	result.DebugLogging = fromCRD.OperatorDebug.DebugLogging

pkg/util/config/config.go

@@ -73,14 +73,16 @@ type Scalyr struct {
 
 // LogicalBackup defines configuration for logical backup
 type LogicalBackup struct {
-	LogicalBackupSchedule          string `name:"logical_backup_schedule" default:"30 00 * * *"`
-	LogicalBackupDockerImage       string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup"`
-	LogicalBackupS3Bucket          string `name:"logical_backup_s3_bucket" default:""`
-	LogicalBackupS3Region          string `name:"logical_backup_s3_region" default:""`
-	LogicalBackupS3Endpoint        string `name:"logical_backup_s3_endpoint" default:""`
-	LogicalBackupS3AccessKeyID     string `name:"logical_backup_s3_access_key_id" default:""`
-	LogicalBackupS3SecretAccessKey string `name:"logical_backup_s3_secret_access_key" default:""`
-	LogicalBackupS3SSE             string `name:"logical_backup_s3_sse" default:"AES256"`
+	LogicalBackupSchedule                     string `name:"logical_backup_schedule" default:"30 00 * * *"`
+	LogicalBackupDockerImage                  string `name:"logical_backup_docker_image" default:"registry.opensource.zalan.do/acid/logical-backup"`
+	LogicalBackupProvider                     string `name:"logical_backup_provider" default:"s3"`
+	LogicalBackupS3Bucket                     string `name:"logical_backup_s3_bucket" default:""`
+	LogicalBackupS3Region                     string `name:"logical_backup_s3_region" default:""`
+	LogicalBackupS3Endpoint                   string `name:"logical_backup_s3_endpoint" default:""`
+	LogicalBackupS3AccessKeyID                string `name:"logical_backup_s3_access_key_id" default:""`
+	LogicalBackupS3SecretAccessKey            string `name:"logical_backup_s3_secret_access_key" default:""`
+	LogicalBackupS3SSE                        string `name:"logical_backup_s3_sse" default:"AES256"`
+	LogicalBackupGoogleApplicationCredentials string `name:"logical_backup_google_application_credentials" default:""`
 }
 
 // Config describes operator config