From 9a09b267581f3aeda653674171150b9627af1a97 Mon Sep 17 00:00:00 2001 From: Aaron Peschel Date: Thu, 24 Jun 2021 18:17:07 -0700 Subject: [PATCH] Add Support for Azure WAL-G Backups This commit adds support for using an Azure storage account as a backup location. It uses the existing GCS functionality as a reference for what to do, and follows the example set by GCS as closely as possible. The decision to name the cloud provider key "aws_or_gcp" is unfortunate while adding support for Azure, but I have left it alone to allow for this changeset to be backwards compatible. --- .../crds/operatorconfigurations.yaml | 2 + charts/postgres-operator/values.yaml | 3 + docs/administrator.md | 57 +++++++++++++++++++ docs/reference/operator_parameters.md | 6 ++ manifests/configmap.yaml | 1 + manifests/operatorconfiguration.crd.yaml | 2 + ...gresql-operator-default-configuration.yaml | 1 + .../v1/operator_configuration_type.go | 1 + pkg/cluster/k8sres.go | 14 +++++ pkg/controller/operator_config.go | 1 + pkg/util/config/config.go | 1 + 11 files changed, 89 insertions(+) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 82a737ae6..5bc95dcbf 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -392,6 +392,8 @@ spec: type: string wal_s3_bucket: type: string + wal_az_storage_account: + type: string logical_backup: type: object properties: diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 287835a24..3994d24ee 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -266,6 +266,9 @@ configAwsOrGcp: # GCS bucket to use for shipping WAL segments with WAL-E # wal_gs_bucket: "" + # Azure Storage Account to use for shipping WAL segments with WAL-G + # wal_az_storage_account: "" + # configure K8s cron job managed by the operator configLogicalBackup: # image for pods of the logical backup job (example runs pg_dumpall) diff --git a/docs/administrator.md b/docs/administrator.md index ad424cab8..eacb9f280 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -808,6 +808,63 @@ pod_environment_configmap: "postgres-operator-system/pod-env-overrides" ... ``` +### Azure setup + +To configure the operator on Azure these prerequisites are needed: + +* A storage account in the same region as the Kubernetes cluster. + +The configuration parameters that we will be using are: + +* `pod_environment_secret` +* `wal_az_storage_account` + +1. Generate the K8s secret resource that will contain your storage account's +access key. You will need a copy of this secret in every namespace you want to +create postgresql clusters. + +The latest version of WAL-G (v1.0) supports the use of a SASS token, but you'll +have to make due with using the primary or secondary access token until the +version of WAL-G is updated in the postgres-operator. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: psql-backup-creds + namespace: default +type: Opaque +stringData: + AZURE_STORAGE_ACCESS_KEY: +``` + +2. Setup pod environment configmap that instructs the operator to use WAL-G, +instead of WAL-E, for backup and restore. +```yml +apiVersion: v1 +kind: ConfigMap +metadata: + name: pod-env-overrides + namespace: postgres-operator-system +data: + # Any env variable used by spilo can be added + USE_WALG_BACKUP: "true" + USE_WALG_RESTORE: "true" + CLONE_USE_WALG_RESTORE: "true" +``` + +3. Setup your operator configuration values. With the `psql-backup-creds` +and `pod-env-overrides` resources applied to your cluster, ensure that the operator's configuration +is set up like the following: +```yml +... +aws_or_gcp: + pod_environment_secret: "pgsql-backup-creds" + pod_environment_configmap: "postgres-operator-system/pod-env-overrides" + wal_az_storage_account: "postgresbackupsbucket28302F2" # name of storage account to save the WAL-G logs +... +``` + ### Restoring physical backups If cluster members have to be (re)initialized restoring physical backups diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 1b1ae852e..c180c73cf 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -553,6 +553,12 @@ yet officially supported. [service accounts](https://cloud.google.com/kubernetes-engine/docs/tutorials/authenticating-to-cloud-platform). The default is empty +* **wal_az_storage_account** + Azure Storage Account to use for shipping WAL segments with WAL-G. The + storage account must exist and be accessible by Postgres pods. Note, only the + name of the storage account is required. + The default is empty. + * **log_s3_bucket** S3 bucket to use for shipping Postgres daily logs. Works only with S3 on AWS. The bucket has to be present and accessible by Postgres pods. The default is diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 7a05135ab..a202fdf2c 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -128,6 +128,7 @@ data: # team_api_role_configuration: "log_statement:all" # teams_api_url: http://fake-teams-api.default.svc.cluster.local # toleration: "" + # wal_az_storage_account: "" # wal_gs_bucket: "" # wal_s3_bucket: "" watched_namespace: "*" # listen to all namespaces diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 806acc8da..e25b7e7af 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -384,6 +384,8 @@ spec: type: string log_s3_bucket: type: string + wal_az_storage_account: + type: string wal_gs_bucket: type: string wal_s3_bucket: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index bd6f321dd..76577c35d 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -120,6 +120,7 @@ configuration: # gcp_credentials: "" # kube_iam_role: "" # log_s3_bucket: "" + # wal_az_storage_account: "" # wal_gs_bucket: "" # wal_s3_bucket: "" logical_backup: diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index cf581431b..b6d1d9bdb 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -131,6 +131,7 @@ type AWSGCPConfiguration struct { AWSRegion string `json:"aws_region,omitempty"` WALGSBucket string `json:"wal_gs_bucket,omitempty"` GCPCredentials string `json:"gcp_credentials,omitempty"` + WALAZStorageAccount string `json:"wal_az_storage_account,omitempty"` LogS3Bucket string `json:"log_s3_bucket,omitempty"` KubeIAMRole string `json:"kube_iam_role,omitempty"` AdditionalSecretMount string `json:"additional_secret_mount,omitempty"` diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index c02a64df0..474f3b60c 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -798,6 +798,12 @@ func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration stri envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""}) } + if c.OpConfig.WALAZStorageAccount != "" { + envVars = append(envVars, v1.EnvVar{Name: "AZURE_STORAGE_ACCOUNT", Value: c.OpConfig.WALAZStorageAccount}) + envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) + envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""}) + } + if c.OpConfig.GCPCredentials != "" { envVars = append(envVars, v1.EnvVar{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: c.OpConfig.GCPCredentials}) } @@ -1806,6 +1812,14 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription) }, } result = append(result, envs...) + } else if c.OpConfig.WALAZStorageAccount != "" { + envs := []v1.EnvVar{ + { + Name: "CLONE_AZURE_STORAGE_ACCOUNT", + Value: c.OpConfig.WALAZStorageAccount, + }, + } + result = append(result, envs...) } else { c.logger.Error("Cannot figure out S3 or GS bucket. Both are empty.") } diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 761cf1b60..02396ba23 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -145,6 +145,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.KubeIAMRole = fromCRD.AWSGCP.KubeIAMRole result.WALGSBucket = fromCRD.AWSGCP.WALGSBucket result.GCPCredentials = fromCRD.AWSGCP.GCPCredentials + result.WALAZStorageAccount = fromCRD.AWSGCP.WALAZStorageAccount result.AdditionalSecretMount = fromCRD.AWSGCP.AdditionalSecretMount result.AdditionalSecretMountPath = util.Coalesce(fromCRD.AWSGCP.AdditionalSecretMountPath, "/meta/credentials") result.EnableEBSGp3Migration = fromCRD.AWSGCP.EnableEBSGp3Migration diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 52530fb12..ec622bfff 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -167,6 +167,7 @@ type Config struct { KubeIAMRole string `name:"kube_iam_role"` WALGSBucket string `name:"wal_gs_bucket"` GCPCredentials string `name:"gcp_credentials"` + WALAZStorageAccount string `name:"wal_az_storage_account"` AdditionalSecretMount string `name:"additional_secret_mount"` AdditionalSecretMountPath string `name:"additional_secret_mount_path" default:"/meta/credentials"` EnableEBSGp3Migration bool `name:"enable_ebs_gp3_migration" default:"false"`