From 351989ae52886b944653fa8de49d27efe000da34 Mon Sep 17 00:00:00 2001 From: Pavel Tumik Date: Wed, 14 Oct 2020 19:02:21 -0700 Subject: [PATCH 1/8] Fix clone from gcs --- pkg/cluster/k8sres.go | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index fef202538..2924c49fa 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1752,11 +1752,27 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription) msg := "Figure out which S3 bucket to use from env" c.logger.Info(msg, description.S3WalPath) + if c.OpConfig.WALES3Bucket != "" { + envs := []v1.EnvVar{ + { + Name: "CLONE_WAL_S3_BUCKET", + Value: c.OpConfig.WALES3Bucket, + }, + } + result = append(result, envs...) + } else if c.OpConfig.WALGSBucket != "" { + envs := []v1.EnvVar{ + { + Name: "CLONE_WAL_GS_BUCKET", + Value: c.OpConfig.WALGSBucket, + }, + } + result = append(result, envs...) + } else { + c.logger.Error("Cannot figure out S3 or GS bucket. Both are empty.") + } + envs := []v1.EnvVar{ - { - Name: "CLONE_WAL_S3_BUCKET", - Value: c.OpConfig.WALES3Bucket, - }, { Name: "CLONE_WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(description.UID), From 266c6e54423098740a6a7d216fcc4a372385fef0 Mon Sep 17 00:00:00 2001 From: Pavel Tumik Date: Wed, 14 Oct 2020 19:10:25 -0700 Subject: [PATCH 2/8] pass google credentials env var if using GS bucket --- pkg/cluster/k8sres.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 2924c49fa..e291dfefa 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1766,6 +1766,10 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription) Name: "CLONE_WAL_GS_BUCKET", Value: c.OpConfig.WALGSBucket, }, + { + Name: "CLONE_GOOGLE_APPLICATION_CREDENTIALS", + Value: c.OpConfig.GCPCredentials, + }, } result = append(result, envs...) } else { From ac4eb350d94a8272dd3c8a0efb49f3d0cf5d20fd Mon Sep 17 00:00:00 2001 From: Pavel Tumik Date: Wed, 14 Oct 2020 20:51:02 -0700 Subject: [PATCH 3/8] remove requirement for timezone as GCS returns timestamp in local time to the region it is in --- charts/postgres-operator/crds/postgresqls.yaml | 7 +++---- manifests/postgresql.crd.yaml | 7 +++---- pkg/apis/acid.zalan.do/v1/crds.go | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 0d444e568..0f7ec7067 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -122,10 +122,9 @@ spec: type: string timestamp: type: string - pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$' - # The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC - # Example: 1996-12-19T16:39:57-08:00 - # Note: this field requires a timezone + pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?.*$' + # The regexp matches the date-time format + # Example: 1996-12-19T16:39:57 uid: format: uuid type: string diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 97b72a8ca..c058c67bb 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -118,10 +118,9 @@ spec: type: string timestamp: type: string - pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$' - # The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC - # Example: 1996-12-19T16:39:57-08:00 - # Note: this field requires a timezone + pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?.*$' + # The regexp matches the date-time format + # Example: 1996-12-19T16:39:57 uid: format: uuid type: string diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 2cfc28856..461709db8 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -169,7 +169,7 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ "timestamp": { Type: "string", Description: "Date-time format that specifies a timezone as an offset relative to UTC e.g. 1996-12-19T16:39:57-08:00", - Pattern: "^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$", + Pattern: "^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\\.[0-9]+)?.*$", }, "uid": { Type: "string", From 1390af0dbbbb848c165580e56056db533233d6c6 Mon Sep 17 00:00:00 2001 From: Pavel Tumik Date: Wed, 14 Oct 2020 20:54:51 -0700 Subject: [PATCH 4/8] Revert "remove requirement for timezone as GCS returns timestamp in local time to the region it is in" This reverts commit ac4eb350d94a8272dd3c8a0efb49f3d0cf5d20fd. --- charts/postgres-operator/crds/postgresqls.yaml | 7 ++++--- manifests/postgresql.crd.yaml | 7 ++++--- pkg/apis/acid.zalan.do/v1/crds.go | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 0f7ec7067..0d444e568 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -122,9 +122,10 @@ spec: type: string timestamp: type: string - pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?.*$' - # The regexp matches the date-time format - # Example: 1996-12-19T16:39:57 + pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$' + # The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC + # Example: 1996-12-19T16:39:57-08:00 + # Note: this field requires a timezone uid: format: uuid type: string diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index c058c67bb..97b72a8ca 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -118,9 +118,10 @@ spec: type: string timestamp: type: string - pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?.*$' - # The regexp matches the date-time format - # Example: 1996-12-19T16:39:57 + pattern: '^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$' + # The regexp matches the date-time format (RFC 3339 Section 5.6) that specifies a timezone as an offset relative to UTC + # Example: 1996-12-19T16:39:57-08:00 + # Note: this field requires a timezone uid: format: uuid type: string diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 461709db8..2cfc28856 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -169,7 +169,7 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ "timestamp": { Type: "string", Description: "Date-time format that specifies a timezone as an offset relative to UTC e.g. 1996-12-19T16:39:57-08:00", - Pattern: "^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\\.[0-9]+)?.*$", + Pattern: "^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\\.[0-9]+)?(([Zz])|([+-]([01][0-9]|2[0-3]):[0-5][0-9]))$", }, "uid": { Type: "string", From 7c50f15d4afa3ac658fda29e6f18f5053a74b978 Mon Sep 17 00:00:00 2001 From: Pavel Tumik Date: Thu, 15 Oct 2020 14:08:05 -0700 Subject: [PATCH 5/8] update GCS documentation --- docs/administrator.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docs/administrator.md b/docs/administrator.md index 1a1b5e8f9..000963434 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -617,6 +617,8 @@ of the backup cron job. `cronjobs` resource from the `batch` API group for the operator service account. See [example RBAC](../manifests/operator-service-account-rbac.yaml) +You cannot use logical backups when you are cloning cluster. For that you need to setup WAL writing into S3 or GCS. + ## Access to cloud resources from clusters in non-cloud environment To access cloud resources like S3 from a cluster on bare metal you can use @@ -685,6 +687,31 @@ aws_or_gcp: ... ``` +### Setup pod environment configmap + +By default postgres-operator is using WAL-E to perform backup and restore. WAL-E doesn't work very well with GCS, so instead we want to use WAL-G. By providing custom pod environment, we can instruct Spilo to use WAL-G for backup and recovery. +```yml +apiVersion: v1 +kind: ConfigMap +metadata: + name: pod-env-overrides + namespace: postgres-operator-system +data: + # Any env variable used by spilo can be added + USE_WALG_BACKUP: "true" + USE_WALG_RESTORE: "true" + CLONE_USE_WALG_RESTORE: "true" +``` + +Then provide this configmap in postgres-operator settings: +```yml +... +# namespaced name of the ConfigMap with environment variables to populate on every pod +pod_environment_configmap: "postgres-operator-system/pod-env-overrides" +... +``` + + ## Sidecars for Postgres clusters A list of sidecars is added to each cluster created by the operator. The default From 1f7bcc75e3561dbb83abb11d99e6d9a7d4e2319e Mon Sep 17 00:00:00 2001 From: Pavel Tumik Date: Tue, 27 Oct 2020 09:50:46 -0700 Subject: [PATCH 6/8] remove sentence about logical backups --- docs/administrator.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/administrator.md b/docs/administrator.md index 000963434..e51b7e3ab 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -617,8 +617,6 @@ of the backup cron job. `cronjobs` resource from the `batch` API group for the operator service account. See [example RBAC](../manifests/operator-service-account-rbac.yaml) -You cannot use logical backups when you are cloning cluster. For that you need to setup WAL writing into S3 or GCS. - ## Access to cloud resources from clusters in non-cloud environment To access cloud resources like S3 from a cluster on bare metal you can use From a874151f791cb438b773c6b3de9bd59d18e083dc Mon Sep 17 00:00:00 2001 From: Pavel Tumik Date: Thu, 29 Oct 2020 07:30:42 -0700 Subject: [PATCH 7/8] reword pod environment configmap section --- docs/administrator.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/administrator.md b/docs/administrator.md index e51b7e3ab..652d1fe40 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -687,7 +687,7 @@ aws_or_gcp: ### Setup pod environment configmap -By default postgres-operator is using WAL-E to perform backup and restore. WAL-E doesn't work very well with GCS, so instead we want to use WAL-G. By providing custom pod environment, we can instruct Spilo to use WAL-G for backup and recovery. +To make postgres-operator work with GCS, use following configmap: ```yml apiVersion: v1 kind: ConfigMap @@ -700,6 +700,7 @@ data: USE_WALG_RESTORE: "true" CLONE_USE_WALG_RESTORE: "true" ``` +This configmap will instruct operator to use WAL-G, instead of WAL-E, that is better suited for GCS. Then provide this configmap in postgres-operator settings: ```yml From 79e73dd07418f8447967fa23134020a5ae1d0607 Mon Sep 17 00:00:00 2001 From: Pavel Tumik Date: Thu, 29 Oct 2020 11:41:33 -0700 Subject: [PATCH 8/8] fix documentation --- docs/administrator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/administrator.md b/docs/administrator.md index 652d1fe40..eb901a3eb 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -700,7 +700,7 @@ data: USE_WALG_RESTORE: "true" CLONE_USE_WALG_RESTORE: "true" ``` -This configmap will instruct operator to use WAL-G, instead of WAL-E, that is better suited for GCS. +This configmap will instruct operator to use WAL-G, instead of WAL-E, for backup and restore. Then provide this configmap in postgres-operator settings: ```yml