From 8780827477d3224343ff6d77efb7f0fddd9bbc66 Mon Sep 17 00:00:00 2001 From: Sudeepta Patra Date: Mon, 11 Jul 2022 14:00:51 +0000 Subject: [PATCH 1/7] modify apis to enable timeline_id parameter in clone section --- pkg/apis/acid.zalan.do/v1/crds.go | 3 +++ pkg/apis/acid.zalan.do/v1/postgresql_type.go | 3 ++- pkg/apis/acid.zalan.do/v1/util_test.go | 8 ++++---- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index e361843be..aab26da68 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -211,6 +211,9 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Type: "string", Format: "uuid", }, + "clone_target_timeline": { + Type: "string", + }, }, }, "connectionPooler": { diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 1e9245fb8..40fca5869 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -197,7 +197,8 @@ type CloneDescription struct { S3Endpoint string `json:"s3_endpoint,omitempty"` S3AccessKeyId string `json:"s3_access_key_id,omitempty"` S3SecretAccessKey string `json:"s3_secret_access_key,omitempty"` - S3ForcePathStyle *bool `json:"s3_force_path_style,omitempty" defaults:"false"` + S3ForcePathStyle *bool `json:"s3_force_path_style,omitempty" defaults:"false"` + TimelineID string `json:"clone_target_timeline" defaults:"latest"` } // Sidecar defines a container to be run in the same pod as the Postgres container. diff --git a/pkg/apis/acid.zalan.do/v1/util_test.go b/pkg/apis/acid.zalan.do/v1/util_test.go index 2ff40d347..f63138bfe 100644 --- a/pkg/apis/acid.zalan.do/v1/util_test.go +++ b/pkg/apis/acid.zalan.do/v1/util_test.go @@ -66,12 +66,12 @@ var cloneClusterDescriptions = []struct { in *CloneDescription err error }{ - {"cluster name invalid but EndTimeSet is not empty", &CloneDescription{"foo+bar", "", "NotEmpty", "", "", "", "", nil}, nil}, - {"expect error as cluster name does not match DNS-1035", &CloneDescription{"foo+bar", "", "", "", "", "", "", nil}, + {"cluster name invalid but EndTimeSet is not empty", &CloneDescription{"foo+bar", "", "NotEmpty", "", "", "", "", nil, 0}, nil}, + {"expect error as cluster name does not match DNS-1035", &CloneDescription{"foo+bar", "", "", "", "", "", "", nil, 0}, errors.New(`clone cluster name must confirm to DNS-1035, regex used for validation is "^[a-z]([-a-z0-9]*[a-z0-9])?$"`)}, - {"expect error as cluster name is too long", &CloneDescription{"foobar123456789012345678901234567890123456789012345678901234567890", "", "", "", "", "", "", nil}, + {"expect error as cluster name is too long", &CloneDescription{"foobar123456789012345678901234567890123456789012345678901234567890", "", "", "", "", "", "", nil, 0}, errors.New("clone cluster name must be no longer than 63 characters")}, - {"common cluster name", &CloneDescription{"foobar", "", "", "", "", "", "", nil}, nil}, + {"common cluster name", &CloneDescription{"foobar", "", "", "", "", "", "", nil, 0}, nil}, } var maintenanceWindows = []struct { From 23270626387b0a913848ba8fa6a6f2555b9b658c Mon Sep 17 00:00:00 2001 From: Sudeepta Patra Date: Mon, 11 Jul 2022 14:04:37 +0000 Subject: [PATCH 2/7] modify cluster pkg to append timelineid --- pkg/cluster/k8sres.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index e84303b1e..10e08d828 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1907,6 +1907,10 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription) result = append(result, v1.EnvVar{Name: "CLONE_AWS_S3_FORCE_PATH_STYLE", Value: s3ForcePathStyle}) } + + if description.TimelineID != "" { + result = append(result, v1.EnvVar{Name: "CLONE_TARGET_TIMELINE", Value: description.TimelineID}) + } } return result From a2236c6c0a92f5109e780c3a74e6a27dd0afcbea Mon Sep 17 00:00:00 2001 From: Sudeepta Patra Date: Sun, 17 Jul 2022 18:03:44 +0000 Subject: [PATCH 3/7] change type from string to int --- pkg/apis/acid.zalan.do/v1/crds.go | 2 +- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 4 ++-- pkg/apis/acid.zalan.do/v1/util_test.go | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index aab26da68..3b07c222d 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -212,7 +212,7 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Format: "uuid", }, "clone_target_timeline": { - Type: "string", + Type: "int32", }, }, }, diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 40fca5869..1004db0f0 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -197,8 +197,8 @@ type CloneDescription struct { S3Endpoint string `json:"s3_endpoint,omitempty"` S3AccessKeyId string `json:"s3_access_key_id,omitempty"` S3SecretAccessKey string `json:"s3_secret_access_key,omitempty"` - S3ForcePathStyle *bool `json:"s3_force_path_style,omitempty" defaults:"false"` - TimelineID string `json:"clone_target_timeline" defaults:"latest"` + S3ForcePathStyle *bool `json:"s3_force_path_style,omitempty" defaults:"false"` + TimelineID int32 `json:"clone_target_timeline" defaults:"latest"` } // Sidecar defines a container to be run in the same pod as the Postgres container. diff --git a/pkg/apis/acid.zalan.do/v1/util_test.go b/pkg/apis/acid.zalan.do/v1/util_test.go index f63138bfe..ebebb36ac 100644 --- a/pkg/apis/acid.zalan.do/v1/util_test.go +++ b/pkg/apis/acid.zalan.do/v1/util_test.go @@ -66,12 +66,12 @@ var cloneClusterDescriptions = []struct { in *CloneDescription err error }{ - {"cluster name invalid but EndTimeSet is not empty", &CloneDescription{"foo+bar", "", "NotEmpty", "", "", "", "", nil, 0}, nil}, - {"expect error as cluster name does not match DNS-1035", &CloneDescription{"foo+bar", "", "", "", "", "", "", nil, 0}, + {"cluster name invalid but EndTimeSet is not empty", &CloneDescription{"foo+bar", "", "NotEmpty", "", "", "", "", nil, ""}, nil}, + {"expect error as cluster name does not match DNS-1035", &CloneDescription{"foo+bar", "", "", "", "", "", "", nil, ""}, errors.New(`clone cluster name must confirm to DNS-1035, regex used for validation is "^[a-z]([-a-z0-9]*[a-z0-9])?$"`)}, - {"expect error as cluster name is too long", &CloneDescription{"foobar123456789012345678901234567890123456789012345678901234567890", "", "", "", "", "", "", nil, 0}, + {"expect error as cluster name is too long", &CloneDescription{"foobar123456789012345678901234567890123456789012345678901234567890", "", "", "", "", "", "", nil, ""}, errors.New("clone cluster name must be no longer than 63 characters")}, - {"common cluster name", &CloneDescription{"foobar", "", "", "", "", "", "", nil, 0}, nil}, + {"common cluster name", &CloneDescription{"foobar", "", "", "", "", "", "", nil, ""}, nil}, } var maintenanceWindows = []struct { From ffbdb9dc2a4b9b2490c5ee63b1acaca570cf13f6 Mon Sep 17 00:00:00 2001 From: Sudeepta Patra Date: Mon, 18 Jul 2022 07:53:13 +0000 Subject: [PATCH 4/7] reverting change f for int --- pkg/apis/acid.zalan.do/v1/util_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/apis/acid.zalan.do/v1/util_test.go b/pkg/apis/acid.zalan.do/v1/util_test.go index ebebb36ac..f63138bfe 100644 --- a/pkg/apis/acid.zalan.do/v1/util_test.go +++ b/pkg/apis/acid.zalan.do/v1/util_test.go @@ -66,12 +66,12 @@ var cloneClusterDescriptions = []struct { in *CloneDescription err error }{ - {"cluster name invalid but EndTimeSet is not empty", &CloneDescription{"foo+bar", "", "NotEmpty", "", "", "", "", nil, ""}, nil}, - {"expect error as cluster name does not match DNS-1035", &CloneDescription{"foo+bar", "", "", "", "", "", "", nil, ""}, + {"cluster name invalid but EndTimeSet is not empty", &CloneDescription{"foo+bar", "", "NotEmpty", "", "", "", "", nil, 0}, nil}, + {"expect error as cluster name does not match DNS-1035", &CloneDescription{"foo+bar", "", "", "", "", "", "", nil, 0}, errors.New(`clone cluster name must confirm to DNS-1035, regex used for validation is "^[a-z]([-a-z0-9]*[a-z0-9])?$"`)}, - {"expect error as cluster name is too long", &CloneDescription{"foobar123456789012345678901234567890123456789012345678901234567890", "", "", "", "", "", "", nil, ""}, + {"expect error as cluster name is too long", &CloneDescription{"foobar123456789012345678901234567890123456789012345678901234567890", "", "", "", "", "", "", nil, 0}, errors.New("clone cluster name must be no longer than 63 characters")}, - {"common cluster name", &CloneDescription{"foobar", "", "", "", "", "", "", nil, ""}, nil}, + {"common cluster name", &CloneDescription{"foobar", "", "", "", "", "", "", nil, 0}, nil}, } var maintenanceWindows = []struct { From 641bb2020a49eae482299e95f648ca9d149e9bb0 Mon Sep 17 00:00:00 2001 From: Sudeepta Patra Date: Mon, 18 Jul 2022 09:56:01 +0000 Subject: [PATCH 5/7] code formatting and change of variable name --- pkg/apis/acid.zalan.do/v1/crds.go | 2 +- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 2 +- pkg/cluster/k8sres.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 3b07c222d..a6b2a3d9a 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -211,7 +211,7 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Type: "string", Format: "uuid", }, - "clone_target_timeline": { + "wal_timeline_id": { Type: "int32", }, }, diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 1004db0f0..d1cb549d3 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -198,7 +198,7 @@ type CloneDescription struct { S3AccessKeyId string `json:"s3_access_key_id,omitempty"` S3SecretAccessKey string `json:"s3_secret_access_key,omitempty"` S3ForcePathStyle *bool `json:"s3_force_path_style,omitempty" defaults:"false"` - TimelineID int32 `json:"clone_target_timeline" defaults:"latest"` + TimelineID int32 `json:"wal_timeline_id" defaults:"latest"` } // Sidecar defines a container to be run in the same pod as the Postgres container. diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 10e08d828..119ab8c17 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1909,7 +1909,7 @@ func (c *Cluster) generateCloneEnvironment(description *acidv1.CloneDescription) } if description.TimelineID != "" { - result = append(result, v1.EnvVar{Name: "CLONE_TARGET_TIMELINE", Value: description.TimelineID}) + result = append(result, v1.EnvVar{Name: "WAL_TIMELINE_ID", Value: description.TimelineID}) } } From baa5b0430209b4d1adb1a3f89aa6cc9e578ea710 Mon Sep 17 00:00:00 2001 From: Sudeepta Patra Date: Mon, 18 Jul 2022 11:45:44 +0000 Subject: [PATCH 6/7] update docs --- .../postgres-operator/crds/postgresqls.yaml | 2 + docs/reference/cluster_manifest.md | 3 ++ go.mod | 41 +------------------ manifests/postgresql.crd.yaml | 2 + 4 files changed, 8 insertions(+), 40 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index d2ad89da6..2976b30d5 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -133,6 +133,8 @@ spec: uid: format: uuid type: string + wal_timeline_id: + type: integer connectionPooler: type: object properties: diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index b41550e22..b82e05bfa 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -372,6 +372,9 @@ under the `clone` top-level key and do not affect the already running cluster. timestamp. When this parameter is set the operator will not consider cloning from the live cluster, even if it is running, and instead goes to S3. Optional. +* **wal_timeline_id** + the timeline ID from which the recovery should take place. Optional. + * **s3_wal_path** the url to S3 bucket containing the WAL archive of the cluster to be cloned. Optional. diff --git a/go.mod b/go.mod index 26e629beb..0841f909e 100644 --- a/go.mod +++ b/go.mod @@ -21,49 +21,10 @@ require ( ) require ( - github.com/PuerkitoBio/purell v1.1.1 // indirect - github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful v2.9.5+incompatible // indirect - github.com/evanphx/json-patch v4.11.0+incompatible // indirect - github.com/go-logr/logr v0.4.0 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.19.5 // indirect - github.com/go-openapi/swag v0.19.14 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/google/go-cmp v0.5.5 // indirect - github.com/google/gofuzz v1.1.0 // indirect - github.com/googleapis/gnostic v0.5.5 // indirect - github.com/imdario/mergo v0.3.5 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/json-iterator/go v1.1.11 // indirect - github.com/kr/text v0.2.0 // indirect - github.com/mailru/easyjson v0.7.6 // indirect - github.com/moby/spdystream v0.2.0 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/objx v0.1.1 // indirect golang.org/x/mod v0.5.1 // indirect - golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 // indirect golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 // indirect - golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect - golang.org/x/text v0.3.6 // indirect - golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect golang.org/x/tools v0.1.7 // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.26.0 // indirect - gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect - k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027 // indirect - k8s.io/klog/v2 v2.9.0 // indirect - k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c // indirect - k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect - sigs.k8s.io/yaml v1.2.0 // indirect ) diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index b113c849f..3039a1af0 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -131,6 +131,8 @@ spec: uid: format: uuid type: string + wal_timeline_id: + type: integer connectionPooler: type: object properties: From 15b693c60d0d8a4c62d672cf60b0a33f42a6144f Mon Sep 17 00:00:00 2001 From: Sudeepta Patra Date: Tue, 19 Jul 2022 09:29:20 +0000 Subject: [PATCH 7/7] remove default from clone struct --- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index d1cb549d3..005a7579d 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -198,7 +198,7 @@ type CloneDescription struct { S3AccessKeyId string `json:"s3_access_key_id,omitempty"` S3SecretAccessKey string `json:"s3_secret_access_key,omitempty"` S3ForcePathStyle *bool `json:"s3_force_path_style,omitempty" defaults:"false"` - TimelineID int32 `json:"wal_timeline_id" defaults:"latest"` + TimelineID int32 `json:"wal_timeline_id,omitempty" } // Sidecar defines a container to be run in the same pod as the Postgres container.