From 1bd12961fd4a9356d10dff94ceab18396fd42680 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 9 Jul 2021 19:11:28 +0200 Subject: [PATCH 01/44] provide event stream API completed codegen manually provide update and compare code for fes add db check for specified databases and tables add unit test --- hack/update-codegen.sh | 23 +- manifests/operator-service-account-rbac.yaml | 14 ++ pkg/apis/acid.zalan.do/v1/crds.go | 47 ++++ pkg/apis/acid.zalan.do/v1/postgresql_type.go | 12 + .../acid.zalan.do/v1/zz_generated.deepcopy.go | 37 +++ pkg/apis/zalando.org/register.go | 6 + .../zalando.org/v1alpha1/fabriceventstream.go | 88 +++++++ pkg/apis/zalando.org/v1alpha1/register.go | 46 ++++ .../v1alpha1/zz_generated.deepcopy.go | 227 ++++++++++++++++++ pkg/cluster/cluster.go | 9 + pkg/cluster/database.go | 38 +++ pkg/cluster/streams.go | 196 +++++++++++++++ pkg/cluster/streams_test.go | 105 ++++++++ .../clientset/versioned/clientset.go | 13 +- .../versioned/fake/clientset_generated.go | 7 + .../clientset/versioned/fake/register.go | 2 + .../clientset/versioned/scheme/register.go | 2 + .../typed/zalando.org/v1alpha1/doc.go | 26 ++ .../zalando.org/v1alpha1/fabriceventstream.go | 184 ++++++++++++++ .../typed/zalando.org/v1alpha1/fake/doc.go | 26 ++ .../v1alpha1/fake/fake_fabriceventstream.go | 136 +++++++++++ .../v1alpha1/fake/fake_zalando.org_client.go | 46 ++++ .../v1alpha1/generated_expansion.go | 27 +++ .../v1alpha1/zalando.org_client.go | 95 ++++++++ .../informers/externalversions/factory.go | 6 + .../informers/externalversions/generic.go | 5 + .../externalversions/zalando.org/interface.go | 52 ++++ .../zalando.org/v1alpha1/fabriceventstream.go | 96 ++++++++ .../zalando.org/v1alpha1/interface.go | 51 ++++ .../v1alpha1/expansion_generated.go | 33 +++ .../zalando.org/v1alpha1/fabriceventstream.go | 105 ++++++++ pkg/util/constants/streams.go | 17 ++ pkg/util/k8sutil/k8sutil.go | 16 +- 33 files changed, 1779 insertions(+), 14 deletions(-) create mode 100644 pkg/apis/zalando.org/register.go create mode 100644 pkg/apis/zalando.org/v1alpha1/fabriceventstream.go create mode 100644 pkg/apis/zalando.org/v1alpha1/register.go create mode 100644 pkg/apis/zalando.org/v1alpha1/zz_generated.deepcopy.go create mode 100644 pkg/cluster/streams.go create mode 100644 pkg/cluster/streams_test.go create mode 100644 pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/doc.go create mode 100644 pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fabriceventstream.go create mode 100644 pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fake/doc.go create mode 100644 pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fake/fake_fabriceventstream.go create mode 100644 pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fake/fake_zalando.org_client.go create mode 100644 pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/generated_expansion.go create mode 100644 pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/zalando.org_client.go create mode 100644 pkg/generated/informers/externalversions/zalando.org/interface.go create mode 100644 pkg/generated/informers/externalversions/zalando.org/v1alpha1/fabriceventstream.go create mode 100644 pkg/generated/informers/externalversions/zalando.org/v1alpha1/interface.go create mode 100644 pkg/generated/listers/zalando.org/v1alpha1/expansion_generated.go create mode 100644 pkg/generated/listers/zalando.org/v1alpha1/fabriceventstream.go create mode 100644 pkg/util/constants/streams.go diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh index 280da9385..18c71f83c 100755 --- a/hack/update-codegen.sh +++ b/hack/update-codegen.sh @@ -1,13 +1,18 @@ #!/usr/bin/env bash +set -eou pipefail -set -o errexit -set -o nounset -set -o pipefail +GOPKG="github.com/zalando/postgres-operator" +SCRIPT_ROOT="$(dirname "${BASH_SOURCE[0]}")/.." -SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. -CODEGEN_PKG=${CODEGEN_PKG:-$(cd ${SCRIPT_ROOT}; ls -d -1 ./vendor/k8s.io/code-generator 2>/dev/null || echo ${GOPATH}/src/k8s.io/code-generator)} +rm -rf "${SCRIPT_ROOT}/generated" -bash "${CODEGEN_PKG}/generate-groups.sh" all \ - github.com/zalando/postgres-operator/pkg/generated github.com/zalando/postgres-operator/pkg/apis \ - "acid.zalan.do:v1" \ - --go-header-file "${SCRIPT_ROOT}"/hack/custom-boilerplate.go.txt +go run k8s.io/code-generator/cmd/deepcopy-gen \ + --input-dirs ${GOPKG}/pkg/apis/acid.zalan.do/v1,${GOPKG}/pkg/apis/zalando.org/v1alpha1 \ + -O zz_generated.deepcopy \ + --bounding-dirs ${GOPKG}/pkg/apis \ + --go-header-file "${SCRIPT_ROOT}/hack/custom-boilerplate.go.txt" \ + -o "${SCRIPT_ROOT}/generated" + +cp -rv "${SCRIPT_ROOT}/generated/${GOPKG}"/* . + +rm -rf "${SCRIPT_ROOT}/generated" \ No newline at end of file diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index f0307f6a0..48da5f06a 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -35,6 +35,20 @@ rules: - get - list - watch +# all verbs allowed for event streams +- apiGroups: + - zalando.org + resources: + - fabriceventstream + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch # to create or get/update CRDs when starting up - apiGroups: - apiextensions.k8s.io diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index a95eeab20..105e14cd7 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -657,6 +657,53 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ }, }, }, + "streams": { + Type: "array", + Items: &apiextv1.JSONSchemaPropsOrArray{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "object", + Required: []string{"type"}, + Properties: map[string]apiextv1.JSONSchemaProps{ + "database": { + Type: "string", + }, + "filter": { + Type: "object", + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "sqsArn": { + Type: "string", + }, + "tables": { + Type: "object", + AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ + Schema: &apiextv1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + "type": { + Type: "string", + Enum: []apiextv1.JSON{ + { + Raw: []byte(`"nakadi"`), + }, + { + Raw: []byte(`"sqs"`), + }, + { + Raw: []byte(`"wal"`), + }, + }, + }, + }, + }, + }, + }, "teamId": { Type: "string", }, diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 7346fb0e5..02eaf6ee6 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -74,6 +74,7 @@ type PostgresSpec struct { ServiceAnnotations map[string]string `json:"serviceAnnotations,omitempty"` TLS *TLSDescription `json:"tls,omitempty"` AdditionalVolumes []AdditionalVolume `json:"additionalVolumes,omitempty"` + Streams []Stream `json:"stream,omitempty"` // deprecated json tags InitContainersOld []v1.Container `json:"init_containers,omitempty"` @@ -224,3 +225,14 @@ type ConnectionPooler struct { Resources `json:"resources,omitempty"` } + +type Stream struct { + Type string `json:"type"` + Database string `json:"database,omitempty"` + Tables map[string]string `json:"tables,omitempty"` + Filter map[string]string `json:"filter,omitempty"` + BatchSize uint32 `json:"batchSize,omitempty"` + SqsArn string `json:"sqsArn,omitempty"` + QueueName string `json:"queueName,omitempty"` + User string `json:"user,omitempty"` +} diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 584a72143..a8be4e143 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -722,6 +722,13 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Streams != nil { + in, out := &in.Streams, &out.Streams + *out = make([]Stream, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.InitContainersOld != nil { in, out := &in.InitContainersOld, &out.InitContainersOld *out = make([]corev1.Container, len(*in)) @@ -1125,6 +1132,36 @@ func (in *StandbyDescription) DeepCopy() *StandbyDescription { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Stream) DeepCopyInto(out *Stream) { + *out = *in + if in.Tables != nil { + in, out := &in.Tables, &out.Tables + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Filter != nil { + in, out := &in.Filter, &out.Filter + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Stream. +func (in *Stream) DeepCopy() *Stream { + if in == nil { + return nil + } + out := new(Stream) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TLSDescription) DeepCopyInto(out *TLSDescription) { *out = *in diff --git a/pkg/apis/zalando.org/register.go b/pkg/apis/zalando.org/register.go new file mode 100644 index 000000000..3dbd3f089 --- /dev/null +++ b/pkg/apis/zalando.org/register.go @@ -0,0 +1,6 @@ +package zalando + +const ( + // GroupName is the group name for the operator CRDs + GroupName = "zalando.org" +) diff --git a/pkg/apis/zalando.org/v1alpha1/fabriceventstream.go b/pkg/apis/zalando.org/v1alpha1/fabriceventstream.go new file mode 100644 index 000000000..37f1449c9 --- /dev/null +++ b/pkg/apis/zalando.org/v1alpha1/fabriceventstream.go @@ -0,0 +1,88 @@ +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// FabricEventStream defines FabricEventStream Custom Resource Definition Object. +type FabricEventStream struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec FabricEventStreamSpec `json:"spec"` +} + +// FabricEventStreamSpec defines the specification for the FabricEventStream TPR. +type FabricEventStreamSpec struct { + ApplicationId string `json:"applicationId"` + EventStreams []EventStream `json:"eventStreams"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// FabricEventStreamList defines a list of FabricEventStreams . +type FabricEventStreamList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []FabricEventStream `json:"items"` +} + +// EventStream defines the source, flow and sink of the event stream +type EventStream struct { + EventStreamFlow EventStreamFlow `json:"flow"` + EventStreamSink EventStreamSink `json:"sink"` + EventStreamSource EventStreamSource `json:"source"` +} + +// EventStreamFlow defines the flow characteristics of the event stream +type EventStreamFlow struct { + Type string `json:"type"` + DataTypeColumn string `json:"dataTypeColumn,omitempty"` + DataOpColumn string `json:"dataOpColumn,omitempty"` + MetadataColumn string `json:"metadataColumn,omitempty"` + DataColumn string `json:"dataColumn,omitempty"` + CallHomeIdColumn string `json:"callHomeIdColumn,omitempty"` + CallHomeUrl string `json:"callHomeUrl,omitempty"` +} + +// EventStreamSink defines the target of the event stream +type EventStreamSink struct { + Type string `json:"type"` + EventType string `json:"eventType,omitempty"` + MaxBatchSize uint32 `json:"maxBatchSize,omitempty"` + QueueName string `json:"queueName,omitempty"` +} + +// EventStreamSource defines the source of the event stream and connection for FES operator +type EventStreamSource struct { + Type string `json:"type"` + Schema string `json:"schema,omitempty" defaults:"public"` + EventStreamTable EventStreamTable `json:"table"` + Filter string `json:"filter,omitempty"` + Connection Connection `json:"jdbcConnection"` +} + +// EventStreamTable defines the name and ID column to be used for streaming +type EventStreamTable struct { + Name string `json:"name"` + IDColumn string `json:"idColumn,omitempty" defaults:"id"` +} + +// Connection to be used for allowing the FES operator to connect to a database +type Connection struct { + Url string `json:"jdbcUrl"` + SlotName string `json:"slotName"` + DBAuth DBAuth `json:"databaseAuthentication"` +} + +// DBAuth specifies the credentials to be used for connecting with the database +type DBAuth struct { + Type string `json:"type"` + Name string `json:"name,omitempty"` + UserKey string `json:"userKey,omitempty"` + PasswordKey string `json:"passwordKey,omitempty"` +} diff --git a/pkg/apis/zalando.org/v1alpha1/register.go b/pkg/apis/zalando.org/v1alpha1/register.go new file mode 100644 index 000000000..07732ae53 --- /dev/null +++ b/pkg/apis/zalando.org/v1alpha1/register.go @@ -0,0 +1,46 @@ +package v1alpha1 + +import ( + "github.com/zalando/postgres-operator/pkg/apis/zalando.org" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/scheme" +) + +// APIVersion of the `fabriceventstream` CRD +const ( + APIVersion = "v1alpha1" +) + +var ( + schemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + // AddToScheme applies all the stored functions to the scheme. A non-nil error + // indicates that one function failed and the attempt was abandoned. + AddToScheme = schemeBuilder.AddToScheme +) + +func init() { + err := AddToScheme(scheme.Scheme) + if err != nil { + panic(err) + } +} + +// SchemeGroupVersion is the group version used to register these objects. +var SchemeGroupVersion = schema.GroupVersion{Group: zalando.GroupName, Version: APIVersion} + +// Resource takes an unqualified resource and returns a Group-qualified GroupResource. +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} + +// addKnownTypes adds the set of types defined in this package to the supplied scheme. +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &FabricEventStream{}, + &FabricEventStreamList{}, + ) + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + return nil +} diff --git a/pkg/apis/zalando.org/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/zalando.org/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 000000000..0327279e5 --- /dev/null +++ b/pkg/apis/zalando.org/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,227 @@ +// +build !ignore_autogenerated + +/* +Copyright 2021 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Connection) DeepCopyInto(out *Connection) { + *out = *in + return +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Connection) DeepCopy() *Connection { + if in == nil { + return nil + } + out := new(Connection) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DBAuth) DeepCopyInto(out *DBAuth) { + *out = *in + return +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DBAuth) DeepCopy() *DBAuth { + if in == nil { + return nil + } + out := new(DBAuth) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStream) DeepCopyInto(out *EventStream) { + *out = *in + return +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStream) DeepCopy() *EventStream { + if in == nil { + return nil + } + out := new(EventStream) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStreamFlow) DeepCopyInto(out *EventStreamFlow) { + *out = *in + return +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStreamFlow) DeepCopy() *EventStreamFlow { + if in == nil { + return nil + } + out := new(EventStreamFlow) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStreamSink) DeepCopyInto(out *EventStreamSink) { + *out = *in + return +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStreamSink) DeepCopy() *EventStreamSink { + if in == nil { + return nil + } + out := new(EventStreamSink) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStreamSource) DeepCopyInto(out *EventStreamSource) { + *out = *in + return +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStreamSource) DeepCopy() *EventStreamSource { + if in == nil { + return nil + } + out := new(EventStreamSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStreamTable) DeepCopyInto(out *EventStreamTable) { + *out = *in + return +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventStreamTable) DeepCopy() *EventStreamTable { + if in == nil { + return nil + } + out := new(EventStreamTable) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FabricEventStreamSpec) DeepCopyInto(out *FabricEventStreamSpec) { + *out = *in + if in.EventStreams != nil { + in, out := &in.EventStreams, &out.EventStreams + *out = make([]EventStream, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FabricEventStreamSpec. +func (in *FabricEventStreamSpec) DeepCopy() *FabricEventStreamSpec { + if in == nil { + return nil + } + out := new(FabricEventStreamSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FabricEventStream) DeepCopyInto(out *FabricEventStream) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FabricEventStream. +func (in *FabricEventStream) DeepCopy() *FabricEventStream { + if in == nil { + return nil + } + out := new(FabricEventStream) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *FabricEventStream) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FabricEventStreamList) DeepCopyInto(out *FabricEventStreamList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]FabricEventStream, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FabricEventStreamList. +func (in *FabricEventStreamList) DeepCopy() *FabricEventStreamList { + if in == nil { + return nil + } + out := new(FabricEventStreamList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *FabricEventStreamList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index c9abb10fd..3165534e8 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -361,6 +361,10 @@ func (c *Cluster) Create() error { // something fails, report warning c.createConnectionPooler(c.installLookupFunction) + if len(c.Spec.Streams) > 0 { + c.createStreams() + } + return nil } @@ -855,6 +859,11 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { updateFailed = true } + if err := c.syncStreams(); err != nil { + c.logger.Errorf("could not sync streams: %v", err) + updateFailed = true + } + if !updateFailed { // Major version upgrade must only fire after success of earlier operations and should stay last if err := c.majorVersionUpgrade(); err != nil { diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index ba4cf223a..6e0a8ae30 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -33,6 +33,8 @@ const ( getExtensionsSQL = `SELECT e.extname, n.nspname FROM pg_catalog.pg_extension e LEFT JOIN pg_catalog.pg_namespace n ON n.oid = e.extnamespace ORDER BY 1;` + tableExistsSQL = `SELECT TRUE FROM pg_tables WHERE tablename = $1 AND schemaname = $2;` + createDatabaseSQL = `CREATE DATABASE "%s" OWNER "%s";` createDatabaseSchemaSQL = `SET ROLE TO "%s"; CREATE SCHEMA IF NOT EXISTS "%s" AUTHORIZATION "%s"` alterDatabaseOwnerSQL = `ALTER DATABASE "%s" OWNER TO "%s";` @@ -506,6 +508,42 @@ func (c *Cluster) execCreateOrAlterExtension(extName, schemaName, statement, doi return nil } +// getExtension returns the list of current database extensions +// The caller is responsible for opening and closing the database connection +func (c *Cluster) tableExists(tableName, schemaName string) (bool, error) { + var ( + rows *sql.Rows + exists bool + err error + ) + + if rows, err = c.pgDb.Query(tableExistsSQL, tableName, schemaName); err != nil { + return false, fmt.Errorf("could not check table for existence: %v", err) + } + + defer func() { + if err2 := rows.Close(); err2 != nil { + if err != nil { + err = fmt.Errorf("error when closing query cursor: %v, previous error: %v", err2, err) + } else { + err = fmt.Errorf("error when closing query cursor: %v", err2) + } + } + }() + + for rows.Next() { + if err = rows.Scan(&exists); err != nil { + return false, fmt.Errorf("error when processing row: %v", err) + } + } + + if exists { + return true, nil + } else { + return false, fmt.Errorf("table %s not found", schemaName+"."+tableName) + } +} + // Creates a connection pool credentials lookup function in every database to // perform remote authentication. func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string, role PostgresRole) error { diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go new file mode 100644 index 000000000..6f1b7efec --- /dev/null +++ b/pkg/cluster/streams.go @@ -0,0 +1,196 @@ +package cluster + +import ( + "context" + "fmt" + "reflect" + "strings" + + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + zalandov1alpha1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1alpha1" + "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/constants" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var outboxTableNameTemplate config.StringTemplate = "{table}_{eventtype}_outbox" + +func (c *Cluster) createStreams() error { + c.setProcessName("creating streams") + + fes := c.generateFabricEventStream() + _, err := c.KubeClient.FabricEventStreamsGetter.FabricEventStreams(c.Namespace).Create(context.TODO(), fes, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("could not create event stream custom resource: %v", err) + } + + return nil +} + +func (c *Cluster) syncStreamDbResources() error { + + for _, stream := range c.Spec.Streams { + if err := c.initDbConnWithName(stream.Database); err != nil { + return fmt.Errorf("could not init connection to database %s specified for event stream: %v", stream.Database, err) + } + + for table, eventType := range stream.Tables { + tableName, schemaName := getTableSchema(table) + if exists, err := c.tableExists(tableName, schemaName); !exists { + return fmt.Errorf("could not find table %s specified for event stream: %v", table, err) + } + // check if outbox table exists and if not, create it + outboxTable := outboxTableNameTemplate.Format("table", tableName, "eventtype", eventType) + if exists, err := c.tableExists(outboxTable, schemaName); !exists { + return fmt.Errorf("could not find outbox table %s specified for event stream: %v", outboxTable, err) + } + } + } + + return nil +} + +func (c *Cluster) generateFabricEventStream() *zalandov1alpha1.FabricEventStream { + eventStreams := make([]zalandov1alpha1.EventStream, 0) + + for _, stream := range c.Spec.Streams { + for table, eventType := range stream.Tables { + streamSource := c.getEventStreamSource(stream, table, eventType) + streamFlow := getEventStreamFlow(stream) + streamSink := getEventStreamSink(stream, eventType) + + eventStreams = append(eventStreams, zalandov1alpha1.EventStream{ + EventStreamFlow: streamFlow, + EventStreamSink: streamSink, + EventStreamSource: streamSource}) + } + } + + return &zalandov1alpha1.FabricEventStream{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.Name + constants.FESsuffix, + Namespace: c.Namespace, + Annotations: c.AnnotationsToPropagate(c.annotationsSet(nil)), + }, + Spec: zalandov1alpha1.FabricEventStreamSpec{ + ApplicationId: "", + EventStreams: eventStreams, + }, + } +} + +func (c *Cluster) getEventStreamSource(stream acidv1.Stream, table, eventType string) zalandov1alpha1.EventStreamSource { + streamFilter := stream.Filter[table] + _, schema := getTableSchema(table) + return zalandov1alpha1.EventStreamSource{ + Type: constants.EventStreamSourcePGType, + Schema: schema, + EventStreamTable: getOutboxTable(table, eventType), + Filter: streamFilter, + Connection: c.getStreamConnection(stream.Database, stream.User), + } +} + +func getEventStreamFlow(stream acidv1.Stream) zalandov1alpha1.EventStreamFlow { + switch stream.Type { + case "nakadi": + return zalandov1alpha1.EventStreamFlow{ + Type: constants.EventStreamFlowPgNakadiType, + DataTypeColumn: constants.EventStreamFlowDataTypeColumn, + DataOpColumn: constants.EventStreamFlowDataOpColumn, + MetadataColumn: constants.EventStreamFlowMetadataColumn, + DataColumn: constants.EventStreamFlowDataColumn} + case "sqs": + return zalandov1alpha1.EventStreamFlow{ + Type: constants.EventStreamFlowPgApiType, + CallHomeIdColumn: "id", + CallHomeUrl: stream.SqsArn} + } + + return zalandov1alpha1.EventStreamFlow{} +} + +func getEventStreamSink(stream acidv1.Stream, eventType string) zalandov1alpha1.EventStreamSink { + switch stream.Type { + case "nakadi": + return zalandov1alpha1.EventStreamSink{ + Type: constants.EventStreamSinkNakadiType, + EventType: eventType, + MaxBatchSize: stream.BatchSize} + case "sqs": + return zalandov1alpha1.EventStreamSink{ + Type: constants.EventStreamSinkSqsType, + QueueName: stream.QueueName} + } + + return zalandov1alpha1.EventStreamSink{} +} + +func getTableSchema(fullTableName string) (tableName, schemaName string) { + schemaName = "public" + tableName = fullTableName + if strings.Contains(fullTableName, ".") { + schemaName = strings.Split(fullTableName, ".")[0] + tableName = strings.Split(fullTableName, ".")[1] + } + + return tableName, schemaName +} + +func getOutboxTable(tableName, eventType string) zalandov1alpha1.EventStreamTable { + return zalandov1alpha1.EventStreamTable{ + Name: outboxTableNameTemplate.Format("table", tableName, "eventtype", eventType), + IDColumn: "id", + } +} + +func (c *Cluster) getStreamConnection(database, user string) zalandov1alpha1.Connection { + return zalandov1alpha1.Connection{ + Url: fmt.Sprintf("jdbc:postgresql://%s.%s/%s?user=%s&ssl=true&sslmode=require", c.Name, c.Namespace, database, user), + SlotName: constants.EventStreamSourceSlotName, + DBAuth: zalandov1alpha1.DBAuth{ + Type: constants.EventStreamSourceAuthType, + Name: c.credentialSecretNameForCluster(user, c.ClusterName), + UserKey: "username", + PasswordKey: "password", + }, + } +} + +func (c *Cluster) syncStreams() error { + + c.setProcessName("syncing streams") + + effectiveStreams, err := c.KubeClient.FabricEventStreamsGetter.FabricEventStreams(c.Namespace).Get(context.TODO(), c.Name+constants.FESsuffix, metav1.GetOptions{}) + if err != nil { + if !k8sutil.ResourceNotFound(err) { + return fmt.Errorf("error during reading of event streams: %v", err) + } + + c.logger.Infof("event streams do not exist") + err := c.createStreams() + if err != nil { + return fmt.Errorf("could not create missing streams: %v", err) + } + } else { + c.syncStreamDbResources() + desiredStreams := c.generateFabricEventStream() + if reflect.DeepEqual(effectiveStreams.Spec, desiredStreams.Spec) { + c.updateStreams(desiredStreams) + } + } + + return nil +} + +func (c *Cluster) updateStreams(newEventStreams *zalandov1alpha1.FabricEventStream) error { + c.setProcessName("updating event streams") + + _, err := c.KubeClient.FabricEventStreamsGetter.FabricEventStreams(c.Namespace).Update(context.TODO(), newEventStreams, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("could not update event stream custom resource: %v", err) + } + + return nil +} diff --git a/pkg/cluster/streams_test.go b/pkg/cluster/streams_test.go new file mode 100644 index 000000000..aebd29ff6 --- /dev/null +++ b/pkg/cluster/streams_test.go @@ -0,0 +1,105 @@ +package cluster + +import ( + "reflect" + + "context" + "testing" + + "github.com/stretchr/testify/assert" + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + fakezalandov1alpha1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake" + "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/constants" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +func newFakeK8sStreamClient() (k8sutil.KubernetesClient, *fake.Clientset) { + zalandoClientSet := fakezalandov1alpha1.NewSimpleClientset() + clientSet := fake.NewSimpleClientset() + + return k8sutil.KubernetesClient{ + FabricEventStreamsGetter: zalandoClientSet.ZalandoV1alpha1(), + }, clientSet +} + +func TestGenerateFabricEventStream(t *testing.T) { + client, _ := newFakeK8sStreamClient() + clusterName := "acid-test-cluster" + namespace := "default" + + pg := acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: acidv1.PostgresSpec{ + Databases: map[string]string{ + "foo": "foo_user", + }, + Streams: []acidv1.Stream{ + { + Type: "nakadi", + Database: "foo", + Tables: map[string]string{ + "bar": "stream_type_a", + }, + BatchSize: uint32(100), + User: "foo_user", + }, + { + Type: "wal", + Database: "foo", + Tables: map[string]string{ + "bar": "stream_type_a", + }, + BatchSize: uint32(100), + User: "zalando", + }, + { + Type: "sqs", + Database: "foo", + SqsArn: "arn:aws:sqs:eu-central-1:111122223333", + QueueName: "foo-queue", + User: "foo_user", + }, + }, + Users: map[string]acidv1.UserFlags{ + "foo_user": {}, + "zalando": {}, + }, + Volume: acidv1.Volume{ + Size: "1Gi", + }, + }, + } + + var cluster = New( + Config{ + OpConfig: config.Config{ + PodManagementPolicy: "ordered_ready", + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + DefaultCPURequest: "300m", + DefaultCPULimit: "300m", + DefaultMemoryRequest: "300Mi", + DefaultMemoryLimit: "300Mi", + PodRoleLabel: "spilo-role", + }, + }, + }, client, pg, logger, eventRecorder) + + cluster.syncStreams() + + streamCRD, err := cluster.KubeClient.FabricEventStreams(namespace).Get(context.TODO(), cluster.Name+constants.FESsuffix, metav1.GetOptions{}) + assert.NoError(t, err) + + result := cluster.generateFabricEventStream() + if !reflect.DeepEqual(result, streamCRD) { + t.Errorf("Malformed FabricEventStream, expected %#v, got %#v", streamCRD, result) + } +} diff --git a/pkg/generated/clientset/versioned/clientset.go b/pkg/generated/clientset/versioned/clientset.go index ab4a88735..73a3863db 100644 --- a/pkg/generated/clientset/versioned/clientset.go +++ b/pkg/generated/clientset/versioned/clientset.go @@ -28,6 +28,7 @@ import ( "fmt" acidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" + zalandov1alpha1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1" discovery "k8s.io/client-go/discovery" rest "k8s.io/client-go/rest" flowcontrol "k8s.io/client-go/util/flowcontrol" @@ -36,13 +37,15 @@ import ( type Interface interface { Discovery() discovery.DiscoveryInterface AcidV1() acidv1.AcidV1Interface + ZalandoV1alpha1() zalandov1alpha1.ZalandoV1alpha1Interface } // Clientset contains the clients for groups. Each group has exactly one // version included in a Clientset. type Clientset struct { *discovery.DiscoveryClient - acidV1 *acidv1.AcidV1Client + acidV1 *acidv1.AcidV1Client + zalandoV1alpha1 *zalandov1alpha1.ZalandoV1alpha1Client } // AcidV1 retrieves the AcidV1Client @@ -50,6 +53,11 @@ func (c *Clientset) AcidV1() acidv1.AcidV1Interface { return c.acidV1 } +// ZalandoV1alpha1 retrieves the ZalandoV1alpha1Client +func (c *Clientset) ZalandoV1alpha1() zalandov1alpha1.ZalandoV1alpha1Interface { + return c.zalandoV1alpha1 +} + // Discovery retrieves the DiscoveryClient func (c *Clientset) Discovery() discovery.DiscoveryInterface { if c == nil { @@ -72,6 +80,7 @@ func NewForConfig(c *rest.Config) (*Clientset, error) { var cs Clientset var err error cs.acidV1, err = acidv1.NewForConfig(&configShallowCopy) + cs.zalandoV1alpha1, err = zalandov1alpha1.NewForConfig(&configShallowCopy) if err != nil { return nil, err } @@ -88,6 +97,7 @@ func NewForConfig(c *rest.Config) (*Clientset, error) { func NewForConfigOrDie(c *rest.Config) *Clientset { var cs Clientset cs.acidV1 = acidv1.NewForConfigOrDie(c) + cs.zalandoV1alpha1 = zalandov1alpha1.NewForConfigOrDie(c) cs.DiscoveryClient = discovery.NewDiscoveryClientForConfigOrDie(c) return &cs @@ -97,6 +107,7 @@ func NewForConfigOrDie(c *rest.Config) *Clientset { func New(c rest.Interface) *Clientset { var cs Clientset cs.acidV1 = acidv1.New(c) + cs.zalandoV1alpha1 = zalandov1alpha1.New(c) cs.DiscoveryClient = discovery.NewDiscoveryClient(c) return &cs diff --git a/pkg/generated/clientset/versioned/fake/clientset_generated.go b/pkg/generated/clientset/versioned/fake/clientset_generated.go index 6ae5db2d3..3e38777fc 100644 --- a/pkg/generated/clientset/versioned/fake/clientset_generated.go +++ b/pkg/generated/clientset/versioned/fake/clientset_generated.go @@ -28,6 +28,8 @@ import ( clientset "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned" acidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" fakeacidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake" + zalandov1alpha1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1" + fakezalandov1alpha1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fake" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/discovery" @@ -86,3 +88,8 @@ var _ clientset.Interface = &Clientset{} func (c *Clientset) AcidV1() acidv1.AcidV1Interface { return &fakeacidv1.FakeAcidV1{Fake: &c.Fake} } + +// ZalandoV1alpha1 retrieves the ZalandoV1alpha1Client +func (c *Clientset) ZalandoV1alpha1() zalandov1alpha1.ZalandoV1alpha1Interface { + return &fakezalandov1alpha1.FakeZalandoV1alpha1{Fake: &c.Fake} +} diff --git a/pkg/generated/clientset/versioned/fake/register.go b/pkg/generated/clientset/versioned/fake/register.go index c4d383aab..b022f3e76 100644 --- a/pkg/generated/clientset/versioned/fake/register.go +++ b/pkg/generated/clientset/versioned/fake/register.go @@ -26,6 +26,7 @@ package fake import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + zalandov1alpha1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -38,6 +39,7 @@ var codecs = serializer.NewCodecFactory(scheme) var localSchemeBuilder = runtime.SchemeBuilder{ acidv1.AddToScheme, + zalandov1alpha1.AddToScheme, } // AddToScheme adds all types of this clientset into the given scheme. This allows composition diff --git a/pkg/generated/clientset/versioned/scheme/register.go b/pkg/generated/clientset/versioned/scheme/register.go index 8be969eb5..5fc5886ea 100644 --- a/pkg/generated/clientset/versioned/scheme/register.go +++ b/pkg/generated/clientset/versioned/scheme/register.go @@ -26,6 +26,7 @@ package scheme import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + zalandov1alpha1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -38,6 +39,7 @@ var Codecs = serializer.NewCodecFactory(Scheme) var ParameterCodec = runtime.NewParameterCodec(Scheme) var localSchemeBuilder = runtime.SchemeBuilder{ acidv1.AddToScheme, + zalandov1alpha1.AddToScheme, } // AddToScheme adds all types of this clientset into the given scheme. This allows composition diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/doc.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/doc.go new file mode 100644 index 000000000..a9896a348 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/doc.go @@ -0,0 +1,26 @@ +/* +Copyright 2021 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated typed clients. +package v1alpha1 diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fabriceventstream.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fabriceventstream.go new file mode 100644 index 000000000..ec93fe65a --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fabriceventstream.go @@ -0,0 +1,184 @@ +/* +Copyright 2021 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + "time" + + v1alpha1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1alpha1" + scheme "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// FabricEventStreamsGetter has a method to return a FabricEventStreamInterface. +// A group's client should implement this interface. +type FabricEventStreamsGetter interface { + FabricEventStreams(namespace string) FabricEventStreamInterface +} + +// FabricEventStreamInterface has methods to work with FabricEventStream resources. +type FabricEventStreamInterface interface { + Create(ctx context.Context, fabricEventStream *v1alpha1.FabricEventStream, opts v1.CreateOptions) (*v1alpha1.FabricEventStream, error) + Update(ctx context.Context, fabricEventStream *v1alpha1.FabricEventStream, opts v1.UpdateOptions) (*v1alpha1.FabricEventStream, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.FabricEventStream, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.FabricEventStreamList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.FabricEventStream, err error) + FabricEventStreamExpansion +} + +// fabricEventStreams implements FabricEventStreamInterface +type fabricEventStreams struct { + client rest.Interface + ns string +} + +// newFabricEventStreams returns a FabricEventStreams +func newFabricEventStreams(c *ZalandoV1alpha1Client, namespace string) *fabricEventStreams { + return &fabricEventStreams{ + client: c.RESTClient(), + ns: namespace, + } +} + +// Get takes name of the fabricEventStream, and returns the corresponding fabricEventStream object, and an error if there is any. +func (c *fabricEventStreams) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.FabricEventStream, err error) { + result = &v1alpha1.FabricEventStream{} + err = c.client.Get(). + Namespace(c.ns). + Resource("fabriceventstreams"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of FabricEventStreams that match those selectors. +func (c *fabricEventStreams) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.FabricEventStreamList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1alpha1.FabricEventStreamList{} + err = c.client.Get(). + Namespace(c.ns). + Resource("fabriceventstreams"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested fabricEventStreams. +func (c *fabricEventStreams) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Namespace(c.ns). + Resource("fabriceventstreams"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a fabricEventStream and creates it. Returns the server's representation of the fabricEventStream, and an error, if there is any. +func (c *fabricEventStreams) Create(ctx context.Context, fabricEventStream *v1alpha1.FabricEventStream, opts v1.CreateOptions) (result *v1alpha1.FabricEventStream, err error) { + result = &v1alpha1.FabricEventStream{} + err = c.client.Post(). + Namespace(c.ns). + Resource("fabriceventstreams"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(fabricEventStream). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a fabricEventStream and updates it. Returns the server's representation of the fabricEventStream, and an error, if there is any. +func (c *fabricEventStreams) Update(ctx context.Context, fabricEventStream *v1alpha1.FabricEventStream, opts v1.UpdateOptions) (result *v1alpha1.FabricEventStream, err error) { + result = &v1alpha1.FabricEventStream{} + err = c.client.Put(). + Namespace(c.ns). + Resource("fabriceventstreams"). + Name(fabricEventStream.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(fabricEventStream). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the fabricEventStream and deletes it. Returns an error if one occurs. +func (c *fabricEventStreams) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("fabriceventstreams"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *fabricEventStreams) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Namespace(c.ns). + Resource("fabriceventstreams"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched fabricEventStream. +func (c *fabricEventStreams) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.FabricEventStream, err error) { + result = &v1alpha1.FabricEventStream{} + err = c.client.Patch(pt). + Namespace(c.ns). + Resource("fabriceventstreams"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fake/doc.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fake/doc.go new file mode 100644 index 000000000..c5fd1c04b --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fake/doc.go @@ -0,0 +1,26 @@ +/* +Copyright 2021 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +// Package fake has the automatically generated clients. +package fake diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fake/fake_fabriceventstream.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fake/fake_fabriceventstream.go new file mode 100644 index 000000000..9fd2485de --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fake/fake_fabriceventstream.go @@ -0,0 +1,136 @@ +/* +Copyright 2021 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1alpha1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeFabricEventStreams implements FabricEventStreamInterface +type FakeFabricEventStreams struct { + Fake *FakeZalandoV1alpha1 + ns string +} + +var fabriceventstreamsResource = schema.GroupVersionResource{Group: "zalando.org", Version: "v1alpha1", Resource: "fabriceventstreams"} + +var fabriceventstreamsKind = schema.GroupVersionKind{Group: "zalando.org", Version: "v1alpha1", Kind: "FabricEventStream"} + +// Get takes name of the fabricEventStream, and returns the corresponding fabricEventStream object, and an error if there is any. +func (c *FakeFabricEventStreams) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.FabricEventStream, err error) { + obj, err := c.Fake. + Invokes(testing.NewGetAction(fabriceventstreamsResource, c.ns, name), &v1alpha1.FabricEventStream{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.FabricEventStream), err +} + +// List takes label and field selectors, and returns the list of FabricEventStreams that match those selectors. +func (c *FakeFabricEventStreams) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.FabricEventStreamList, err error) { + obj, err := c.Fake. + Invokes(testing.NewListAction(fabriceventstreamsResource, fabriceventstreamsKind, c.ns, opts), &v1alpha1.FabricEventStreamList{}) + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.FabricEventStreamList{ListMeta: obj.(*v1alpha1.FabricEventStreamList).ListMeta} + for _, item := range obj.(*v1alpha1.FabricEventStreamList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested fabricEventStreams. +func (c *FakeFabricEventStreams) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchAction(fabriceventstreamsResource, c.ns, opts)) + +} + +// Create takes the representation of a fabricEventStream and creates it. Returns the server's representation of the fabricEventStream, and an error, if there is any. +func (c *FakeFabricEventStreams) Create(ctx context.Context, fabricEventStream *v1alpha1.FabricEventStream, opts v1.CreateOptions) (result *v1alpha1.FabricEventStream, err error) { + obj, err := c.Fake. + Invokes(testing.NewCreateAction(fabriceventstreamsResource, c.ns, fabricEventStream), &v1alpha1.FabricEventStream{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.FabricEventStream), err +} + +// Update takes the representation of a fabricEventStream and updates it. Returns the server's representation of the fabricEventStream, and an error, if there is any. +func (c *FakeFabricEventStreams) Update(ctx context.Context, fabricEventStream *v1alpha1.FabricEventStream, opts v1.UpdateOptions) (result *v1alpha1.FabricEventStream, err error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateAction(fabriceventstreamsResource, c.ns, fabricEventStream), &v1alpha1.FabricEventStream{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.FabricEventStream), err +} + +// Delete takes name of the fabricEventStream and deletes it. Returns an error if one occurs. +func (c *FakeFabricEventStreams) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteAction(fabriceventstreamsResource, c.ns, name), &v1alpha1.FabricEventStream{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeFabricEventStreams) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(fabriceventstreamsResource, c.ns, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha1.FabricEventStreamList{}) + return err +} + +// Patch applies the patch and returns the patched fabricEventStream. +func (c *FakeFabricEventStreams) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.FabricEventStream, err error) { + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(fabriceventstreamsResource, c.ns, name, pt, data, subresources...), &v1alpha1.FabricEventStream{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.FabricEventStream), err +} diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fake/fake_zalando.org_client.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fake/fake_zalando.org_client.go new file mode 100644 index 000000000..f8583cae9 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fake/fake_zalando.org_client.go @@ -0,0 +1,46 @@ +/* +Copyright 2021 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1alpha1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1" + rest "k8s.io/client-go/rest" + testing "k8s.io/client-go/testing" +) + +type FakeZalandoV1alpha1 struct { + *testing.Fake +} + +func (c *FakeZalandoV1alpha1) FabricEventStreams(namespace string) v1alpha1.FabricEventStreamInterface { + return &FakeFabricEventStreams{c, namespace} +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *FakeZalandoV1alpha1) RESTClient() rest.Interface { + var ret *rest.RESTClient + return ret +} diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/generated_expansion.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/generated_expansion.go new file mode 100644 index 000000000..89c4f1d5d --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/generated_expansion.go @@ -0,0 +1,27 @@ +/* +Copyright 2021 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +type FabricEventStreamExpansion interface{} diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/zalando.org_client.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/zalando.org_client.go new file mode 100644 index 000000000..56436b2f4 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/zalando.org_client.go @@ -0,0 +1,95 @@ +/* +Copyright 2021 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + v1alpha1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1alpha1" + "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme" + rest "k8s.io/client-go/rest" +) + +type ZalandoV1alpha1Interface interface { + RESTClient() rest.Interface + FabricEventStreamsGetter +} + +// ZalandoV1alpha1Client is used to interact with features provided by the zalando.org group. +type ZalandoV1alpha1Client struct { + restClient rest.Interface +} + +func (c *ZalandoV1alpha1Client) FabricEventStreams(namespace string) FabricEventStreamInterface { + return newFabricEventStreams(c, namespace) +} + +// NewForConfig creates a new ZalandoV1alpha1Client for the given config. +func NewForConfig(c *rest.Config) (*ZalandoV1alpha1Client, error) { + config := *c + if err := setConfigDefaults(&config); err != nil { + return nil, err + } + client, err := rest.RESTClientFor(&config) + if err != nil { + return nil, err + } + return &ZalandoV1alpha1Client{client}, nil +} + +// NewForConfigOrDie creates a new ZalandoV1alpha1Client for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *ZalandoV1alpha1Client { + client, err := NewForConfig(c) + if err != nil { + panic(err) + } + return client +} + +// New creates a new ZalandoV1alpha1Client for the given RESTClient. +func New(c rest.Interface) *ZalandoV1alpha1Client { + return &ZalandoV1alpha1Client{c} +} + +func setConfigDefaults(config *rest.Config) error { + gv := v1alpha1.SchemeGroupVersion + config.GroupVersion = &gv + config.APIPath = "/apis" + config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() + + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } + + return nil +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *ZalandoV1alpha1Client) RESTClient() rest.Interface { + if c == nil { + return nil + } + return c.restClient +} diff --git a/pkg/generated/informers/externalversions/factory.go b/pkg/generated/informers/externalversions/factory.go index e4b1efdc6..b3938ec74 100644 --- a/pkg/generated/informers/externalversions/factory.go +++ b/pkg/generated/informers/externalversions/factory.go @@ -32,6 +32,7 @@ import ( versioned "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned" acidzalando "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/acid.zalan.do" internalinterfaces "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/internalinterfaces" + zalandoorg "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/zalando.org" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -179,8 +180,13 @@ type SharedInformerFactory interface { WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool Acid() acidzalando.Interface + Zalando() zalandoorg.Interface } func (f *sharedInformerFactory) Acid() acidzalando.Interface { return acidzalando.New(f, f.namespace, f.tweakListOptions) } + +func (f *sharedInformerFactory) Zalando() zalandoorg.Interface { + return zalandoorg.New(f, f.namespace, f.tweakListOptions) +} diff --git a/pkg/generated/informers/externalversions/generic.go b/pkg/generated/informers/externalversions/generic.go index 5fd693558..23515c041 100644 --- a/pkg/generated/informers/externalversions/generic.go +++ b/pkg/generated/informers/externalversions/generic.go @@ -28,6 +28,7 @@ import ( "fmt" v1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + v1alpha1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1alpha1" schema "k8s.io/apimachinery/pkg/runtime/schema" cache "k8s.io/client-go/tools/cache" ) @@ -64,6 +65,10 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource case v1.SchemeGroupVersion.WithResource("postgresqls"): return &genericInformer{resource: resource.GroupResource(), informer: f.Acid().V1().Postgresqls().Informer()}, nil + // Group=zalando.org, Version=v1alpha1 + case v1alpha1.SchemeGroupVersion.WithResource("fabriceventstreams"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Zalando().V1alpha1().FabricEventStreams().Informer()}, nil + } return nil, fmt.Errorf("no informer found for %v", resource) diff --git a/pkg/generated/informers/externalversions/zalando.org/interface.go b/pkg/generated/informers/externalversions/zalando.org/interface.go new file mode 100644 index 000000000..c8cb52a09 --- /dev/null +++ b/pkg/generated/informers/externalversions/zalando.org/interface.go @@ -0,0 +1,52 @@ +/* +Copyright 2021 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package zalando + +import ( + internalinterfaces "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/internalinterfaces" + v1alpha1 "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/zalando.org/v1alpha1" +) + +// Interface provides access to each of this group's versions. +type Interface interface { + // V1alpha1 provides access to shared informers for resources in V1alpha1. + V1alpha1() v1alpha1.Interface +} + +type group struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// V1alpha1 returns a new v1alpha1.Interface. +func (g *group) V1alpha1() v1alpha1.Interface { + return v1alpha1.New(g.factory, g.namespace, g.tweakListOptions) +} diff --git a/pkg/generated/informers/externalversions/zalando.org/v1alpha1/fabriceventstream.go b/pkg/generated/informers/externalversions/zalando.org/v1alpha1/fabriceventstream.go new file mode 100644 index 000000000..7d29d716d --- /dev/null +++ b/pkg/generated/informers/externalversions/zalando.org/v1alpha1/fabriceventstream.go @@ -0,0 +1,96 @@ +/* +Copyright 2021 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + time "time" + + zalandoorgv1alpha1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1alpha1" + versioned "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned" + internalinterfaces "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/internalinterfaces" + v1alpha1 "github.com/zalando/postgres-operator/pkg/generated/listers/zalando.org/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// FabricEventStreamInformer provides access to a shared informer and lister for +// FabricEventStreams. +type FabricEventStreamInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1alpha1.FabricEventStreamLister +} + +type fabricEventStreamInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewFabricEventStreamInformer constructs a new informer for FabricEventStream type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFabricEventStreamInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredFabricEventStreamInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredFabricEventStreamInformer constructs a new informer for FabricEventStream type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredFabricEventStreamInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.ZalandoV1alpha1().FabricEventStreams(namespace).List(context.TODO(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.ZalandoV1alpha1().FabricEventStreams(namespace).Watch(context.TODO(), options) + }, + }, + &zalandoorgv1alpha1.FabricEventStream{}, + resyncPeriod, + indexers, + ) +} + +func (f *fabricEventStreamInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredFabricEventStreamInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *fabricEventStreamInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&zalandoorgv1alpha1.FabricEventStream{}, f.defaultInformer) +} + +func (f *fabricEventStreamInformer) Lister() v1alpha1.FabricEventStreamLister { + return v1alpha1.NewFabricEventStreamLister(f.Informer().GetIndexer()) +} diff --git a/pkg/generated/informers/externalversions/zalando.org/v1alpha1/interface.go b/pkg/generated/informers/externalversions/zalando.org/v1alpha1/interface.go new file mode 100644 index 000000000..7815569a2 --- /dev/null +++ b/pkg/generated/informers/externalversions/zalando.org/v1alpha1/interface.go @@ -0,0 +1,51 @@ +/* +Copyright 2021 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + internalinterfaces "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/internalinterfaces" +) + +// Interface provides access to all the informers in this group version. +type Interface interface { + // FabricEventStreams returns a FabricEventStreamInformer. + FabricEventStreams() FabricEventStreamInformer +} + +type version struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// FabricEventStreams returns a FabricEventStreamInformer. +func (v *version) FabricEventStreams() FabricEventStreamInformer { + return &fabricEventStreamInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} diff --git a/pkg/generated/listers/zalando.org/v1alpha1/expansion_generated.go b/pkg/generated/listers/zalando.org/v1alpha1/expansion_generated.go new file mode 100644 index 000000000..16438da8b --- /dev/null +++ b/pkg/generated/listers/zalando.org/v1alpha1/expansion_generated.go @@ -0,0 +1,33 @@ +/* +Copyright 2021 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +// FabricEventStreamListerExpansion allows custom methods to be added to +// FabricEventStreamLister. +type FabricEventStreamListerExpansion interface{} + +// FabricEventStreamNamespaceListerExpansion allows custom methods to be added to +// FabricEventStreamNamespaceLister. +type FabricEventStreamNamespaceListerExpansion interface{} diff --git a/pkg/generated/listers/zalando.org/v1alpha1/fabriceventstream.go b/pkg/generated/listers/zalando.org/v1alpha1/fabriceventstream.go new file mode 100644 index 000000000..244d11bbf --- /dev/null +++ b/pkg/generated/listers/zalando.org/v1alpha1/fabriceventstream.go @@ -0,0 +1,105 @@ +/* +Copyright 2021 Compose, Zalando SE + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + v1alpha1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1alpha1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// FabricEventStreamLister helps list FabricEventStreams. +// All objects returned here must be treated as read-only. +type FabricEventStreamLister interface { + // List lists all FabricEventStreams in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1alpha1.FabricEventStream, err error) + // FabricEventStreams returns an object that can list and get FabricEventStreams. + FabricEventStreams(namespace string) FabricEventStreamNamespaceLister + FabricEventStreamListerExpansion +} + +// fabricEventStreamLister implements the FabricEventStreamLister interface. +type fabricEventStreamLister struct { + indexer cache.Indexer +} + +// NewFabricEventStreamLister returns a new FabricEventStreamLister. +func NewFabricEventStreamLister(indexer cache.Indexer) FabricEventStreamLister { + return &fabricEventStreamLister{indexer: indexer} +} + +// List lists all FabricEventStreams in the indexer. +func (s *fabricEventStreamLister) List(selector labels.Selector) (ret []*v1alpha1.FabricEventStream, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.FabricEventStream)) + }) + return ret, err +} + +// FabricEventStreams returns an object that can list and get FabricEventStreams. +func (s *fabricEventStreamLister) FabricEventStreams(namespace string) FabricEventStreamNamespaceLister { + return fabricEventStreamNamespaceLister{indexer: s.indexer, namespace: namespace} +} + +// FabricEventStreamNamespaceLister helps list and get FabricEventStreams. +// All objects returned here must be treated as read-only. +type FabricEventStreamNamespaceLister interface { + // List lists all FabricEventStreams in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1alpha1.FabricEventStream, err error) + // Get retrieves the FabricEventStream from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*v1alpha1.FabricEventStream, error) + FabricEventStreamNamespaceListerExpansion +} + +// fabricEventStreamNamespaceLister implements the FabricEventStreamNamespaceLister +// interface. +type fabricEventStreamNamespaceLister struct { + indexer cache.Indexer + namespace string +} + +// List lists all FabricEventStreams in the indexer for a given namespace. +func (s fabricEventStreamNamespaceLister) List(selector labels.Selector) (ret []*v1alpha1.FabricEventStream, err error) { + err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.FabricEventStream)) + }) + return ret, err +} + +// Get retrieves the FabricEventStream from the indexer for a given namespace and name. +func (s fabricEventStreamNamespaceLister) Get(name string) (*v1alpha1.FabricEventStream, error) { + obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1alpha1.Resource("fabriceventstream"), name) + } + return obj.(*v1alpha1.FabricEventStream), nil +} diff --git a/pkg/util/constants/streams.go b/pkg/util/constants/streams.go new file mode 100644 index 000000000..c8360c11b --- /dev/null +++ b/pkg/util/constants/streams.go @@ -0,0 +1,17 @@ +package constants + +// PostgreSQL specific constants +const ( + FESsuffix = "-event-streams" + EventStreamSourcePGType = "PostgresLogicalReplication" + EventStreamSourceSlotName = "fes" + EventStreamSourceAuthType = "DatabaseAuthenticationSecret" + EventStreamFlowPgNakadiType = "PostgresWalToNakadiDataEvent" + EventStreamFlowPgApiType = "PostgresWalToApiCallHomeEvent" + EventStreamFlowDataTypeColumn = "data_type" + EventStreamFlowDataOpColumn = "data_op" + EventStreamFlowMetadataColumn = "metadata" + EventStreamFlowDataColumn = "data" + EventStreamSinkNakadiType = "Nakadi" + EventStreamSinkSqsType = "Sqs" +) diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index dd6ec1e8b..67192dd21 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -12,8 +12,9 @@ import ( clientbatchv1beta1 "k8s.io/client-go/kubernetes/typed/batch/v1beta1" apiacidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - acidv1client "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned" + zalandoclient "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned" acidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" + zalandov1alpha1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1" "github.com/zalando/postgres-operator/pkg/spec" apiappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -58,9 +59,11 @@ type KubernetesClient struct { acidv1.OperatorConfigurationsGetter acidv1.PostgresTeamsGetter acidv1.PostgresqlsGetter + zalandov1alpha1.FabricEventStreamsGetter - RESTClient rest.Interface - AcidV1ClientSet *acidv1client.Clientset + RESTClient rest.Interface + AcidV1ClientSet *zalandoclient.Clientset + ZalandoV1Alpha1ClientSet *zalandoclient.Clientset } type mockSecret struct { @@ -158,14 +161,19 @@ func NewFromConfig(cfg *rest.Config) (KubernetesClient, error) { kubeClient.CustomResourceDefinitionsGetter = apiextClient.ApiextensionsV1() - kubeClient.AcidV1ClientSet = acidv1client.NewForConfigOrDie(cfg) + kubeClient.AcidV1ClientSet = zalandoclient.NewForConfigOrDie(cfg) if err != nil { return kubeClient, fmt.Errorf("could not create acid.zalan.do clientset: %v", err) } + kubeClient.ZalandoV1Alpha1ClientSet = zalandoclient.NewForConfigOrDie(cfg) + if err != nil { + return kubeClient, fmt.Errorf("could not create zalando.org clientset: %v", err) + } kubeClient.OperatorConfigurationsGetter = kubeClient.AcidV1ClientSet.AcidV1() kubeClient.PostgresTeamsGetter = kubeClient.AcidV1ClientSet.AcidV1() kubeClient.PostgresqlsGetter = kubeClient.AcidV1ClientSet.AcidV1() + kubeClient.FabricEventStreamsGetter = kubeClient.ZalandoV1Alpha1ClientSet.ZalandoV1alpha1() return kubeClient, nil } From 11f3715ed1ba0e88dc2621bcc790103d0b1410fe Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 29 Jul 2021 14:29:42 +0200 Subject: [PATCH 02/44] check manifest settings for logical decoding before creating streams --- pkg/cluster/streams.go | 42 +++++++++++++++++++++++++++++++++-- pkg/cluster/streams_test.go | 17 +++++++++++++- pkg/util/constants/streams.go | 2 +- 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index 6f1b7efec..b84f4424c 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -19,8 +19,13 @@ var outboxTableNameTemplate config.StringTemplate = "{table}_{eventtype}_outbox" func (c *Cluster) createStreams() error { c.setProcessName("creating streams") + logicalDecodingEnabled, err := c.logicalDecodingEnabled() + if !logicalDecodingEnabled || err != nil { + return fmt.Errorf("logical decoding setup incomplete: %v", err) + } + fes := c.generateFabricEventStream() - _, err := c.KubeClient.FabricEventStreamsGetter.FabricEventStreams(c.Namespace).Create(context.TODO(), fes, metav1.CreateOptions{}) + _, err = c.KubeClient.FabricEventStreamsGetter.FabricEventStreams(c.Namespace).Create(context.TODO(), fes, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("could not create event stream custom resource: %v", err) } @@ -28,6 +33,29 @@ func (c *Cluster) createStreams() error { return nil } +func (c *Cluster) logicalDecodingEnabled() (bool, error) { + var errors []string + + walLevel := c.Spec.PostgresqlParam.Parameters["wal_level"] + if walLevel == "" || walLevel != "logical" { + errors = append(errors, "setting 'wal_level: logical' missing under spec.postgresql.parameters") + } + + for _, stream := range c.Spec.Streams { + slotName := c.getLogicalReplicationSlot(stream.Database) + + if slotName == "" { + errors = append(errors, fmt.Sprintf("no logical replication slot defined under spec.patroni.slots for database %q", stream.Database)) + } + } + + if len(errors) > 0 { + return false, fmt.Errorf("logical decoding setup incomplete: %v", errors) + } + + return true, nil +} + func (c *Cluster) syncStreamDbResources() error { for _, stream := range c.Spec.Streams { @@ -148,7 +176,7 @@ func getOutboxTable(tableName, eventType string) zalandov1alpha1.EventStreamTabl func (c *Cluster) getStreamConnection(database, user string) zalandov1alpha1.Connection { return zalandov1alpha1.Connection{ Url: fmt.Sprintf("jdbc:postgresql://%s.%s/%s?user=%s&ssl=true&sslmode=require", c.Name, c.Namespace, database, user), - SlotName: constants.EventStreamSourceSlotName, + SlotName: c.getLogicalReplicationSlot(database), DBAuth: zalandov1alpha1.DBAuth{ Type: constants.EventStreamSourceAuthType, Name: c.credentialSecretNameForCluster(user, c.ClusterName), @@ -158,6 +186,16 @@ func (c *Cluster) getStreamConnection(database, user string) zalandov1alpha1.Con } } +func (c *Cluster) getLogicalReplicationSlot(database string) string { + for slotName, slot := range c.Spec.Patroni.Slots { + if strings.HasPrefix(slotName, constants.EventStreamSourceSlotPrefix) && slot["type"] == "logical" && slot["database"] == database { + return slotName + } + } + + return "" +} + func (c *Cluster) syncStreams() error { c.setProcessName("syncing streams") diff --git a/pkg/cluster/streams_test.go b/pkg/cluster/streams_test.go index aebd29ff6..06f46e8fc 100644 --- a/pkg/cluster/streams_test.go +++ b/pkg/cluster/streams_test.go @@ -40,6 +40,20 @@ func TestGenerateFabricEventStream(t *testing.T) { Databases: map[string]string{ "foo": "foo_user", }, + Patroni: acidv1.Patroni{ + Slots: map[string]map[string]string{ + "fes": { + "type": "logical", + "database": "foo", + "plugin": "wal2json", + }, + }, + }, + PostgresqlParam: acidv1.PostgresqlParam{ + Parameters: map[string]string{ + "wal_level": "logical", + }, + }, Streams: []acidv1.Stream{ { Type: "nakadi", @@ -93,7 +107,8 @@ func TestGenerateFabricEventStream(t *testing.T) { }, }, client, pg, logger, eventRecorder) - cluster.syncStreams() + err := cluster.syncStreams() + assert.NoError(t, err) streamCRD, err := cluster.KubeClient.FabricEventStreams(namespace).Get(context.TODO(), cluster.Name+constants.FESsuffix, metav1.GetOptions{}) assert.NoError(t, err) diff --git a/pkg/util/constants/streams.go b/pkg/util/constants/streams.go index c8360c11b..36eb376cf 100644 --- a/pkg/util/constants/streams.go +++ b/pkg/util/constants/streams.go @@ -4,7 +4,7 @@ package constants const ( FESsuffix = "-event-streams" EventStreamSourcePGType = "PostgresLogicalReplication" - EventStreamSourceSlotName = "fes" + EventStreamSourceSlotPrefix = "fes" EventStreamSourceAuthType = "DatabaseAuthenticationSecret" EventStreamFlowPgNakadiType = "PostgresWalToNakadiDataEvent" EventStreamFlowPgApiType = "PostgresWalToApiCallHomeEvent" From 70df4515bcb0f6de73bfff4e0cd2aa089c47372f Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 30 Jul 2021 14:11:12 +0200 Subject: [PATCH 03/44] intermediate commit --- pkg/cluster/streams.go | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index b84f4424c..a2adb1b15 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -19,8 +19,8 @@ var outboxTableNameTemplate config.StringTemplate = "{table}_{eventtype}_outbox" func (c *Cluster) createStreams() error { c.setProcessName("creating streams") - logicalDecodingEnabled, err := c.logicalDecodingEnabled() - if !logicalDecodingEnabled || err != nil { + err := c.syncLogicalDecoding() + if err != nil { return fmt.Errorf("logical decoding setup incomplete: %v", err) } @@ -33,27 +33,31 @@ func (c *Cluster) createStreams() error { return nil } -func (c *Cluster) logicalDecodingEnabled() (bool, error) { - var errors []string +func (c *Cluster) syncLogicalDecoding() error { walLevel := c.Spec.PostgresqlParam.Parameters["wal_level"] if walLevel == "" || walLevel != "logical" { - errors = append(errors, "setting 'wal_level: logical' missing under spec.postgresql.parameters") + c.logger.Debugf("setting wal level to 'logical' in postgres configuration") + pods, err := c.listPods() + if err != nil || len(pods) == 0 { + return err + } + for _, pod := range pods { + if err := c.patroni.SetPostgresParameters(&pod, map[string]string{"wal_level": "logical"}); err == nil { + return fmt.Errorf("could not set wal_level to 'logical' calling Patroni REST API: %v", err) + } + } } for _, stream := range c.Spec.Streams { slotName := c.getLogicalReplicationSlot(stream.Database) if slotName == "" { - errors = append(errors, fmt.Sprintf("no logical replication slot defined under spec.patroni.slots for database %q", stream.Database)) + c.logger.Debugf("creating logical replication slot %d in database %d", constants.EventStreamSourceSlotPrefix+stream.Database, stream.Database) } } - if len(errors) > 0 { - return false, fmt.Errorf("logical decoding setup incomplete: %v", errors) - } - - return true, nil + return nil } func (c *Cluster) syncStreamDbResources() error { From eb7050571da7decf4d614fb35f49d8fad303f867 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 10 Aug 2021 11:26:23 +0200 Subject: [PATCH 04/44] operator updates Postgres config and creates replication user --- .../postgres-operator/crds/postgresqls.yaml | 30 +++++ manifests/postgresql.crd.yaml | 30 +++++ pkg/apis/acid.zalan.do/v1/crds.go | 14 ++- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 15 ++- pkg/cluster/cluster.go | 19 +++- pkg/cluster/streams.go | 103 +++++++++++------- pkg/cluster/streams_test.go | 37 ++----- pkg/cluster/sync.go | 79 ++++---------- pkg/util/constants/streams.go | 2 +- 9 files changed, 192 insertions(+), 137 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 7604e8d5a..82aba3767 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -470,6 +470,36 @@ spec: properties: s3_wal_path: type: string + streams: + type: array + nullable: true + items: + type: object + required: + - streamType + properties: + batchSize: + type: integer + database: + type: string + filter: + type: object + additionalProperties: + type: string + queueName: + type: string + sqsArn: + type: string + tables: + type: object + additionalProperties: + type: string + streamType: + type: string + enum: + - "nakadi" + - "sqs" + - "wal" teamId: type: string tls: diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 652a66fda..87a03392b 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -466,6 +466,36 @@ spec: properties: s3_wal_path: type: string + streams: + type: array + nullable: true + items: + type: object + required: + - streamType + properties: + batchSize: + type: integer + database: + type: string + filter: + type: object + additionalProperties: + type: string + queueName: + type: string + sqsArn: + type: string + tables: + type: object + additionalProperties: + type: string + streamType: + type: string + enum: + - "nakadi" + - "sqs" + - "wal" teamId: type: string tls: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 105e14cd7..d0c4fa103 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -662,8 +662,11 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Items: &apiextv1.JSONSchemaPropsOrArray{ Schema: &apiextv1.JSONSchemaProps{ Type: "object", - Required: []string{"type"}, + Required: []string{"streamType"}, Properties: map[string]apiextv1.JSONSchemaProps{ + "batchSize": { + Type: "integer", + }, "database": { Type: "string", }, @@ -671,10 +674,13 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Type: "object", AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ Schema: &apiextv1.JSONSchemaProps{ - Type: "string", + Type: "string", }, }, }, + "queueName": { + Type: "string", + }, "sqsArn": { Type: "string", }, @@ -682,11 +688,11 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Type: "object", AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ Schema: &apiextv1.JSONSchemaProps{ - Type: "string", + Type: "string", }, }, }, - "type": { + "streamType": { Type: "string", Enum: []apiextv1.JSON{ { diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 02eaf6ee6..a4a8f4477 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -227,12 +227,11 @@ type ConnectionPooler struct { } type Stream struct { - Type string `json:"type"` - Database string `json:"database,omitempty"` - Tables map[string]string `json:"tables,omitempty"` - Filter map[string]string `json:"filter,omitempty"` - BatchSize uint32 `json:"batchSize,omitempty"` - SqsArn string `json:"sqsArn,omitempty"` - QueueName string `json:"queueName,omitempty"` - User string `json:"user,omitempty"` + StreamType string `json:"streamType"` + Database string `json:"database,omitempty"` + Tables map[string]string `json:"tables,omitempty"` + Filter map[string]string `json:"filter,omitempty"` + BatchSize uint32 `json:"batchSize,omitempty"` + SqsArn string `json:"sqsArn,omitempty"` + QueueName string `json:"queueName,omitempty"` } diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 3165534e8..977c3c4e5 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -362,7 +362,7 @@ func (c *Cluster) Create() error { c.createConnectionPooler(c.installLookupFunction) if len(c.Spec.Streams) > 0 { - c.createStreams() + c.syncStreams() } return nil @@ -1052,6 +1052,23 @@ func (c *Cluster) initSystemUsers() { c.systemUsers[constants.ConnectionPoolerUserKeyName] = connectionPoolerUser } } + + // replication users for event streams are another exception + // the operator will create one replication user for all streams + if len(c.Spec.Streams) > 0 { + username := constants.EventStreamSourceSlotPrefix + constants.UserRoleNameSuffix + streamUser := spec.PgUser{ + Origin: spec.RoleConnectionPooler, + Name: username, + Namespace: c.Namespace, + Flags: []string{constants.RoleFlagLogin, constants.RoleFlagReplication}, + Password: util.RandomPassword(constants.PasswordLength), + } + + if _, exists := c.pgUsers[username]; !exists { + c.pgUsers[username] = streamUser + } + } } func (c *Cluster) initPreparedDatabaseRoles() error { diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index a2adb1b15..8bc3129d8 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -8,6 +8,7 @@ import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" zalandov1alpha1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1alpha1" + "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" @@ -19,13 +20,8 @@ var outboxTableNameTemplate config.StringTemplate = "{table}_{eventtype}_outbox" func (c *Cluster) createStreams() error { c.setProcessName("creating streams") - err := c.syncLogicalDecoding() - if err != nil { - return fmt.Errorf("logical decoding setup incomplete: %v", err) - } - fes := c.generateFabricEventStream() - _, err = c.KubeClient.FabricEventStreamsGetter.FabricEventStreams(c.Namespace).Create(context.TODO(), fes, metav1.CreateOptions{}) + _, err := c.KubeClient.FabricEventStreamsGetter.FabricEventStreams(c.Namespace).Create(context.TODO(), fes, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("could not create event stream custom resource: %v", err) } @@ -33,27 +29,61 @@ func (c *Cluster) createStreams() error { return nil } -func (c *Cluster) syncLogicalDecoding() error { +func (c *Cluster) updateStreams(newEventStreams *zalandov1alpha1.FabricEventStream) error { + c.setProcessName("updating event streams") - walLevel := c.Spec.PostgresqlParam.Parameters["wal_level"] - if walLevel == "" || walLevel != "logical" { - c.logger.Debugf("setting wal level to 'logical' in postgres configuration") - pods, err := c.listPods() - if err != nil || len(pods) == 0 { - return err - } - for _, pod := range pods { - if err := c.patroni.SetPostgresParameters(&pod, map[string]string{"wal_level": "logical"}); err == nil { - return fmt.Errorf("could not set wal_level to 'logical' calling Patroni REST API: %v", err) - } - } + _, err := c.KubeClient.FabricEventStreamsGetter.FabricEventStreams(c.Namespace).Update(context.TODO(), newEventStreams, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("could not update event stream custom resource: %v", err) } + return nil +} + +func (c *Cluster) syncPostgresConfig() error { + + desiredPostgresConfig := make(map[string]interface{}) + slots := make(map[string]map[string]string) + + c.logger.Debugf("setting wal level to 'logical' in postgres configuration") + desiredPostgresConfig["postgresql"] = map[string]interface{}{patroniPGParametersParameterName: map[string]string{"wal_level": "logical"}} + for _, stream := range c.Spec.Streams { slotName := c.getLogicalReplicationSlot(stream.Database) if slotName == "" { - c.logger.Debugf("creating logical replication slot %d in database %d", constants.EventStreamSourceSlotPrefix+stream.Database, stream.Database) + c.logger.Debugf("creating logical replication slot %q in database %q", constants.EventStreamSourceSlotPrefix+stream.Database, stream.Database) + slot := map[string]string{ + "database": stream.Database, + "plugin": "wal2json", + "type": "logical", + } + slots[constants.EventStreamSourceSlotPrefix+stream.Database] = slot + } + } + + if len(slots) > 0 { + desiredPostgresConfig["slots"] = slots + } else { + return nil + } + + pods, err := c.listPods() + if err != nil || len(pods) == 0 { + return err + } + for i, pod := range pods { + podName := util.NameFromMeta(pods[i].ObjectMeta) + effectivePostgresConfig, err := c.patroni.GetConfig(&pod) + if err != nil { + c.logger.Warningf("could not get Postgres config from pod %s: %v", podName, err) + continue + } + + _, err = c.checkAndSetGlobalPostgreSQLConfiguration(&pod, effectivePostgresConfig, desiredPostgresConfig) + if err != nil { + c.logger.Warningf("could not set PostgreSQL configuration options for pod %s: %v", podName, err) + continue } } @@ -64,18 +94,18 @@ func (c *Cluster) syncStreamDbResources() error { for _, stream := range c.Spec.Streams { if err := c.initDbConnWithName(stream.Database); err != nil { - return fmt.Errorf("could not init connection to database %s specified for event stream: %v", stream.Database, err) + return fmt.Errorf("could not init connection to database %q specified for event stream: %v", stream.Database, err) } for table, eventType := range stream.Tables { tableName, schemaName := getTableSchema(table) if exists, err := c.tableExists(tableName, schemaName); !exists { - return fmt.Errorf("could not find table %s specified for event stream: %v", table, err) + return fmt.Errorf("could not find table %q specified for event stream: %v", table, err) } // check if outbox table exists and if not, create it outboxTable := outboxTableNameTemplate.Format("table", tableName, "eventtype", eventType) if exists, err := c.tableExists(outboxTable, schemaName); !exists { - return fmt.Errorf("could not find outbox table %s specified for event stream: %v", outboxTable, err) + return fmt.Errorf("could not find outbox table %q specified for event stream: %v", outboxTable, err) } } } @@ -120,12 +150,12 @@ func (c *Cluster) getEventStreamSource(stream acidv1.Stream, table, eventType st Schema: schema, EventStreamTable: getOutboxTable(table, eventType), Filter: streamFilter, - Connection: c.getStreamConnection(stream.Database, stream.User), + Connection: c.getStreamConnection(stream.Database, constants.EventStreamSourceSlotPrefix+constants.UserRoleNameSuffix), } } func getEventStreamFlow(stream acidv1.Stream) zalandov1alpha1.EventStreamFlow { - switch stream.Type { + switch stream.StreamType { case "nakadi": return zalandov1alpha1.EventStreamFlow{ Type: constants.EventStreamFlowPgNakadiType, @@ -144,7 +174,7 @@ func getEventStreamFlow(stream acidv1.Stream) zalandov1alpha1.EventStreamFlow { } func getEventStreamSink(stream acidv1.Stream, eventType string) zalandov1alpha1.EventStreamSink { - switch stream.Type { + switch stream.StreamType { case "nakadi": return zalandov1alpha1.EventStreamSink{ Type: constants.EventStreamSinkNakadiType, @@ -204,6 +234,11 @@ func (c *Cluster) syncStreams() error { c.setProcessName("syncing streams") + err := c.syncPostgresConfig() + if err != nil { + return fmt.Errorf("logical decoding setup incomplete: %v", err) + } + effectiveStreams, err := c.KubeClient.FabricEventStreamsGetter.FabricEventStreams(c.Namespace).Get(context.TODO(), c.Name+constants.FESsuffix, metav1.GetOptions{}) if err != nil { if !k8sutil.ResourceNotFound(err) { @@ -216,7 +251,10 @@ func (c *Cluster) syncStreams() error { return fmt.Errorf("could not create missing streams: %v", err) } } else { - c.syncStreamDbResources() + err := c.syncStreamDbResources() + if err != nil { + return fmt.Warnf("database setup incomplete: %v", err) + } desiredStreams := c.generateFabricEventStream() if reflect.DeepEqual(effectiveStreams.Spec, desiredStreams.Spec) { c.updateStreams(desiredStreams) @@ -225,14 +263,3 @@ func (c *Cluster) syncStreams() error { return nil } - -func (c *Cluster) updateStreams(newEventStreams *zalandov1alpha1.FabricEventStream) error { - c.setProcessName("updating event streams") - - _, err := c.KubeClient.FabricEventStreamsGetter.FabricEventStreams(c.Namespace).Update(context.TODO(), newEventStreams, metav1.UpdateOptions{}) - if err != nil { - return fmt.Errorf("could not update event stream custom resource: %v", err) - } - - return nil -} diff --git a/pkg/cluster/streams_test.go b/pkg/cluster/streams_test.go index 06f46e8fc..8119168b0 100644 --- a/pkg/cluster/streams_test.go +++ b/pkg/cluster/streams_test.go @@ -23,6 +23,7 @@ func newFakeK8sStreamClient() (k8sutil.KubernetesClient, *fake.Clientset) { return k8sutil.KubernetesClient{ FabricEventStreamsGetter: zalandoClientSet.ZalandoV1alpha1(), + PodsGetter: clientSet.CoreV1(), }, clientSet } @@ -40,50 +41,32 @@ func TestGenerateFabricEventStream(t *testing.T) { Databases: map[string]string{ "foo": "foo_user", }, - Patroni: acidv1.Patroni{ - Slots: map[string]map[string]string{ - "fes": { - "type": "logical", - "database": "foo", - "plugin": "wal2json", - }, - }, - }, - PostgresqlParam: acidv1.PostgresqlParam{ - Parameters: map[string]string{ - "wal_level": "logical", - }, - }, Streams: []acidv1.Stream{ { - Type: "nakadi", - Database: "foo", + StreamType: "nakadi", + Database: "foo", Tables: map[string]string{ "bar": "stream_type_a", }, BatchSize: uint32(100), - User: "foo_user", }, { - Type: "wal", - Database: "foo", + StreamType: "wal", + Database: "foo", Tables: map[string]string{ "bar": "stream_type_a", }, BatchSize: uint32(100), - User: "zalando", }, { - Type: "sqs", - Database: "foo", - SqsArn: "arn:aws:sqs:eu-central-1:111122223333", - QueueName: "foo-queue", - User: "foo_user", + StreamType: "sqs", + Database: "foo", + SqsArn: "arn:aws:sqs:eu-central-1:111122223333", + QueueName: "foo-queue", }, }, Users: map[string]acidv1.UserFlags{ - "foo_user": {}, - "zalando": {}, + "foo_user": []string{"replication"}, }, Volume: acidv1.Volume{ Size: "1Gi", diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 4937a2034..e9cb1abd5 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -265,10 +265,11 @@ func (c *Cluster) syncPodDisruptionBudget(isUpdate bool) error { func (c *Cluster) syncStatefulSet() error { var ( masterPod *v1.Pod - postgresConfig map[string]interface{} + effectivePostgresConfig map[string]interface{} instanceRestartRequired bool ) + desiredPostgresConfig := make(map[string]interface{}) podsToRecreate := make([]v1.Pod, 0) switchoverCandidates := make([]spec.NamespacedName, 0) @@ -394,14 +395,25 @@ func (c *Cluster) syncStatefulSet() error { // get Postgres config, compare with manifest and update via Patroni PATCH endpoint if it differs // Patroni's config endpoint is just a "proxy" to DCS. It is enough to patch it only once and it doesn't matter which pod is used. + desiredPostgresConfig["postgresql"] = map[string]interface{}{patroniPGParametersParameterName: c.Spec.Parameters} + desiredPostgresConfig["loop_wait"] = c.Spec.Patroni.LoopWait + desiredPostgresConfig["maximum_lag_on_failover"] = c.Spec.Patroni.MaximumLagOnFailover + desiredPostgresConfig["pg_hba"] = c.Spec.Patroni.PgHba + desiredPostgresConfig["retry_timeout"] = c.Spec.Patroni.RetryTimeout + desiredPostgresConfig["slots"] = c.Spec.Patroni.Slots + desiredPostgresConfig["synchronous_mode"] = c.Spec.Patroni.SynchronousMode + desiredPostgresConfig["synchronous_mode_strict"] = c.Spec.Patroni.SynchronousModeStrict + desiredPostgresConfig["ttl"] = c.Spec.Patroni.TTL + for i, pod := range pods { podName := util.NameFromMeta(pods[i].ObjectMeta) - config, err := c.patroni.GetConfig(&pod) + effectivePostgresConfig, err := c.patroni.GetConfig(&pod) if err != nil { c.logger.Warningf("could not get Postgres config from pod %s: %v", podName, err) continue } - instanceRestartRequired, err = c.checkAndSetGlobalPostgreSQLConfiguration(&pod, config) + + instanceRestartRequired, err = c.checkAndSetGlobalPostgreSQLConfiguration(&pod, effectivePostgresConfig, desiredPostgresConfig) if err != nil { c.logger.Warningf("could not set PostgreSQL configuration options for pod %s: %v", podName, err) continue @@ -412,7 +424,7 @@ func (c *Cluster) syncStatefulSet() error { // if the config update requires a restart, call Patroni restart for replicas first, then master if instanceRestartRequired { c.logger.Debug("restarting Postgres server within pods") - ttl, ok := postgresConfig["ttl"].(int32) + ttl, ok := effectivePostgresConfig["ttl"].(int32) if !ok { ttl = 30 } @@ -493,62 +505,13 @@ func (c *Cluster) AnnotationsToPropagate(annotations map[string]string) map[stri // checkAndSetGlobalPostgreSQLConfiguration checks whether cluster-wide API parameters // (like max_connections) have changed and if necessary sets it via the Patroni API -func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, patroniConfig map[string]interface{}) (bool, error) { - configToSet := make(map[string]interface{}) - parametersToSet := make(map[string]string) - effectivePgParameters := make(map[string]interface{}) - - // read effective Patroni config if set - if patroniConfig != nil { - effectivePostgresql := patroniConfig["postgresql"].(map[string]interface{}) - effectivePgParameters = effectivePostgresql[patroniPGParametersParameterName].(map[string]interface{}) - } - - // compare parameters under postgresql section with c.Spec.Postgresql.Parameters from manifest - desiredPgParameters := c.Spec.Parameters - for desiredOption, desiredValue := range desiredPgParameters { - effectiveValue := effectivePgParameters[desiredOption] - if isBootstrapOnlyParameter(desiredOption) && (effectiveValue != desiredValue) { - parametersToSet[desiredOption] = desiredValue - } - } - - if len(parametersToSet) > 0 { - configToSet["postgresql"] = map[string]interface{}{patroniPGParametersParameterName: parametersToSet} - } - - // compare other options from config with c.Spec.Patroni from manifest - desiredPatroniConfig := c.Spec.Patroni - if desiredPatroniConfig.LoopWait > 0 && desiredPatroniConfig.LoopWait != uint32(patroniConfig["loop_wait"].(float64)) { - configToSet["loop_wait"] = desiredPatroniConfig.LoopWait - } - if desiredPatroniConfig.MaximumLagOnFailover > 0 && desiredPatroniConfig.MaximumLagOnFailover != float32(patroniConfig["maximum_lag_on_failover"].(float64)) { - configToSet["maximum_lag_on_failover"] = desiredPatroniConfig.MaximumLagOnFailover - } - if desiredPatroniConfig.PgHba != nil && !reflect.DeepEqual(desiredPatroniConfig.PgHba, (patroniConfig["pg_hba"])) { - configToSet["pg_hba"] = desiredPatroniConfig.PgHba - } - if desiredPatroniConfig.RetryTimeout > 0 && desiredPatroniConfig.RetryTimeout != uint32(patroniConfig["retry_timeout"].(float64)) { - configToSet["retry_timeout"] = desiredPatroniConfig.RetryTimeout - } - if desiredPatroniConfig.Slots != nil && !reflect.DeepEqual(desiredPatroniConfig.Slots, patroniConfig["slots"]) { - configToSet["slots"] = desiredPatroniConfig.Slots - } - if desiredPatroniConfig.SynchronousMode != patroniConfig["synchronous_mode"] { - configToSet["synchronous_mode"] = desiredPatroniConfig.SynchronousMode - } - if desiredPatroniConfig.SynchronousModeStrict != patroniConfig["synchronous_mode_strict"] { - configToSet["synchronous_mode_strict"] = desiredPatroniConfig.SynchronousModeStrict - } - if desiredPatroniConfig.TTL > 0 && desiredPatroniConfig.TTL != uint32(patroniConfig["ttl"].(float64)) { - configToSet["ttl"] = desiredPatroniConfig.TTL - } +func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, effectivePatroniConfig, desiredPatroniConfig map[string]interface{}) (bool, error) { - if len(configToSet) == 0 { + if reflect.DeepEqual(effectivePatroniConfig, desiredPatroniConfig) { return false, nil } - configToSetJson, err := json.Marshal(configToSet) + configToSetJson, err := json.Marshal(desiredPatroniConfig) if err != nil { c.logger.Debugf("could not convert config patch to JSON: %v", err) } @@ -558,8 +521,8 @@ func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, patroniC podName := util.NameFromMeta(pod.ObjectMeta) c.logger.Debugf("patching Postgres config via Patroni API on pod %s with following options: %s", podName, configToSetJson) - if err = c.patroni.SetConfig(pod, configToSet); err != nil { - return true, fmt.Errorf("could not patch postgres parameters with a pod %s: %v", podName, err) + if err = c.patroni.SetConfig(pod, desiredPatroniConfig); err != nil { + return true, fmt.Errorf("could not patch postgres parameters within pod %s: %v", podName, err) } return true, nil diff --git a/pkg/util/constants/streams.go b/pkg/util/constants/streams.go index 36eb376cf..7a2c9f81f 100644 --- a/pkg/util/constants/streams.go +++ b/pkg/util/constants/streams.go @@ -4,7 +4,7 @@ package constants const ( FESsuffix = "-event-streams" EventStreamSourcePGType = "PostgresLogicalReplication" - EventStreamSourceSlotPrefix = "fes" + EventStreamSourceSlotPrefix = "fes_" EventStreamSourceAuthType = "DatabaseAuthenticationSecret" EventStreamFlowPgNakadiType = "PostgresWalToNakadiDataEvent" EventStreamFlowPgApiType = "PostgresWalToApiCallHomeEvent" From 14e989ac839780dd973757f510799f764bc92757 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 10 Aug 2021 11:56:15 +0200 Subject: [PATCH 05/44] update log messages --- pkg/cluster/streams.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index 8bc3129d8..bbfe17eca 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -236,7 +236,7 @@ func (c *Cluster) syncStreams() error { err := c.syncPostgresConfig() if err != nil { - return fmt.Errorf("logical decoding setup incomplete: %v", err) + return fmt.Errorf("could not update Postgres config for event streaming: %v", err) } effectiveStreams, err := c.KubeClient.FabricEventStreamsGetter.FabricEventStreams(c.Namespace).Get(context.TODO(), c.Name+constants.FESsuffix, metav1.GetOptions{}) @@ -253,7 +253,7 @@ func (c *Cluster) syncStreams() error { } else { err := c.syncStreamDbResources() if err != nil { - return fmt.Warnf("database setup incomplete: %v", err) + c.logger.Warnf("database setup might be incomplete : %v", err) } desiredStreams := c.generateFabricEventStream() if reflect.DeepEqual(effectiveStreams.Spec, desiredStreams.Spec) { From 4862b05c69b8e5f8642082efcc81d171f1d16ecd Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 10 Aug 2021 17:29:59 +0200 Subject: [PATCH 06/44] fix Postgres config sync --- .../postgres-operator/crds/postgresqls.yaml | 20 ++++---- manifests/postgresql.crd.yaml | 20 ++++---- pkg/cluster/sync.go | 51 ++++++++++++++----- 3 files changed, 58 insertions(+), 33 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 82aba3767..4f6028f6d 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -470,7 +470,7 @@ spec: properties: s3_wal_path: type: string - streams: + streams: type: array nullable: true items: @@ -479,18 +479,18 @@ spec: - streamType properties: batchSize: - type: integer - database: - type: string - filter: + type: integer + database: + type: string + filter: type: object additionalProperties: type: string - queueName: - type: string - sqsArn: - type: string - tables: + queueName: + type: string + sqsArn: + type: string + tables: type: object additionalProperties: type: string diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 87a03392b..41511c955 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -466,7 +466,7 @@ spec: properties: s3_wal_path: type: string - streams: + streams: type: array nullable: true items: @@ -475,18 +475,18 @@ spec: - streamType properties: batchSize: - type: integer - database: - type: string - filter: + type: integer + database: + type: string + filter: type: object additionalProperties: type: string - queueName: - type: string - sqsArn: - type: string - tables: + queueName: + type: string + sqsArn: + type: string + tables: type: object additionalProperties: type: string diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index e9cb1abd5..0b33ade4d 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -395,15 +395,33 @@ func (c *Cluster) syncStatefulSet() error { // get Postgres config, compare with manifest and update via Patroni PATCH endpoint if it differs // Patroni's config endpoint is just a "proxy" to DCS. It is enough to patch it only once and it doesn't matter which pod is used. - desiredPostgresConfig["postgresql"] = map[string]interface{}{patroniPGParametersParameterName: c.Spec.Parameters} - desiredPostgresConfig["loop_wait"] = c.Spec.Patroni.LoopWait - desiredPostgresConfig["maximum_lag_on_failover"] = c.Spec.Patroni.MaximumLagOnFailover - desiredPostgresConfig["pg_hba"] = c.Spec.Patroni.PgHba - desiredPostgresConfig["retry_timeout"] = c.Spec.Patroni.RetryTimeout - desiredPostgresConfig["slots"] = c.Spec.Patroni.Slots - desiredPostgresConfig["synchronous_mode"] = c.Spec.Patroni.SynchronousMode - desiredPostgresConfig["synchronous_mode_strict"] = c.Spec.Patroni.SynchronousModeStrict - desiredPostgresConfig["ttl"] = c.Spec.Patroni.TTL + if len(c.Spec.Parameters) > 0 { + desiredPostgresConfig["postgresql"] = map[string]interface{}{patroniPGParametersParameterName: c.Spec.Parameters} + } + if c.Spec.Patroni.LoopWait > 0 { + desiredPostgresConfig["loop_wait"] = c.Spec.Patroni.LoopWait + } + if c.Spec.Patroni.MaximumLagOnFailover > 0 { + desiredPostgresConfig["maximum_lag_on_failover"] = c.Spec.Patroni.MaximumLagOnFailover + } + if c.Spec.Patroni.PgHba != nil { + desiredPostgresConfig["pg_hba"] = c.Spec.Patroni.PgHba + } + if c.Spec.Patroni.RetryTimeout > 0 { + desiredPostgresConfig["retry_timeout"] = c.Spec.Patroni.RetryTimeout + } + if c.Spec.Patroni.Slots != nil { + desiredPostgresConfig["slots"] = c.Spec.Patroni.Slots + } + if c.Spec.Patroni.TTL > 0 { + desiredPostgresConfig["ttl"] = c.Spec.Patroni.TTL + } + if effectivePostgresConfig["synchronous_mode"] != desiredPostgresConfig["synchronous_mode"] { + desiredPostgresConfig["synchronous_mode"] = c.Spec.Patroni.SynchronousMode + } + if effectivePostgresConfig["synchronous_mode_strict"] != desiredPostgresConfig["synchronous_mode_strict"] { + desiredPostgresConfig["synchronous_mode_strict"] = c.Spec.Patroni.SynchronousModeStrict + } for i, pod := range pods { podName := util.NameFromMeta(pods[i].ObjectMeta) @@ -505,13 +523,20 @@ func (c *Cluster) AnnotationsToPropagate(annotations map[string]string) map[stri // checkAndSetGlobalPostgreSQLConfiguration checks whether cluster-wide API parameters // (like max_connections) have changed and if necessary sets it via the Patroni API -func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, effectivePatroniConfig, desiredPatroniConfig map[string]interface{}) (bool, error) { +func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, effectivePostgresConfig, desiredPostgresConfig map[string]interface{}) (bool, error) { - if reflect.DeepEqual(effectivePatroniConfig, desiredPatroniConfig) { + configUpdateRequired := false + for parameter, value := range desiredPostgresConfig { + if !reflect.DeepEqual(effectivePostgresConfig[parameter], value) { + configUpdateRequired = true + break + } + } + if !configUpdateRequired { return false, nil } - configToSetJson, err := json.Marshal(desiredPatroniConfig) + configToSetJson, err := json.Marshal(desiredPostgresConfig) if err != nil { c.logger.Debugf("could not convert config patch to JSON: %v", err) } @@ -521,7 +546,7 @@ func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, effectiv podName := util.NameFromMeta(pod.ObjectMeta) c.logger.Debugf("patching Postgres config via Patroni API on pod %s with following options: %s", podName, configToSetJson) - if err = c.patroni.SetConfig(pod, desiredPatroniConfig); err != nil { + if err = c.patroni.SetConfig(pod, desiredPostgresConfig); err != nil { return true, fmt.Errorf("could not patch postgres parameters within pod %s: %v", podName, err) } From 1f8cfc5c980d7abab86e29030c1e134769421278 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 11 Aug 2021 09:15:12 +0200 Subject: [PATCH 07/44] small fix for setting bools in Postgres config --- e2e/tests/test_e2e.py | 10 +++++----- pkg/cluster/sync.go | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 6a4bf78ca..ac9bb9398 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -1098,19 +1098,19 @@ def test_patroni_config_update(self): def compare_config(): effective_config = k8s.patroni_rest(masterPod.metadata.name, "config") - desired_patroni = pg_patch_config["spec"]["patroni"] + desired_config = pg_patch_config["spec"]["patroni"] desired_parameters = pg_patch_config["spec"]["postgresql"]["parameters"] effective_parameters = effective_config["postgresql"]["parameters"] self.assertEqual(desired_parameters["max_connections"], effective_parameters["max_connections"], "max_connections not updated") self.assertTrue(effective_config["slots"] is not None, "physical replication slot not added") - self.assertEqual(desired_patroni["ttl"], effective_config["ttl"], + self.assertEqual(desired_config["ttl"], effective_config["ttl"], "ttl not updated") - self.assertEqual(desired_patroni["loop_wait"], effective_config["loop_wait"], + self.assertEqual(desired_config["loop_wait"], effective_config["loop_wait"], "loop_wait not updated") - self.assertEqual(desired_patroni["retry_timeout"], effective_config["retry_timeout"], + self.assertEqual(desired_config["retry_timeout"], effective_config["retry_timeout"], "retry_timeout not updated") - self.assertEqual(desired_patroni["synchronous_mode"], effective_config["synchronous_mode"], + self.assertEqual(desired_config["synchronous_mode"], effective_config["synchronous_mode"], "synchronous_mode not updated") return True diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 0b33ade4d..de3d1ad11 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -416,10 +416,10 @@ func (c *Cluster) syncStatefulSet() error { if c.Spec.Patroni.TTL > 0 { desiredPostgresConfig["ttl"] = c.Spec.Patroni.TTL } - if effectivePostgresConfig["synchronous_mode"] != desiredPostgresConfig["synchronous_mode"] { + if effectivePostgresConfig["synchronous_mode"] != c.Spec.Patroni.SynchronousMode { desiredPostgresConfig["synchronous_mode"] = c.Spec.Patroni.SynchronousMode } - if effectivePostgresConfig["synchronous_mode_strict"] != desiredPostgresConfig["synchronous_mode_strict"] { + if effectivePostgresConfig["synchronous_mode_strict"] != c.Spec.Patroni.SynchronousModeStrict { desiredPostgresConfig["synchronous_mode_strict"] = c.Spec.Patroni.SynchronousModeStrict } From 54257b8249bbdce7f5ef42c518890cdf8d001305 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 17 Aug 2021 17:30:12 +0200 Subject: [PATCH 08/44] update docs and revert codegen --- docs/reference/cluster_manifest.md | 38 ++++++++++++++++++++++++++++++ hack/update-codegen.sh | 23 +++++++----------- 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 1b2d71a66..66867872c 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -473,3 +473,41 @@ Those parameters are grouped under the `tls` top-level key. relative to the "/tls/", which is mount path of the tls secret. If `caSecretName` is defined, the ca.crt path is relative to "/tlsca/", otherwise to the same "/tls/". + +## Change data capture streams + +This sections enables change data capture (CDC) streams e.g. into Zalando’s +distributed event broker [Nakadi](https://nakadi.io/). Parameters grouped +under the `streams` top-level key will be used by the operator to create a +CRD for Zalando's internal CDC operator. Each stream can have the following +properties: + +* **streamType** + Defines the sink. Either `nakadi`, `sqs` or `wal` (which is just plain wal + files). Required. + +* **database** + Name of the database from where events will be published via Postgres' + logical decoding feature. The operator will take care of updating the + database configuration (setting `wal_level: logical`, creating logical + replication slots, using output plugin `wal2json` and creating a dedicated + replication user). + +* **tables** + Defines a map of table names and event types. The CDC operator is following + the [outbox pattern](https://debezium.io/blog/2019/02/19/reliable-microservices-data-exchange-with-the-outbox-pattern/) + meaning changes are only consumed from an extra table that already has the + structure of the event in the target sink. The operator will assume that this + outbox table is called like `__outbox`. + +* **filter** + Streamed events can be filtered by a jsonpath expression for each table. + +* **batchSize** + Defines the size of batches in which events are consumed. + +* **sqsArn** + ARN to the SQS service used as event sink when streamType is `sqs`. + +* **queueName** + Name of the queue to be used in SQS service when streamType is `sqs`. diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh index 18c71f83c..7043053ea 100755 --- a/hack/update-codegen.sh +++ b/hack/update-codegen.sh @@ -1,18 +1,13 @@ #!/usr/bin/env bash -set -eou pipefail -GOPKG="github.com/zalando/postgres-operator" -SCRIPT_ROOT="$(dirname "${BASH_SOURCE[0]}")/.." +set -o errexit +set -o nounset +set -o pipefail -rm -rf "${SCRIPT_ROOT}/generated" +SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. +CODEGEN_PKG=${CODEGEN_PKG:-$(cd ${SCRIPT_ROOT}; ls -d -1 ./vendor/k8s.io/code-generator 2>/dev/null || echo ${GOPATH}/src/k8s.io/code-generator)} -go run k8s.io/code-generator/cmd/deepcopy-gen \ - --input-dirs ${GOPKG}/pkg/apis/acid.zalan.do/v1,${GOPKG}/pkg/apis/zalando.org/v1alpha1 \ - -O zz_generated.deepcopy \ - --bounding-dirs ${GOPKG}/pkg/apis \ - --go-header-file "${SCRIPT_ROOT}/hack/custom-boilerplate.go.txt" \ - -o "${SCRIPT_ROOT}/generated" - -cp -rv "${SCRIPT_ROOT}/generated/${GOPKG}"/* . - -rm -rf "${SCRIPT_ROOT}/generated" \ No newline at end of file +bash "${CODEGEN_PKG}/generate-groups.sh" all \ + github.com/zalando/postgres-operator/pkg/generated github.com/zalando/postgres-operator/pkg/apis \ + "acid.zalan.do:v1" \ + --go-header-file "${SCRIPT_ROOT}"/hack/custom-boilerplate.go.txt \ No newline at end of file From 6120ef7da414dc1a4590f2aa0b3bfc0dcfa49e44 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 20 Aug 2021 12:51:40 +0200 Subject: [PATCH 09/44] name FES like the Postgres cluster --- docs/reference/cluster_manifest.md | 4 ++-- pkg/cluster/streams.go | 4 ++-- pkg/cluster/streams_test.go | 3 +-- pkg/util/constants/streams.go | 1 - 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 66867872c..4b380dbf3 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -479,8 +479,8 @@ Those parameters are grouped under the `tls` top-level key. This sections enables change data capture (CDC) streams e.g. into Zalando’s distributed event broker [Nakadi](https://nakadi.io/). Parameters grouped under the `streams` top-level key will be used by the operator to create a -CRD for Zalando's internal CDC operator. Each stream can have the following -properties: +CRD for Zalando's internal CDC operator named like the Postgres cluster. +Each stream object can have the following properties: * **streamType** Defines the sink. Either `nakadi`, `sqs` or `wal` (which is just plain wal diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index bbfe17eca..d9cc19e1f 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -131,7 +131,7 @@ func (c *Cluster) generateFabricEventStream() *zalandov1alpha1.FabricEventStream return &zalandov1alpha1.FabricEventStream{ ObjectMeta: metav1.ObjectMeta{ - Name: c.Name + constants.FESsuffix, + Name: c.Name, Namespace: c.Namespace, Annotations: c.AnnotationsToPropagate(c.annotationsSet(nil)), }, @@ -239,7 +239,7 @@ func (c *Cluster) syncStreams() error { return fmt.Errorf("could not update Postgres config for event streaming: %v", err) } - effectiveStreams, err := c.KubeClient.FabricEventStreamsGetter.FabricEventStreams(c.Namespace).Get(context.TODO(), c.Name+constants.FESsuffix, metav1.GetOptions{}) + effectiveStreams, err := c.KubeClient.FabricEventStreamsGetter.FabricEventStreams(c.Namespace).Get(context.TODO(), c.Name, metav1.GetOptions{}) if err != nil { if !k8sutil.ResourceNotFound(err) { return fmt.Errorf("error during reading of event streams: %v", err) diff --git a/pkg/cluster/streams_test.go b/pkg/cluster/streams_test.go index 8119168b0..5ba6a5147 100644 --- a/pkg/cluster/streams_test.go +++ b/pkg/cluster/streams_test.go @@ -10,7 +10,6 @@ import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" fakezalandov1alpha1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake" "github.com/zalando/postgres-operator/pkg/util/config" - "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -93,7 +92,7 @@ func TestGenerateFabricEventStream(t *testing.T) { err := cluster.syncStreams() assert.NoError(t, err) - streamCRD, err := cluster.KubeClient.FabricEventStreams(namespace).Get(context.TODO(), cluster.Name+constants.FESsuffix, metav1.GetOptions{}) + streamCRD, err := cluster.KubeClient.FabricEventStreams(namespace).Get(context.TODO(), cluster.Name, metav1.GetOptions{}) assert.NoError(t, err) result := cluster.generateFabricEventStream() diff --git a/pkg/util/constants/streams.go b/pkg/util/constants/streams.go index 7a2c9f81f..a7ea2b0b8 100644 --- a/pkg/util/constants/streams.go +++ b/pkg/util/constants/streams.go @@ -2,7 +2,6 @@ package constants // PostgreSQL specific constants const ( - FESsuffix = "-event-streams" EventStreamSourcePGType = "PostgresLogicalReplication" EventStreamSourceSlotPrefix = "fes_" EventStreamSourceAuthType = "DatabaseAuthenticationSecret" From 2726492fd30eb11b4cfb961442b52f75f4a18947 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 23 Aug 2021 15:14:11 +0200 Subject: [PATCH 10/44] add delete case and fix updating streams + update unit test --- pkg/cluster/cluster.go | 4 ++ pkg/cluster/database.go | 38 ------------------ pkg/cluster/streams.go | 37 ++++++----------- pkg/cluster/streams_test.go | 79 ++++++++++++++++++++++++++++++++++--- 4 files changed, 89 insertions(+), 69 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 977c3c4e5..609066767 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -899,6 +899,10 @@ func (c *Cluster) Delete() { defer c.mu.Unlock() c.eventRecorder.Event(c.GetReference(), v1.EventTypeNormal, "Delete", "Started deletion of new cluster resources") + if err := c.deleteStreams(); err != nil { + c.logger.Warningf("could not delete event streams: %v", err) + } + // delete the backup job before the stateful set of the cluster to prevent connections to non-existing pods // deleting the cron job also removes pods and batch jobs it created if err := c.deleteLogicalBackupJob(); err != nil { diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index 6e0a8ae30..ba4cf223a 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -33,8 +33,6 @@ const ( getExtensionsSQL = `SELECT e.extname, n.nspname FROM pg_catalog.pg_extension e LEFT JOIN pg_catalog.pg_namespace n ON n.oid = e.extnamespace ORDER BY 1;` - tableExistsSQL = `SELECT TRUE FROM pg_tables WHERE tablename = $1 AND schemaname = $2;` - createDatabaseSQL = `CREATE DATABASE "%s" OWNER "%s";` createDatabaseSchemaSQL = `SET ROLE TO "%s"; CREATE SCHEMA IF NOT EXISTS "%s" AUTHORIZATION "%s"` alterDatabaseOwnerSQL = `ALTER DATABASE "%s" OWNER TO "%s";` @@ -508,42 +506,6 @@ func (c *Cluster) execCreateOrAlterExtension(extName, schemaName, statement, doi return nil } -// getExtension returns the list of current database extensions -// The caller is responsible for opening and closing the database connection -func (c *Cluster) tableExists(tableName, schemaName string) (bool, error) { - var ( - rows *sql.Rows - exists bool - err error - ) - - if rows, err = c.pgDb.Query(tableExistsSQL, tableName, schemaName); err != nil { - return false, fmt.Errorf("could not check table for existence: %v", err) - } - - defer func() { - if err2 := rows.Close(); err2 != nil { - if err != nil { - err = fmt.Errorf("error when closing query cursor: %v, previous error: %v", err2, err) - } else { - err = fmt.Errorf("error when closing query cursor: %v", err2) - } - } - }() - - for rows.Next() { - if err = rows.Scan(&exists); err != nil { - return false, fmt.Errorf("error when processing row: %v", err) - } - } - - if exists { - return true, nil - } else { - return false, fmt.Errorf("table %s not found", schemaName+"."+tableName) - } -} - // Creates a connection pool credentials lookup function in every database to // perform remote authentication. func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string, role PostgresRole) error { diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index d9cc19e1f..99a4513d6 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -40,6 +40,17 @@ func (c *Cluster) updateStreams(newEventStreams *zalandov1alpha1.FabricEventStre return nil } +func (c *Cluster) deleteStreams() error { + c.setProcessName("updating event streams") + + err := c.KubeClient.FabricEventStreamsGetter.FabricEventStreams(c.Namespace).Delete(context.TODO(), c.Name, metav1.DeleteOptions{}) + if err != nil { + return fmt.Errorf("could not delete event stream custom resource: %v", err) + } + + return nil +} + func (c *Cluster) syncPostgresConfig() error { desiredPostgresConfig := make(map[string]interface{}) @@ -90,29 +101,6 @@ func (c *Cluster) syncPostgresConfig() error { return nil } -func (c *Cluster) syncStreamDbResources() error { - - for _, stream := range c.Spec.Streams { - if err := c.initDbConnWithName(stream.Database); err != nil { - return fmt.Errorf("could not init connection to database %q specified for event stream: %v", stream.Database, err) - } - - for table, eventType := range stream.Tables { - tableName, schemaName := getTableSchema(table) - if exists, err := c.tableExists(tableName, schemaName); !exists { - return fmt.Errorf("could not find table %q specified for event stream: %v", table, err) - } - // check if outbox table exists and if not, create it - outboxTable := outboxTableNameTemplate.Format("table", tableName, "eventtype", eventType) - if exists, err := c.tableExists(outboxTable, schemaName); !exists { - return fmt.Errorf("could not find outbox table %q specified for event stream: %v", outboxTable, err) - } - } - } - - return nil -} - func (c *Cluster) generateFabricEventStream() *zalandov1alpha1.FabricEventStream { eventStreams := make([]zalandov1alpha1.EventStream, 0) @@ -251,12 +239,11 @@ func (c *Cluster) syncStreams() error { return fmt.Errorf("could not create missing streams: %v", err) } } else { - err := c.syncStreamDbResources() if err != nil { c.logger.Warnf("database setup might be incomplete : %v", err) } desiredStreams := c.generateFabricEventStream() - if reflect.DeepEqual(effectiveStreams.Spec, desiredStreams.Spec) { + if !reflect.DeepEqual(effectiveStreams.Spec, desiredStreams.Spec) { c.updateStreams(desiredStreams) } } diff --git a/pkg/cluster/streams_test.go b/pkg/cluster/streams_test.go index 5ba6a5147..2db9c149b 100644 --- a/pkg/cluster/streams_test.go +++ b/pkg/cluster/streams_test.go @@ -1,6 +1,7 @@ package cluster import ( + "encoding/json" "reflect" "context" @@ -13,6 +14,7 @@ import ( "github.com/zalando/postgres-operator/pkg/util/k8sutil" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/fake" ) @@ -22,16 +24,19 @@ func newFakeK8sStreamClient() (k8sutil.KubernetesClient, *fake.Clientset) { return k8sutil.KubernetesClient{ FabricEventStreamsGetter: zalandoClientSet.ZalandoV1alpha1(), + PostgresqlsGetter: zalandoClientSet.AcidV1(), PodsGetter: clientSet.CoreV1(), }, clientSet } -func TestGenerateFabricEventStream(t *testing.T) { - client, _ := newFakeK8sStreamClient() - clusterName := "acid-test-cluster" - namespace := "default" - - pg := acidv1.Postgresql{ +var ( + clusterName string = "acid-test-cluster" + namespace string = "default" + pg = acidv1.Postgresql{ + TypeMeta: metav1.TypeMeta{ + Kind: "Postgresql", + APIVersion: "acid.zalan.do/v1", + }, ObjectMeta: metav1.ObjectMeta{ Name: clusterName, Namespace: namespace, @@ -72,6 +77,10 @@ func TestGenerateFabricEventStream(t *testing.T) { }, }, } +) + +func TestGenerateFabricEventStream(t *testing.T) { + client, _ := newFakeK8sStreamClient() var cluster = New( Config{ @@ -100,3 +109,61 @@ func TestGenerateFabricEventStream(t *testing.T) { t.Errorf("Malformed FabricEventStream, expected %#v, got %#v", streamCRD, result) } } + +func TestUpdateFabricEventStream(t *testing.T) { + client, _ := newFakeK8sStreamClient() + + var cluster = New( + Config{ + OpConfig: config.Config{ + PodManagementPolicy: "ordered_ready", + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + DefaultCPURequest: "300m", + DefaultCPULimit: "300m", + DefaultMemoryRequest: "300Mi", + DefaultMemoryLimit: "300Mi", + PodRoleLabel: "spilo-role", + }, + }, + }, client, pg, logger, eventRecorder) + + _, err := cluster.KubeClient.Postgresqls(namespace).Create( + context.TODO(), &pg, metav1.CreateOptions{}) + assert.NoError(t, err) + err = cluster.syncStreams() + assert.NoError(t, err) + + var pgSpec acidv1.PostgresSpec + pgSpec.Streams = []acidv1.Stream{ + { + StreamType: "nakadi", + Database: "foo", + Tables: map[string]string{ + "bar": "stream_type_b", + }, + BatchSize: uint32(250), + }, + } + patch, err := json.Marshal(struct { + PostgresSpec interface{} `json:"spec"` + }{&pgSpec}) + assert.NoError(t, err) + + pgPatched, err := cluster.KubeClient.Postgresqls(namespace).Patch( + context.TODO(), cluster.Name, types.MergePatchType, patch, metav1.PatchOptions{}, "spec") + assert.NoError(t, err) + + cluster.Postgresql.Spec = pgPatched.Spec + err = cluster.syncStreams() + assert.NoError(t, err) + + streamCRD, err := cluster.KubeClient.FabricEventStreams(namespace).Get(context.TODO(), cluster.Name, metav1.GetOptions{}) + assert.NoError(t, err) + + result := cluster.generateFabricEventStream() + if !reflect.DeepEqual(result, streamCRD) { + t.Errorf("Malformed FabricEventStream, expected %#v, got %#v", streamCRD, result) + } +} From 7a84c0852c66a9e9c9330cef719fb3e8ff7030d6 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 25 Aug 2021 18:15:19 +0200 Subject: [PATCH 11/44] remove wal type and distinguish source between nakadi and sqs --- .../postgres-operator/crds/postgresqls.yaml | 1 - manifests/complete-postgres-manifest.yaml | 16 ++++++++++ manifests/postgresql.crd.yaml | 1 - pkg/apis/acid.zalan.do/v1/crds.go | 3 -- .../zalando.org/v1alpha1/fabriceventstream.go | 2 +- pkg/cluster/streams.go | 31 ++++++++++++++----- pkg/cluster/streams_test.go | 11 ++----- 7 files changed, 44 insertions(+), 21 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 4f6028f6d..743f75470 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -499,7 +499,6 @@ spec: enum: - "nakadi" - "sqs" - - "wal" teamId: type: string tls: diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 6e2acbdd3..30212618b 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -189,3 +189,19 @@ spec: # operator: In # values: # - enabled + +# Enables change data capture streams for defined database tables +# streams: +# - type: nakadi +# batchSize: 100 +# database: foo +# tables: +# ta: event_type_a +# tb: event_type_b +# - type: sqs +# database: foo +# tables: +# ta: "" +# tb: "" +# sqsArn: arn:aws:sqs:eu-central-1:111122223333 +# queueName: foo-queue diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 41511c955..b6fc64c80 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -495,7 +495,6 @@ spec: enum: - "nakadi" - "sqs" - - "wal" teamId: type: string tls: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index d0c4fa103..617a219f9 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -701,9 +701,6 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ { Raw: []byte(`"sqs"`), }, - { - Raw: []byte(`"wal"`), - }, }, }, }, diff --git a/pkg/apis/zalando.org/v1alpha1/fabriceventstream.go b/pkg/apis/zalando.org/v1alpha1/fabriceventstream.go index 37f1449c9..fbfad76f1 100644 --- a/pkg/apis/zalando.org/v1alpha1/fabriceventstream.go +++ b/pkg/apis/zalando.org/v1alpha1/fabriceventstream.go @@ -69,7 +69,7 @@ type EventStreamSource struct { // EventStreamTable defines the name and ID column to be used for streaming type EventStreamTable struct { Name string `json:"name"` - IDColumn string `json:"idColumn,omitempty" defaults:"id"` + IDColumn string `json:"idColumn,omitempty"` } // Connection to be used for allowing the FES operator to connect to a database diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index 99a4513d6..72e03fb20 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -131,15 +131,26 @@ func (c *Cluster) generateFabricEventStream() *zalandov1alpha1.FabricEventStream } func (c *Cluster) getEventStreamSource(stream acidv1.Stream, table, eventType string) zalandov1alpha1.EventStreamSource { - streamFilter := stream.Filter[table] _, schema := getTableSchema(table) - return zalandov1alpha1.EventStreamSource{ - Type: constants.EventStreamSourcePGType, - Schema: schema, - EventStreamTable: getOutboxTable(table, eventType), - Filter: streamFilter, - Connection: c.getStreamConnection(stream.Database, constants.EventStreamSourceSlotPrefix+constants.UserRoleNameSuffix), + switch stream.StreamType { + case "nakadi": + streamFilter := stream.Filter[table] + return zalandov1alpha1.EventStreamSource{ + Type: constants.EventStreamSourcePGType, + Schema: schema, + EventStreamTable: getOutboxTable(table, eventType), + Filter: streamFilter, + Connection: c.getStreamConnection(stream.Database, constants.EventStreamSourceSlotPrefix+constants.UserRoleNameSuffix), + } + case "sqs": + return zalandov1alpha1.EventStreamSource{ + Type: constants.EventStreamSourcePGType, + EventStreamTable: getSqsTable(table), + Connection: c.getStreamConnection(stream.Database, constants.EventStreamSourceSlotPrefix+constants.UserRoleNameSuffix), + } } + + return zalandov1alpha1.EventStreamSource{} } func getEventStreamFlow(stream acidv1.Stream) zalandov1alpha1.EventStreamFlow { @@ -195,6 +206,12 @@ func getOutboxTable(tableName, eventType string) zalandov1alpha1.EventStreamTabl } } +func getSqsTable(tableName string) zalandov1alpha1.EventStreamTable { + return zalandov1alpha1.EventStreamTable{ + Name: outboxTableNameTemplate.Format("table", tableName), + } +} + func (c *Cluster) getStreamConnection(database, user string) zalandov1alpha1.Connection { return zalandov1alpha1.Connection{ Url: fmt.Sprintf("jdbc:postgresql://%s.%s/%s?user=%s&ssl=true&sslmode=require", c.Name, c.Namespace, database, user), diff --git a/pkg/cluster/streams_test.go b/pkg/cluster/streams_test.go index 2db9c149b..71d263025 100644 --- a/pkg/cluster/streams_test.go +++ b/pkg/cluster/streams_test.go @@ -55,18 +55,13 @@ var ( BatchSize: uint32(100), }, { - StreamType: "wal", + StreamType: "sqs", Database: "foo", Tables: map[string]string{ "bar": "stream_type_a", }, - BatchSize: uint32(100), - }, - { - StreamType: "sqs", - Database: "foo", - SqsArn: "arn:aws:sqs:eu-central-1:111122223333", - QueueName: "foo-queue", + SqsArn: "arn:aws:sqs:eu-central-1:111122223333", + QueueName: "foo-queue", }, }, Users: map[string]acidv1.UserFlags{ From d2addd3b17aa5a3b0b3be65c0190314bf24b9850 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 27 Aug 2021 13:00:38 +0200 Subject: [PATCH 12/44] re-add 3rd type, remove callHome flow --- .../postgres-operator/crds/postgresqls.yaml | 3 ++ manifests/complete-postgres-manifest.yaml | 7 ++++ manifests/postgresql.crd.yaml | 3 ++ pkg/apis/acid.zalan.do/v1/crds.go | 6 +++ pkg/apis/acid.zalan.do/v1/postgresql_type.go | 1 + .../zalando.org/v1alpha1/fabriceventstream.go | 2 + pkg/cluster/streams.go | 38 ++++++++++++------- pkg/cluster/streams_test.go | 8 ++++ pkg/util/constants/streams.go | 25 ++++++------ 9 files changed, 68 insertions(+), 25 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 743f75470..40474dd56 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -490,6 +490,8 @@ spec: type: string sqsArn: type: string + sqsFifo: + type: boolean tables: type: object additionalProperties: @@ -499,6 +501,7 @@ spec: enum: - "nakadi" - "sqs" + - "wal" teamId: type: string tls: diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 30212618b..74c68ae36 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -198,10 +198,17 @@ spec: # tables: # ta: event_type_a # tb: event_type_b +# - type: wal +# batchSize: 100 +# database: foo +# tables: +# public.tx: event_type_a +# public.ty: event_type_b # - type: sqs # database: foo # tables: # ta: "" # tb: "" # sqsArn: arn:aws:sqs:eu-central-1:111122223333 +# sqsFifo: true # queueName: foo-queue diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index b6fc64c80..4086ab1d7 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -486,6 +486,8 @@ spec: type: string sqsArn: type: string + sqsFifo: + type: boolean tables: type: object additionalProperties: @@ -495,6 +497,7 @@ spec: enum: - "nakadi" - "sqs" + - "wal" teamId: type: string tls: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 617a219f9..787d3a236 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -684,6 +684,9 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ "sqsArn": { Type: "string", }, + "sqsFifo": { + Type: "boolean", + }, "tables": { Type: "object", AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ @@ -701,6 +704,9 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ { Raw: []byte(`"sqs"`), }, + { + Raw: []byte(`"wal"`), + }, }, }, }, diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index a4a8f4477..04d36276b 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -233,5 +233,6 @@ type Stream struct { Filter map[string]string `json:"filter,omitempty"` BatchSize uint32 `json:"batchSize,omitempty"` SqsArn string `json:"sqsArn,omitempty"` + SqsFifo bool `json:"sqsFifo,omitempty"` QueueName string `json:"queueName,omitempty"` } diff --git a/pkg/apis/zalando.org/v1alpha1/fabriceventstream.go b/pkg/apis/zalando.org/v1alpha1/fabriceventstream.go index fbfad76f1..36f68ca2f 100644 --- a/pkg/apis/zalando.org/v1alpha1/fabriceventstream.go +++ b/pkg/apis/zalando.org/v1alpha1/fabriceventstream.go @@ -45,6 +45,7 @@ type EventStreamFlow struct { DataOpColumn string `json:"dataOpColumn,omitempty"` MetadataColumn string `json:"metadataColumn,omitempty"` DataColumn string `json:"dataColumn,omitempty"` + PayloadColumn string `json:"payloadColumn,omitempty"` CallHomeIdColumn string `json:"callHomeIdColumn,omitempty"` CallHomeUrl string `json:"callHomeUrl,omitempty"` } @@ -55,6 +56,7 @@ type EventStreamSink struct { EventType string `json:"eventType,omitempty"` MaxBatchSize uint32 `json:"maxBatchSize,omitempty"` QueueName string `json:"queueName,omitempty"` + QueueUrl string `json:"queueUrl,omitempty"` } // EventStreamSource defines the source of the event stream and connection for FES operator diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index 72e03fb20..4134e1723 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -142,10 +142,11 @@ func (c *Cluster) getEventStreamSource(stream acidv1.Stream, table, eventType st Filter: streamFilter, Connection: c.getStreamConnection(stream.Database, constants.EventStreamSourceSlotPrefix+constants.UserRoleNameSuffix), } - case "sqs": + case "default": return zalandov1alpha1.EventStreamSource{ Type: constants.EventStreamSourcePGType, - EventStreamTable: getSqsTable(table), + Schema: schema, + EventStreamTable: getSourceTable(table), Connection: c.getStreamConnection(stream.Database, constants.EventStreamSourceSlotPrefix+constants.UserRoleNameSuffix), } } @@ -161,12 +162,13 @@ func getEventStreamFlow(stream acidv1.Stream) zalandov1alpha1.EventStreamFlow { DataTypeColumn: constants.EventStreamFlowDataTypeColumn, DataOpColumn: constants.EventStreamFlowDataOpColumn, MetadataColumn: constants.EventStreamFlowMetadataColumn, - DataColumn: constants.EventStreamFlowDataColumn} - case "sqs": + DataColumn: constants.EventStreamFlowDataColumn, + } + case "default": return zalandov1alpha1.EventStreamFlow{ - Type: constants.EventStreamFlowPgApiType, - CallHomeIdColumn: "id", - CallHomeUrl: stream.SqsArn} + Type: constants.EventStreamFlowPgGenericType, + PayloadColumn: constants.EventStreamFlowPayloadColumn, + } } return zalandov1alpha1.EventStreamFlow{} @@ -174,15 +176,23 @@ func getEventStreamFlow(stream acidv1.Stream) zalandov1alpha1.EventStreamFlow { func getEventStreamSink(stream acidv1.Stream, eventType string) zalandov1alpha1.EventStreamSink { switch stream.StreamType { - case "nakadi": + case "sqs": + sqsSinkType := constants.EventStreamSinkSqsStandardType + if stream.SqsFifo { + sqsSinkType = constants.EventStreamSinkSqsFifoType + } + return zalandov1alpha1.EventStreamSink{ + Type: sqsSinkType, + QueueName: stream.QueueName, + QueueUrl: stream.SqsArn, + MaxBatchSize: stream.BatchSize, + } + case "default": return zalandov1alpha1.EventStreamSink{ Type: constants.EventStreamSinkNakadiType, EventType: eventType, - MaxBatchSize: stream.BatchSize} - case "sqs": - return zalandov1alpha1.EventStreamSink{ - Type: constants.EventStreamSinkSqsType, - QueueName: stream.QueueName} + MaxBatchSize: stream.BatchSize, + } } return zalandov1alpha1.EventStreamSink{} @@ -206,7 +216,7 @@ func getOutboxTable(tableName, eventType string) zalandov1alpha1.EventStreamTabl } } -func getSqsTable(tableName string) zalandov1alpha1.EventStreamTable { +func getSourceTable(tableName string) zalandov1alpha1.EventStreamTable { return zalandov1alpha1.EventStreamTable{ Name: outboxTableNameTemplate.Format("table", tableName), } diff --git a/pkg/cluster/streams_test.go b/pkg/cluster/streams_test.go index 71d263025..78c1a4ec7 100644 --- a/pkg/cluster/streams_test.go +++ b/pkg/cluster/streams_test.go @@ -54,6 +54,14 @@ var ( }, BatchSize: uint32(100), }, + { + StreamType: "wal", + Database: "foo", + Tables: map[string]string{ + "bar": "stream_type_a", + }, + BatchSize: uint32(100), + }, { StreamType: "sqs", Database: "foo", diff --git a/pkg/util/constants/streams.go b/pkg/util/constants/streams.go index a7ea2b0b8..3eb28b5f3 100644 --- a/pkg/util/constants/streams.go +++ b/pkg/util/constants/streams.go @@ -2,15 +2,18 @@ package constants // PostgreSQL specific constants const ( - EventStreamSourcePGType = "PostgresLogicalReplication" - EventStreamSourceSlotPrefix = "fes_" - EventStreamSourceAuthType = "DatabaseAuthenticationSecret" - EventStreamFlowPgNakadiType = "PostgresWalToNakadiDataEvent" - EventStreamFlowPgApiType = "PostgresWalToApiCallHomeEvent" - EventStreamFlowDataTypeColumn = "data_type" - EventStreamFlowDataOpColumn = "data_op" - EventStreamFlowMetadataColumn = "metadata" - EventStreamFlowDataColumn = "data" - EventStreamSinkNakadiType = "Nakadi" - EventStreamSinkSqsType = "Sqs" + EventStreamSourcePGType = "PostgresLogicalReplication" + EventStreamSourceSlotPrefix = "fes_" + EventStreamSourceAuthType = "DatabaseAuthenticationSecret" + EventStreamFlowPgNakadiType = "PostgresWalToNakadiDataEvent" + EventStreamFlowPgGenericType = "PostgresWalToGenericNakadiEvent" + EventStreamFlowPgApiType = "PostgresWalToApiCallHomeEvent" + EventStreamFlowDataTypeColumn = "data_type" + EventStreamFlowDataOpColumn = "data_op" + EventStreamFlowMetadataColumn = "metadata" + EventStreamFlowDataColumn = "data" + EventStreamFlowPayloadColumn = "payload" + EventStreamSinkNakadiType = "Nakadi" + EventStreamSinkSqsStandardType = "SqsStandard" + EventStreamSinkSqsFifoType = "SqsFifo" ) From e4c7e3648f4f5849a026d99bc2a21c78d96dcbce Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 16 Sep 2021 15:58:11 +0200 Subject: [PATCH 13/44] remove sqs and a few bugs --- .../postgres-operator/crds/postgresqls.yaml | 9 ------ docs/reference/cluster_manifest.md | 11 ++----- manifests/complete-postgres-manifest.yaml | 20 ++++-------- manifests/operator-service-account-rbac.yaml | 2 +- manifests/postgresql.crd.yaml | 9 ------ pkg/apis/acid.zalan.do/v1/crds.go | 15 +-------- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 7 ++--- pkg/cluster/cluster.go | 6 ++-- pkg/cluster/streams.go | 31 ++++++------------- pkg/cluster/streams_test.go | 9 ------ pkg/cluster/sync.go | 14 ++++++--- pkg/util/constants/streams.go | 26 +++++++--------- 12 files changed, 47 insertions(+), 112 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index fdfc67775..cf4cd4251 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -477,8 +477,6 @@ spec: nullable: true items: type: object - required: - - streamType properties: batchSize: type: integer @@ -488,12 +486,6 @@ spec: type: object additionalProperties: type: string - queueName: - type: string - sqsArn: - type: string - sqsFifo: - type: boolean tables: type: object additionalProperties: @@ -502,7 +494,6 @@ spec: type: string enum: - "nakadi" - - "sqs" - "wal" teamId: type: string diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index aef50c7f1..f879e7938 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -523,8 +523,9 @@ CRD for Zalando's internal CDC operator named like the Postgres cluster. Each stream object can have the following properties: * **streamType** - Defines the sink. Either `nakadi`, `sqs` or `wal` (which is just plain wal - files). Required. + Defines the stream flow. Choose `nakadi` when you want to specify certain + nakadi event types of or `wal` if changes should be mapped to a generic + event type. Default is `wal`. * **database** Name of the database from where events will be published via Postgres' @@ -545,9 +546,3 @@ Each stream object can have the following properties: * **batchSize** Defines the size of batches in which events are consumed. - -* **sqsArn** - ARN to the SQS service used as event sink when streamType is `sqs`. - -* **queueName** - Name of the queue to be used in SQS service when streamType is `sqs`. diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 81e26c10b..68977bbcd 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -198,23 +198,15 @@ spec: # Enables change data capture streams for defined database tables # streams: -# - type: nakadi +# - streamType: nakadi # batchSize: 100 # database: foo # tables: -# ta: event_type_a -# tb: event_type_b -# - type: wal +# ta: event_type_a +# tb: event_type_b +# - streamType: wal # batchSize: 100 # database: foo # tables: -# public.tx: event_type_a -# public.ty: event_type_b -# - type: sqs -# database: foo -# tables: -# ta: "" -# tb: "" -# sqsArn: arn:aws:sqs:eu-central-1:111122223333 -# sqsFifo: true -# queueName: foo-queue +# public.tx: event_type_a +# public.ty: event_type_b diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index 48da5f06a..e983d71a0 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -39,7 +39,7 @@ rules: - apiGroups: - zalando.org resources: - - fabriceventstream + - fabriceventstreams verbs: - create - delete diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 1fc508d7e..e2aa227a0 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -473,8 +473,6 @@ spec: nullable: true items: type: object - required: - - streamType properties: batchSize: type: integer @@ -484,12 +482,6 @@ spec: type: object additionalProperties: type: string - queueName: - type: string - sqsArn: - type: string - sqsFifo: - type: boolean tables: type: object additionalProperties: @@ -498,7 +490,6 @@ spec: type: string enum: - "nakadi" - - "sqs" - "wal" teamId: type: string diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 171b768d8..c3ee7d81d 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -664,8 +664,7 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Type: "array", Items: &apiextv1.JSONSchemaPropsOrArray{ Schema: &apiextv1.JSONSchemaProps{ - Type: "object", - Required: []string{"streamType"}, + Type: "object", Properties: map[string]apiextv1.JSONSchemaProps{ "batchSize": { Type: "integer", @@ -681,15 +680,6 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ }, }, }, - "queueName": { - Type: "string", - }, - "sqsArn": { - Type: "string", - }, - "sqsFifo": { - Type: "boolean", - }, "tables": { Type: "object", AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ @@ -704,9 +694,6 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ { Raw: []byte(`"nakadi"`), }, - { - Raw: []byte(`"sqs"`), - }, { Raw: []byte(`"wal"`), }, diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index efaca70cf..af45d7c04 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -74,7 +74,7 @@ type PostgresSpec struct { ServiceAnnotations map[string]string `json:"serviceAnnotations,omitempty"` TLS *TLSDescription `json:"tls,omitempty"` AdditionalVolumes []AdditionalVolume `json:"additionalVolumes,omitempty"` - Streams []Stream `json:"stream,omitempty"` + Streams []Stream `json:"streams,omitempty"` // deprecated json tags InitContainersOld []v1.Container `json:"init_containers,omitempty"` @@ -229,12 +229,9 @@ type ConnectionPooler struct { } type Stream struct { - StreamType string `json:"streamType"` + StreamType string `json:"streamType,omitempty"` Database string `json:"database,omitempty"` Tables map[string]string `json:"tables,omitempty"` Filter map[string]string `json:"filter,omitempty"` BatchSize uint32 `json:"batchSize,omitempty"` - SqsArn string `json:"sqsArn,omitempty"` - SqsFifo bool `json:"sqsFifo,omitempty"` - QueueName string `json:"queueName,omitempty"` } diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 924f1a737..569ba463c 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -361,8 +361,8 @@ func (c *Cluster) Create() error { // something fails, report warning c.createConnectionPooler(c.installLookupFunction) - if len(c.Spec.Streams) > 0 { - c.syncStreams() + if err = c.syncStreams(); err != nil { + return fmt.Errorf("could not create streams: %v", err) } return nil @@ -1060,7 +1060,7 @@ func (c *Cluster) initSystemUsers() { // replication users for event streams are another exception // the operator will create one replication user for all streams if len(c.Spec.Streams) > 0 { - username := constants.EventStreamSourceSlotPrefix + constants.UserRoleNameSuffix + username := constants.EventStreamSourceSlotPrefix + "user" streamUser := spec.PgUser{ Origin: spec.RoleConnectionPooler, Name: username, diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index 4134e1723..5d33fa1c3 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -56,14 +56,13 @@ func (c *Cluster) syncPostgresConfig() error { desiredPostgresConfig := make(map[string]interface{}) slots := make(map[string]map[string]string) - c.logger.Debugf("setting wal level to 'logical' in postgres configuration") + // if streams are defined wal_level must be switched to logical and slots have to be defined desiredPostgresConfig["postgresql"] = map[string]interface{}{patroniPGParametersParameterName: map[string]string{"wal_level": "logical"}} for _, stream := range c.Spec.Streams { slotName := c.getLogicalReplicationSlot(stream.Database) if slotName == "" { - c.logger.Debugf("creating logical replication slot %q in database %q", constants.EventStreamSourceSlotPrefix+stream.Database, stream.Database) slot := map[string]string{ "database": stream.Database, "plugin": "wal2json", @@ -74,6 +73,10 @@ func (c *Cluster) syncPostgresConfig() error { } if len(slots) > 0 { + c.logger.Debugf("setting wal level to 'logical' in Postgres configuration") + for slotName, slot := range slots { + c.logger.Debugf("creating logical replication slot %q in database %q", slotName, slot["database"]) + } desiredPostgresConfig["slots"] = slots } else { return nil @@ -175,27 +178,11 @@ func getEventStreamFlow(stream acidv1.Stream) zalandov1alpha1.EventStreamFlow { } func getEventStreamSink(stream acidv1.Stream, eventType string) zalandov1alpha1.EventStreamSink { - switch stream.StreamType { - case "sqs": - sqsSinkType := constants.EventStreamSinkSqsStandardType - if stream.SqsFifo { - sqsSinkType = constants.EventStreamSinkSqsFifoType - } - return zalandov1alpha1.EventStreamSink{ - Type: sqsSinkType, - QueueName: stream.QueueName, - QueueUrl: stream.SqsArn, - MaxBatchSize: stream.BatchSize, - } - case "default": - return zalandov1alpha1.EventStreamSink{ - Type: constants.EventStreamSinkNakadiType, - EventType: eventType, - MaxBatchSize: stream.BatchSize, - } + return zalandov1alpha1.EventStreamSink{ + Type: constants.EventStreamSinkNakadiType, + EventType: eventType, + MaxBatchSize: stream.BatchSize, } - - return zalandov1alpha1.EventStreamSink{} } func getTableSchema(fullTableName string) (tableName, schemaName string) { diff --git a/pkg/cluster/streams_test.go b/pkg/cluster/streams_test.go index 78c1a4ec7..5976d97b3 100644 --- a/pkg/cluster/streams_test.go +++ b/pkg/cluster/streams_test.go @@ -62,15 +62,6 @@ var ( }, BatchSize: uint32(100), }, - { - StreamType: "sqs", - Database: "foo", - Tables: map[string]string{ - "bar": "stream_type_a", - }, - SqsArn: "arn:aws:sqs:eu-central-1:111122223333", - QueueName: "foo-queue", - }, }, Users: map[string]acidv1.UserFlags{ "foo_user": []string{"replication"}, diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index cad1c4a23..45d58f73a 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -72,7 +72,7 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { return err } - c.logger.Debugf("syncing statefulsets") + c.logger.Debug("syncing statefulsets") if err = c.syncStatefulSet(); err != nil { if !k8sutil.ResourceAlreadyExists(err) { err = fmt.Errorf("could not sync statefulsets: %v", err) @@ -98,17 +98,17 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { // create database objects unless we are running without pods or disabled that feature explicitly if !(c.databaseAccessDisabled() || c.getNumberOfInstances(&newSpec.Spec) <= 0 || c.Spec.StandbyCluster != nil) { - c.logger.Debugf("syncing roles") + c.logger.Debug("syncing roles") if err = c.syncRoles(); err != nil { err = fmt.Errorf("could not sync roles: %v", err) return err } - c.logger.Debugf("syncing databases") + c.logger.Debug("syncing databases") if err = c.syncDatabases(); err != nil { err = fmt.Errorf("could not sync databases: %v", err) return err } - c.logger.Debugf("syncing prepared databases with schemas") + c.logger.Debug("syncing prepared databases with schemas") if err = c.syncPreparedDatabases(); err != nil { err = fmt.Errorf("could not sync prepared database: %v", err) return err @@ -120,6 +120,12 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { return fmt.Errorf("could not sync connection pooler: %v", err) } + c.logger.Debug("syncing streams") + if err = c.syncStreams(); err != nil { + err = fmt.Errorf("could not sync streams: %v", err) + return err + } + // Major version upgrade must only run after success of all earlier operations, must remain last item in sync if err := c.majorVersionUpgrade(); err != nil { c.logger.Errorf("major version upgrade failed: %v", err) diff --git a/pkg/util/constants/streams.go b/pkg/util/constants/streams.go index 3eb28b5f3..1a8028747 100644 --- a/pkg/util/constants/streams.go +++ b/pkg/util/constants/streams.go @@ -2,18 +2,16 @@ package constants // PostgreSQL specific constants const ( - EventStreamSourcePGType = "PostgresLogicalReplication" - EventStreamSourceSlotPrefix = "fes_" - EventStreamSourceAuthType = "DatabaseAuthenticationSecret" - EventStreamFlowPgNakadiType = "PostgresWalToNakadiDataEvent" - EventStreamFlowPgGenericType = "PostgresWalToGenericNakadiEvent" - EventStreamFlowPgApiType = "PostgresWalToApiCallHomeEvent" - EventStreamFlowDataTypeColumn = "data_type" - EventStreamFlowDataOpColumn = "data_op" - EventStreamFlowMetadataColumn = "metadata" - EventStreamFlowDataColumn = "data" - EventStreamFlowPayloadColumn = "payload" - EventStreamSinkNakadiType = "Nakadi" - EventStreamSinkSqsStandardType = "SqsStandard" - EventStreamSinkSqsFifoType = "SqsFifo" + EventStreamSourcePGType = "PostgresLogicalReplication" + EventStreamSourceSlotPrefix = "fes_" + EventStreamSourceAuthType = "DatabaseAuthenticationSecret" + EventStreamFlowPgNakadiType = "PostgresWalToNakadiDataEvent" + EventStreamFlowPgGenericType = "PostgresWalToGenericNakadiEvent" + EventStreamFlowPgApiType = "PostgresWalToApiCallHomeEvent" + EventStreamFlowDataTypeColumn = "data_type" + EventStreamFlowDataOpColumn = "data_op" + EventStreamFlowMetadataColumn = "metadata" + EventStreamFlowDataColumn = "data" + EventStreamFlowPayloadColumn = "payload" + EventStreamSinkNakadiType = "Nakadi" ) From a17d63088b7d293fdb062d8eae8f744e72a72000 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 21 Sep 2021 12:31:05 +0200 Subject: [PATCH 14/44] remove fields from FES api and fix update --- .../zalando.org/v1alpha1/fabriceventstream.go | 16 ++--- pkg/cluster/cluster.go | 2 +- pkg/cluster/streams.go | 65 ++++++++----------- pkg/util/constants/streams.go | 3 +- 4 files changed, 34 insertions(+), 52 deletions(-) diff --git a/pkg/apis/zalando.org/v1alpha1/fabriceventstream.go b/pkg/apis/zalando.org/v1alpha1/fabriceventstream.go index 36f68ca2f..111179d07 100644 --- a/pkg/apis/zalando.org/v1alpha1/fabriceventstream.go +++ b/pkg/apis/zalando.org/v1alpha1/fabriceventstream.go @@ -40,14 +40,12 @@ type EventStream struct { // EventStreamFlow defines the flow characteristics of the event stream type EventStreamFlow struct { - Type string `json:"type"` - DataTypeColumn string `json:"dataTypeColumn,omitempty"` - DataOpColumn string `json:"dataOpColumn,omitempty"` - MetadataColumn string `json:"metadataColumn,omitempty"` - DataColumn string `json:"dataColumn,omitempty"` - PayloadColumn string `json:"payloadColumn,omitempty"` - CallHomeIdColumn string `json:"callHomeIdColumn,omitempty"` - CallHomeUrl string `json:"callHomeUrl,omitempty"` + Type string `json:"type"` + DataTypeColumn string `json:"dataTypeColumn,omitempty"` + DataOpColumn string `json:"dataOpColumn,omitempty"` + MetadataColumn string `json:"metadataColumn,omitempty"` + DataColumn string `json:"dataColumn,omitempty"` + PayloadColumn string `json:"payloadColumn,omitempty"` } // EventStreamSink defines the target of the event stream @@ -55,8 +53,6 @@ type EventStreamSink struct { Type string `json:"type"` EventType string `json:"eventType,omitempty"` MaxBatchSize uint32 `json:"maxBatchSize,omitempty"` - QueueName string `json:"queueName,omitempty"` - QueueUrl string `json:"queueUrl,omitempty"` } // EventStreamSource defines the source of the event stream and connection for FES operator diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 569ba463c..195bfba27 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -1060,7 +1060,7 @@ func (c *Cluster) initSystemUsers() { // replication users for event streams are another exception // the operator will create one replication user for all streams if len(c.Spec.Streams) > 0 { - username := constants.EventStreamSourceSlotPrefix + "user" + username := constants.EventStreamSourceSlotPrefix + constants.UserRoleNameSuffix streamUser := spec.PgUser{ Origin: spec.RoleConnectionPooler, Name: username, diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index 5d33fa1c3..e9e3ce66c 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -21,7 +21,7 @@ func (c *Cluster) createStreams() error { c.setProcessName("creating streams") fes := c.generateFabricEventStream() - _, err := c.KubeClient.FabricEventStreamsGetter.FabricEventStreams(c.Namespace).Create(context.TODO(), fes, metav1.CreateOptions{}) + _, err := c.KubeClient.FabricEventStreams(c.Namespace).Create(context.TODO(), fes, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("could not create event stream custom resource: %v", err) } @@ -32,7 +32,7 @@ func (c *Cluster) createStreams() error { func (c *Cluster) updateStreams(newEventStreams *zalandov1alpha1.FabricEventStream) error { c.setProcessName("updating event streams") - _, err := c.KubeClient.FabricEventStreamsGetter.FabricEventStreams(c.Namespace).Update(context.TODO(), newEventStreams, metav1.UpdateOptions{}) + _, err := c.KubeClient.FabricEventStreams(newEventStreams.Namespace).Update(context.TODO(), newEventStreams, metav1.UpdateOptions{}) if err != nil { return fmt.Errorf("could not update event stream custom resource: %v", err) } @@ -43,7 +43,7 @@ func (c *Cluster) updateStreams(newEventStreams *zalandov1alpha1.FabricEventStre func (c *Cluster) deleteStreams() error { c.setProcessName("updating event streams") - err := c.KubeClient.FabricEventStreamsGetter.FabricEventStreams(c.Namespace).Delete(context.TODO(), c.Name, metav1.DeleteOptions{}) + err := c.KubeClient.FabricEventStreams(c.Namespace).Delete(context.TODO(), c.Name, metav1.DeleteOptions{}) if err != nil { return fmt.Errorf("could not delete event stream custom resource: %v", err) } @@ -68,7 +68,7 @@ func (c *Cluster) syncPostgresConfig() error { "plugin": "wal2json", "type": "logical", } - slots[constants.EventStreamSourceSlotPrefix+stream.Database] = slot + slots[constants.EventStreamSourceSlotPrefix+"_"+stream.Database] = slot } } @@ -121,6 +121,10 @@ func (c *Cluster) generateFabricEventStream() *zalandov1alpha1.FabricEventStream } return &zalandov1alpha1.FabricEventStream{ + TypeMeta: metav1.TypeMeta{ + Kind: "FabricEventStream", + APIVersion: "zalando.org/v1alphav1", + }, ObjectMeta: metav1.ObjectMeta{ Name: c.Name, Namespace: c.Namespace, @@ -135,26 +139,14 @@ func (c *Cluster) generateFabricEventStream() *zalandov1alpha1.FabricEventStream func (c *Cluster) getEventStreamSource(stream acidv1.Stream, table, eventType string) zalandov1alpha1.EventStreamSource { _, schema := getTableSchema(table) - switch stream.StreamType { - case "nakadi": - streamFilter := stream.Filter[table] - return zalandov1alpha1.EventStreamSource{ - Type: constants.EventStreamSourcePGType, - Schema: schema, - EventStreamTable: getOutboxTable(table, eventType), - Filter: streamFilter, - Connection: c.getStreamConnection(stream.Database, constants.EventStreamSourceSlotPrefix+constants.UserRoleNameSuffix), - } - case "default": - return zalandov1alpha1.EventStreamSource{ - Type: constants.EventStreamSourcePGType, - Schema: schema, - EventStreamTable: getSourceTable(table), - Connection: c.getStreamConnection(stream.Database, constants.EventStreamSourceSlotPrefix+constants.UserRoleNameSuffix), - } + streamFilter := stream.Filter[table] + return zalandov1alpha1.EventStreamSource{ + Type: constants.EventStreamSourcePGType, + Schema: schema, + EventStreamTable: getOutboxTable(table, eventType), + Filter: streamFilter, + Connection: c.getStreamConnection(stream.Database, constants.EventStreamSourceSlotPrefix+constants.UserRoleNameSuffix), } - - return zalandov1alpha1.EventStreamSource{} } func getEventStreamFlow(stream acidv1.Stream) zalandov1alpha1.EventStreamFlow { @@ -167,7 +159,7 @@ func getEventStreamFlow(stream acidv1.Stream) zalandov1alpha1.EventStreamFlow { MetadataColumn: constants.EventStreamFlowMetadataColumn, DataColumn: constants.EventStreamFlowDataColumn, } - case "default": + case "wal": return zalandov1alpha1.EventStreamFlow{ Type: constants.EventStreamFlowPgGenericType, PayloadColumn: constants.EventStreamFlowPayloadColumn, @@ -203,19 +195,13 @@ func getOutboxTable(tableName, eventType string) zalandov1alpha1.EventStreamTabl } } -func getSourceTable(tableName string) zalandov1alpha1.EventStreamTable { - return zalandov1alpha1.EventStreamTable{ - Name: outboxTableNameTemplate.Format("table", tableName), - } -} - func (c *Cluster) getStreamConnection(database, user string) zalandov1alpha1.Connection { return zalandov1alpha1.Connection{ Url: fmt.Sprintf("jdbc:postgresql://%s.%s/%s?user=%s&ssl=true&sslmode=require", c.Name, c.Namespace, database, user), SlotName: c.getLogicalReplicationSlot(database), DBAuth: zalandov1alpha1.DBAuth{ Type: constants.EventStreamSourceAuthType, - Name: c.credentialSecretNameForCluster(user, c.ClusterName), + Name: c.credentialSecretNameForCluster(user, c.Name), UserKey: "username", PasswordKey: "password", }, @@ -224,12 +210,12 @@ func (c *Cluster) getStreamConnection(database, user string) zalandov1alpha1.Con func (c *Cluster) getLogicalReplicationSlot(database string) string { for slotName, slot := range c.Spec.Patroni.Slots { - if strings.HasPrefix(slotName, constants.EventStreamSourceSlotPrefix) && slot["type"] == "logical" && slot["database"] == database { + if slot["type"] == "logical" && slot["database"] == database { return slotName } } - return "" + return constants.EventStreamSourceSlotPrefix + "_" + database } func (c *Cluster) syncStreams() error { @@ -247,18 +233,19 @@ func (c *Cluster) syncStreams() error { return fmt.Errorf("error during reading of event streams: %v", err) } - c.logger.Infof("event streams do not exist") + c.logger.Infof("event streams do not exist, create it") err := c.createStreams() if err != nil { - return fmt.Errorf("could not create missing streams: %v", err) + return fmt.Errorf("event stream creation failed: %v", err) } } else { - if err != nil { - c.logger.Warnf("database setup might be incomplete : %v", err) - } desiredStreams := c.generateFabricEventStream() if !reflect.DeepEqual(effectiveStreams.Spec, desiredStreams.Spec) { - c.updateStreams(desiredStreams) + desiredStreams.ObjectMeta.ResourceVersion = effectiveStreams.ObjectMeta.ResourceVersion + err = c.updateStreams(desiredStreams) + if err != nil { + return fmt.Errorf("event stream update failed: %v", err) + } } } diff --git a/pkg/util/constants/streams.go b/pkg/util/constants/streams.go index 1a8028747..fb721d309 100644 --- a/pkg/util/constants/streams.go +++ b/pkg/util/constants/streams.go @@ -3,11 +3,10 @@ package constants // PostgreSQL specific constants const ( EventStreamSourcePGType = "PostgresLogicalReplication" - EventStreamSourceSlotPrefix = "fes_" + EventStreamSourceSlotPrefix = "fes" EventStreamSourceAuthType = "DatabaseAuthenticationSecret" EventStreamFlowPgNakadiType = "PostgresWalToNakadiDataEvent" EventStreamFlowPgGenericType = "PostgresWalToGenericNakadiEvent" - EventStreamFlowPgApiType = "PostgresWalToApiCallHomeEvent" EventStreamFlowDataTypeColumn = "data_type" EventStreamFlowDataOpColumn = "data_op" EventStreamFlowMetadataColumn = "metadata" From 223ffa75efc12c7b7ee00ba3eb10bfa32b297b70 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 22 Sep 2021 11:47:31 +0200 Subject: [PATCH 15/44] remove streamType field --- .../postgres-operator/crds/postgresqls.yaml | 8 +++--- docs/reference/cluster_manifest.md | 13 ++++------ manifests/complete-postgres-manifest.yaml | 18 +++++-------- manifests/postgresql.crd.yaml | 8 +++--- pkg/apis/acid.zalan.do/v1/crds.go | 14 ++-------- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 9 +++---- .../zalando.org/v1alpha1/fabriceventstream.go | 10 +++---- pkg/cluster/streams.go | 26 +++++-------------- pkg/cluster/streams_test.go | 14 ++-------- pkg/util/constants/streams.go | 16 ++++-------- 10 files changed, 40 insertions(+), 96 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index cf4cd4251..763b4e6c5 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -477,6 +477,9 @@ spec: nullable: true items: type: object + required: + - database + - tables properties: batchSize: type: integer @@ -490,11 +493,6 @@ spec: type: object additionalProperties: type: string - streamType: - type: string - enum: - - "nakadi" - - "wal" teamId: type: string tls: diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index f879e7938..ee8555bea 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -522,27 +522,24 @@ under the `streams` top-level key will be used by the operator to create a CRD for Zalando's internal CDC operator named like the Postgres cluster. Each stream object can have the following properties: -* **streamType** - Defines the stream flow. Choose `nakadi` when you want to specify certain - nakadi event types of or `wal` if changes should be mapped to a generic - event type. Default is `wal`. - * **database** Name of the database from where events will be published via Postgres' logical decoding feature. The operator will take care of updating the database configuration (setting `wal_level: logical`, creating logical replication slots, using output plugin `wal2json` and creating a dedicated - replication user). + replication user). Required. * **tables** Defines a map of table names and event types. The CDC operator is following the [outbox pattern](https://debezium.io/blog/2019/02/19/reliable-microservices-data-exchange-with-the-outbox-pattern/) meaning changes are only consumed from an extra table that already has the structure of the event in the target sink. The operator will assume that this - outbox table is called like `
__outbox`. + outbox table is called like `
__outbox`. Required. * **filter** Streamed events can be filtered by a jsonpath expression for each table. + Optional. * **batchSize** - Defines the size of batches in which events are consumed. + Defines the size of batches in which events are consumed. Optional. + Defaults to 1. diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 68977bbcd..a28e5999e 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -198,15 +198,11 @@ spec: # Enables change data capture streams for defined database tables # streams: -# - streamType: nakadi -# batchSize: 100 -# database: foo +# - database: foo # tables: -# ta: event_type_a -# tb: event_type_b -# - streamType: wal -# batchSize: 100 -# database: foo -# tables: -# public.tx: event_type_a -# public.ty: event_type_b +# data.ta: event_type_a +# data.tb: event_type_b +# # Optional. Filter ignores events before a certain txnId and lsn. Can be used to skip bad events +# filter: +# data.ta: "[?(@.source.txId > 500 && @.source.lsn > 123456)]" +# batchSize: 1000 diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index e2aa227a0..6e6b091e7 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -473,6 +473,9 @@ spec: nullable: true items: type: object + required: + - database + - tables properties: batchSize: type: integer @@ -486,11 +489,6 @@ spec: type: object additionalProperties: type: string - streamType: - type: string - enum: - - "nakadi" - - "wal" teamId: type: string tls: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index c3ee7d81d..214c8b3b4 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -664,7 +664,8 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Type: "array", Items: &apiextv1.JSONSchemaPropsOrArray{ Schema: &apiextv1.JSONSchemaProps{ - Type: "object", + Type: "object", + Required: []string{"database", "tables"}, Properties: map[string]apiextv1.JSONSchemaProps{ "batchSize": { Type: "integer", @@ -688,17 +689,6 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ }, }, }, - "streamType": { - Type: "string", - Enum: []apiextv1.JSON{ - { - Raw: []byte(`"nakadi"`), - }, - { - Raw: []byte(`"wal"`), - }, - }, - }, }, }, }, diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index af45d7c04..0499763db 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -229,9 +229,8 @@ type ConnectionPooler struct { } type Stream struct { - StreamType string `json:"streamType,omitempty"` - Database string `json:"database,omitempty"` - Tables map[string]string `json:"tables,omitempty"` - Filter map[string]string `json:"filter,omitempty"` - BatchSize uint32 `json:"batchSize,omitempty"` + Database string `json:"database"` + Tables map[string]string `json:"tables"` + Filter map[string]string `json:"filter,omitempty"` + BatchSize uint32 `json:"batchSize,omitempty"` } diff --git a/pkg/apis/zalando.org/v1alpha1/fabriceventstream.go b/pkg/apis/zalando.org/v1alpha1/fabriceventstream.go index 111179d07..2ef0b6405 100644 --- a/pkg/apis/zalando.org/v1alpha1/fabriceventstream.go +++ b/pkg/apis/zalando.org/v1alpha1/fabriceventstream.go @@ -40,12 +40,8 @@ type EventStream struct { // EventStreamFlow defines the flow characteristics of the event stream type EventStreamFlow struct { - Type string `json:"type"` - DataTypeColumn string `json:"dataTypeColumn,omitempty"` - DataOpColumn string `json:"dataOpColumn,omitempty"` - MetadataColumn string `json:"metadataColumn,omitempty"` - DataColumn string `json:"dataColumn,omitempty"` - PayloadColumn string `json:"payloadColumn,omitempty"` + Type string `json:"type"` + PayloadColumn string `json:"payloadColumn,omitempty" defaults:"payload"` } // EventStreamSink defines the target of the event stream @@ -67,7 +63,7 @@ type EventStreamSource struct { // EventStreamTable defines the name and ID column to be used for streaming type EventStreamTable struct { Name string `json:"name"` - IDColumn string `json:"idColumn,omitempty"` + IDColumn string `json:"idColumn,omitempty" defaults:"id"` } // Connection to be used for allowing the FES operator to connect to a database diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index e9e3ce66c..b6e1ef29a 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -150,23 +150,9 @@ func (c *Cluster) getEventStreamSource(stream acidv1.Stream, table, eventType st } func getEventStreamFlow(stream acidv1.Stream) zalandov1alpha1.EventStreamFlow { - switch stream.StreamType { - case "nakadi": - return zalandov1alpha1.EventStreamFlow{ - Type: constants.EventStreamFlowPgNakadiType, - DataTypeColumn: constants.EventStreamFlowDataTypeColumn, - DataOpColumn: constants.EventStreamFlowDataOpColumn, - MetadataColumn: constants.EventStreamFlowMetadataColumn, - DataColumn: constants.EventStreamFlowDataColumn, - } - case "wal": - return zalandov1alpha1.EventStreamFlow{ - Type: constants.EventStreamFlowPgGenericType, - PayloadColumn: constants.EventStreamFlowPayloadColumn, - } + return zalandov1alpha1.EventStreamFlow{ + Type: constants.EventStreamFlowPgGenericType, } - - return zalandov1alpha1.EventStreamFlow{} } func getEventStreamSink(stream acidv1.Stream, eventType string) zalandov1alpha1.EventStreamSink { @@ -190,8 +176,7 @@ func getTableSchema(fullTableName string) (tableName, schemaName string) { func getOutboxTable(tableName, eventType string) zalandov1alpha1.EventStreamTable { return zalandov1alpha1.EventStreamTable{ - Name: outboxTableNameTemplate.Format("table", tableName, "eventtype", eventType), - IDColumn: "id", + Name: outboxTableNameTemplate.Format("table", tableName, "eventtype", eventType), } } @@ -236,15 +221,16 @@ func (c *Cluster) syncStreams() error { c.logger.Infof("event streams do not exist, create it") err := c.createStreams() if err != nil { - return fmt.Errorf("event stream creation failed: %v", err) + return fmt.Errorf("event streams creation failed: %v", err) } } else { desiredStreams := c.generateFabricEventStream() if !reflect.DeepEqual(effectiveStreams.Spec, desiredStreams.Spec) { + c.logger.Debug("updating event streams") desiredStreams.ObjectMeta.ResourceVersion = effectiveStreams.ObjectMeta.ResourceVersion err = c.updateStreams(desiredStreams) if err != nil { - return fmt.Errorf("event stream update failed: %v", err) + return fmt.Errorf("event streams update failed: %v", err) } } } diff --git a/pkg/cluster/streams_test.go b/pkg/cluster/streams_test.go index 5976d97b3..617efcc31 100644 --- a/pkg/cluster/streams_test.go +++ b/pkg/cluster/streams_test.go @@ -47,16 +47,7 @@ var ( }, Streams: []acidv1.Stream{ { - StreamType: "nakadi", - Database: "foo", - Tables: map[string]string{ - "bar": "stream_type_a", - }, - BatchSize: uint32(100), - }, - { - StreamType: "wal", - Database: "foo", + Database: "foo", Tables: map[string]string{ "bar": "stream_type_a", }, @@ -132,8 +123,7 @@ func TestUpdateFabricEventStream(t *testing.T) { var pgSpec acidv1.PostgresSpec pgSpec.Streams = []acidv1.Stream{ { - StreamType: "nakadi", - Database: "foo", + Database: "foo", Tables: map[string]string{ "bar": "stream_type_b", }, diff --git a/pkg/util/constants/streams.go b/pkg/util/constants/streams.go index fb721d309..25f72d85a 100644 --- a/pkg/util/constants/streams.go +++ b/pkg/util/constants/streams.go @@ -2,15 +2,9 @@ package constants // PostgreSQL specific constants const ( - EventStreamSourcePGType = "PostgresLogicalReplication" - EventStreamSourceSlotPrefix = "fes" - EventStreamSourceAuthType = "DatabaseAuthenticationSecret" - EventStreamFlowPgNakadiType = "PostgresWalToNakadiDataEvent" - EventStreamFlowPgGenericType = "PostgresWalToGenericNakadiEvent" - EventStreamFlowDataTypeColumn = "data_type" - EventStreamFlowDataOpColumn = "data_op" - EventStreamFlowMetadataColumn = "metadata" - EventStreamFlowDataColumn = "data" - EventStreamFlowPayloadColumn = "payload" - EventStreamSinkNakadiType = "Nakadi" + EventStreamSourcePGType = "PostgresLogicalReplication" + EventStreamSourceSlotPrefix = "fes" + EventStreamSourceAuthType = "DatabaseAuthenticationSecret" + EventStreamFlowPgGenericType = "PostgresWalToGenericNakadiEvent" + EventStreamSinkNakadiType = "Nakadi" ) From c5ea2989ca4c99831c41c07aafe4c4a9f66fb287 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 22 Sep 2021 16:45:06 +0200 Subject: [PATCH 16/44] check if fes CRD exists before syncing --- pkg/cluster/streams.go | 29 +++++++++++++++++++++++++---- pkg/util/constants/streams.go | 2 ++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index b6e1ef29a..173305033 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -41,9 +41,14 @@ func (c *Cluster) updateStreams(newEventStreams *zalandov1alpha1.FabricEventStre } func (c *Cluster) deleteStreams() error { - c.setProcessName("updating event streams") + c.setProcessName("deleting event streams") + + _, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), constants.EventStreamSourceCRDName, metav1.GetOptions{}) + if k8sutil.ResourceNotFound(err) { + return nil + } - err := c.KubeClient.FabricEventStreams(c.Namespace).Delete(context.TODO(), c.Name, metav1.DeleteOptions{}) + err = c.KubeClient.FabricEventStreams(c.Namespace).Delete(context.TODO(), c.Name, metav1.DeleteOptions{}) if err != nil { return fmt.Errorf("could not delete event stream custom resource: %v", err) } @@ -122,7 +127,7 @@ func (c *Cluster) generateFabricEventStream() *zalandov1alpha1.FabricEventStream return &zalandov1alpha1.FabricEventStream{ TypeMeta: metav1.TypeMeta{ - Kind: "FabricEventStream", + Kind: constants.EventStreamSourceCRDKind, APIVersion: "zalando.org/v1alphav1", }, ObjectMeta: metav1.ObjectMeta{ @@ -205,6 +210,22 @@ func (c *Cluster) getLogicalReplicationSlot(database string) string { func (c *Cluster) syncStreams() error { + _, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), constants.EventStreamSourceCRDName, metav1.GetOptions{}) + if k8sutil.ResourceNotFound(err) { + c.logger.Debugf("event stream CRD not installed, skipping") + return nil + } + + err = c.createOrUpdateStreams() + if err != nil { + return err + } + + return nil +} + +func (c *Cluster) createOrUpdateStreams() error { + c.setProcessName("syncing streams") err := c.syncPostgresConfig() @@ -212,7 +233,7 @@ func (c *Cluster) syncStreams() error { return fmt.Errorf("could not update Postgres config for event streaming: %v", err) } - effectiveStreams, err := c.KubeClient.FabricEventStreamsGetter.FabricEventStreams(c.Namespace).Get(context.TODO(), c.Name, metav1.GetOptions{}) + effectiveStreams, err := c.KubeClient.FabricEventStreams(c.Namespace).Get(context.TODO(), c.Name, metav1.GetOptions{}) if err != nil { if !k8sutil.ResourceNotFound(err) { return fmt.Errorf("error during reading of event streams: %v", err) diff --git a/pkg/util/constants/streams.go b/pkg/util/constants/streams.go index 25f72d85a..82d61c5c2 100644 --- a/pkg/util/constants/streams.go +++ b/pkg/util/constants/streams.go @@ -2,6 +2,8 @@ package constants // PostgreSQL specific constants const ( + EventStreamSourceCRDKind = "FabricEventStream" + EventStreamSourceCRDName = "fabriceventstreams.zalando.org" EventStreamSourcePGType = "PostgresLogicalReplication" EventStreamSourceSlotPrefix = "fes" EventStreamSourceAuthType = "DatabaseAuthenticationSecret" From 9abd6f7fcc396ad9facd46d9b1c1c3812d242886 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 22 Sep 2021 16:59:15 +0200 Subject: [PATCH 17/44] avoid CRD getter in unit test --- pkg/cluster/streams.go | 1 + pkg/cluster/streams_test.go | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index 173305033..07c94f473 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -43,6 +43,7 @@ func (c *Cluster) updateStreams(newEventStreams *zalandov1alpha1.FabricEventStre func (c *Cluster) deleteStreams() error { c.setProcessName("deleting event streams") + // check if stream CRD is installed before trying a delete _, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), constants.EventStreamSourceCRDName, metav1.GetOptions{}) if k8sutil.ResourceNotFound(err) { return nil diff --git a/pkg/cluster/streams_test.go b/pkg/cluster/streams_test.go index 617efcc31..725c856b1 100644 --- a/pkg/cluster/streams_test.go +++ b/pkg/cluster/streams_test.go @@ -83,7 +83,7 @@ func TestGenerateFabricEventStream(t *testing.T) { }, }, client, pg, logger, eventRecorder) - err := cluster.syncStreams() + err := cluster.createOrUpdateStreams() assert.NoError(t, err) streamCRD, err := cluster.KubeClient.FabricEventStreams(namespace).Get(context.TODO(), cluster.Name, metav1.GetOptions{}) @@ -117,7 +117,7 @@ func TestUpdateFabricEventStream(t *testing.T) { _, err := cluster.KubeClient.Postgresqls(namespace).Create( context.TODO(), &pg, metav1.CreateOptions{}) assert.NoError(t, err) - err = cluster.syncStreams() + err = cluster.createOrUpdateStreams() assert.NoError(t, err) var pgSpec acidv1.PostgresSpec @@ -140,7 +140,7 @@ func TestUpdateFabricEventStream(t *testing.T) { assert.NoError(t, err) cluster.Postgresql.Spec = pgPatched.Spec - err = cluster.syncStreams() + err = cluster.createOrUpdateStreams() assert.NoError(t, err) streamCRD, err := cluster.KubeClient.FabricEventStreams(namespace).Get(context.TODO(), cluster.Name, metav1.GetOptions{}) From c8d89f9c6505a0f1680fbbec00f0573b897a3593 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 1 Oct 2021 18:50:40 +0200 Subject: [PATCH 18/44] reflect code review --- pkg/apis/zalando.org/v1alpha1/register.go | 4 +--- pkg/cluster/cluster.go | 2 +- pkg/cluster/streams.go | 6 ++++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/apis/zalando.org/v1alpha1/register.go b/pkg/apis/zalando.org/v1alpha1/register.go index 07732ae53..0136a82ec 100644 --- a/pkg/apis/zalando.org/v1alpha1/register.go +++ b/pkg/apis/zalando.org/v1alpha1/register.go @@ -15,9 +15,7 @@ const ( var ( schemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) - // AddToScheme applies all the stored functions to the scheme. A non-nil error - // indicates that one function failed and the attempt was abandoned. - AddToScheme = schemeBuilder.AddToScheme + AddToScheme = schemeBuilder.AddToScheme ) func init() { diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 195bfba27..1281dc377 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -362,7 +362,7 @@ func (c *Cluster) Create() error { c.createConnectionPooler(c.installLookupFunction) if err = c.syncStreams(); err != nil { - return fmt.Errorf("could not create streams: %v", err) + c.logger.Errorf("could not create streams: %v", err) } return nil diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index 07c94f473..4ba0558a9 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -79,7 +79,7 @@ func (c *Cluster) syncPostgresConfig() error { } if len(slots) > 0 { - c.logger.Debugf("setting wal level to 'logical' in Postgres configuration") + c.logger.Debugf("setting wal level to 'logical' in Postgres configuration to allow for decoding changes") for slotName, slot := range slots { c.logger.Debugf("creating logical replication slot %q in database %q", slotName, slot["database"]) } @@ -90,7 +90,7 @@ func (c *Cluster) syncPostgresConfig() error { pods, err := c.listPods() if err != nil || len(pods) == 0 { - return err + c.logger.Warnf("could not list pods of the statefulset: %v", err) } for i, pod := range pods { podName := util.NameFromMeta(pods[i].ObjectMeta) @@ -135,6 +135,8 @@ func (c *Cluster) generateFabricEventStream() *zalandov1alpha1.FabricEventStream Name: c.Name, Namespace: c.Namespace, Annotations: c.AnnotationsToPropagate(c.annotationsSet(nil)), + // make cluster StatefulSet the owner (like with connection pooler objects) + OwnerReferences: c.ownerReferences(), }, Spec: zalandov1alpha1.FabricEventStreamSpec{ ApplicationId: "", From db76b8c642aecc313805f2c7636e305df4b7c041 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 18 Oct 2021 17:36:34 +0200 Subject: [PATCH 19/44] minor fix for restart TTL --- pkg/cluster/streams.go | 10 +++++----- pkg/cluster/sync.go | 10 +++------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index fbca158d7..3b5134dba 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -59,8 +59,8 @@ func (c *Cluster) deleteStreams() error { func (c *Cluster) syncPostgresConfig() error { - desiredPostgresConfig := c.Spec.Patroni - slots := desiredPostgresConfig.Slots + desiredPatroniConfig := c.Spec.Patroni + slots := desiredPatroniConfig.Slots for _, stream := range c.Spec.Streams { slotName := c.getLogicalReplicationSlot(stream.Database) @@ -80,7 +80,7 @@ func (c *Cluster) syncPostgresConfig() error { for slotName, slot := range slots { c.logger.Debugf("creating logical replication slot %q in database %q", slotName, slot["database"]) } - desiredPostgresConfig.Slots = slots + desiredPatroniConfig.Slots = slots } else { return nil } @@ -94,13 +94,13 @@ func (c *Cluster) syncPostgresConfig() error { } for i, pod := range pods { podName := util.NameFromMeta(pods[i].ObjectMeta) - effectivePostgresConfig, effectivePgParameters, err := c.patroni.GetConfig(&pod) + effectivePatroniConfig, effectivePgParameters, err := c.patroni.GetConfig(&pod) if err != nil { c.logger.Warningf("could not get Postgres config from pod %s: %v", podName, err) continue } - _, err = c.checkAndSetGlobalPostgreSQLConfiguration(&pod, effectivePostgresConfig, desiredPostgresConfig, effectivePgParameters, desiredPgParameters) + _, err = c.checkAndSetGlobalPostgreSQLConfiguration(&pod, effectivePatroniConfig, desiredPatroniConfig, effectivePgParameters, desiredPgParameters) if err != nil { c.logger.Warningf("could not set PostgreSQL configuration options for pod %s: %v", podName, err) continue diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index f20acd21b..e679b8a7f 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -271,7 +271,7 @@ func (c *Cluster) syncPodDisruptionBudget(isUpdate bool) error { func (c *Cluster) syncStatefulSet() error { var ( masterPod *v1.Pod - effectivePostgresConfig map[string]interface{} + restartTTL uint32 instanceRestartRequired bool ) @@ -404,7 +404,6 @@ func (c *Cluster) syncStatefulSet() error { emptyPatroniConfig := acidv1.Patroni{} podName := util.NameFromMeta(pods[i].ObjectMeta) patroniConfig, pgParameters, err := c.patroni.GetConfig(&pod) - if err != nil { c.logger.Warningf("could not get Postgres config from pod %s: %v", podName, err) continue @@ -418,6 +417,7 @@ func (c *Cluster) syncStatefulSet() error { c.logger.Warningf("could not set PostgreSQL configuration options for pod %s: %v", podName, err) continue } + restartTTL = patroniConfig.TTL break } } @@ -425,10 +425,6 @@ func (c *Cluster) syncStatefulSet() error { // if the config update requires a restart, call Patroni restart for replicas first, then master if instanceRestartRequired { c.logger.Debug("restarting Postgres server within pods") - ttl, ok := effectivePostgresConfig["ttl"].(int32) - if !ok { - ttl = 30 - } for i, pod := range pods { role := PostgresRole(pod.Labels[c.OpConfig.PodRoleLabel]) if role == Master { @@ -436,7 +432,7 @@ func (c *Cluster) syncStatefulSet() error { continue } c.restartInstance(&pod) - time.Sleep(time.Duration(ttl) * time.Second) + time.Sleep(time.Duration(restartTTL) * time.Second) } if masterPod != nil { From fa63803be9464c02edb54d3541e7c7c520164a5e Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 26 Oct 2021 17:13:18 +0200 Subject: [PATCH 20/44] minor changes for comments --- pkg/cluster/sync.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index a788b1922..9491fc194 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -420,6 +420,7 @@ func (c *Cluster) syncStatefulSet() error { // empty config probably means cluster is not fully initialized yet, e.g. restoring from backup // do not attempt a restart if !reflect.DeepEqual(patroniConfig, emptyPatroniConfig) || len(pgParameters) > 0 { + // compare config returned from Patroni with what is specified in the manifest restartMasterFirst, err = c.checkAndSetGlobalPostgreSQLConfiguration(&pod, patroniConfig, c.Spec.Patroni, pgParameters, c.Spec.Parameters) if err != nil { @@ -568,7 +569,7 @@ func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, effectiv configToSet["slots"] = slotsToSet } - // compare parameters under postgresql section with c.Spec.Postgresql.Parameters from manifest + // compare effective and desired parameters under postgresql section in Patroni config for desiredOption, desiredValue := range desiredPgParameters { effectiveValue := effectivePgParameters[desiredOption] if isBootstrapOnlyParameter(desiredOption) && (effectiveValue != desiredValue) { @@ -611,7 +612,7 @@ func (c *Cluster) checkAndSetGlobalPostgreSQLConfiguration(pod *v1.Pod, effectiv c.logger.Debugf("patching Postgres config via Patroni API on pod %s with following options: %s", podName, configToSetJson) if err = c.patroni.SetConfig(pod, configToSet); err != nil { - return requiresMasterRestart, fmt.Errorf("could not patch postgres parameters with a pod %s: %v", podName, err) + return requiresMasterRestart, fmt.Errorf("could not patch postgres parameters within pod %s: %v", podName, err) } return requiresMasterRestart, nil From 06956d172da0c93b4c5c819940c7ea00366aed34 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 25 Nov 2021 16:05:48 +0100 Subject: [PATCH 21/44] existing slot must use the same plugin --- pkg/cluster/streams.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index 3b5134dba..b00b6d09c 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -203,7 +203,7 @@ func (c *Cluster) getStreamConnection(database, user string) zalandov1alpha1.Con func (c *Cluster) getLogicalReplicationSlot(database string) string { for slotName, slot := range c.Spec.Patroni.Slots { - if slot["type"] == "logical" && slot["database"] == database { + if slot["type"] == "logical" && slot["database"] == database && slot["plugin"] == "wal2json" { return slotName } } From 837969ed76d0b548d30def1acda95f2f249785d4 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 2 Dec 2021 16:10:20 +0100 Subject: [PATCH 22/44] Update docs/reference/cluster_manifest.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Paŭlo Ebermann --- docs/reference/cluster_manifest.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index ff492743a..626f832fe 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -530,11 +530,11 @@ Each stream object can have the following properties: replication user). Required. * **tables** - Defines a map of table names and event types. The CDC operator is following - the [outbox pattern](https://debezium.io/blog/2019/02/19/reliable-microservices-data-exchange-with-the-outbox-pattern/) - meaning changes are only consumed from an extra table that already has the - structure of the event in the target sink. The operator will assume that this - outbox table is called like `
__outbox`. Required. + Defines a map of (outbox) table names and event types. The CDC operator is following + the [outbox pattern](https://debezium.io/blog/2019/02/19/reliable-microservices-data-exchange-with-the-outbox-pattern/). + This means that the application will put events into a column in the outbox table + in the structure of the target event type, and the CDC operator will capture them + shortly after the transaction is committed. Required. * **filter** Streamed events can be filtered by a jsonpath expression for each table. From bde522ba2ae18c376142182e149e6d2a1b12c862 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 2 Dec 2021 17:31:31 +0100 Subject: [PATCH 23/44] make id and payload columns configurable --- .../postgres-operator/crds/postgresqls.yaml | 11 ++++++- docs/reference/cluster_manifest.md | 14 +++++---- manifests/complete-postgres-manifest.yaml | 8 +++-- manifests/postgresql.crd.yaml | 11 ++++++- pkg/apis/acid.zalan.do/v1/crds.go | 14 ++++++++- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 14 ++++++--- .../acid.zalan.do/v1/zz_generated.deepcopy.go | 18 +++++++++++- pkg/cluster/streams.go | 29 +++++++++---------- pkg/cluster/streams_test.go | 16 +++++++--- 9 files changed, 101 insertions(+), 34 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 1c6064a90..f8da45c23 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -489,7 +489,16 @@ spec: tables: type: object additionalProperties: - type: string + type: object + required: + - evenType + properties: + eventType: + type: string + idColumn: + type: string + payloadColumn: + type: string teamId: type: string tls: diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 626f832fe..cd9d8c147 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -530,11 +530,15 @@ Each stream object can have the following properties: replication user). Required. * **tables** - Defines a map of (outbox) table names and event types. The CDC operator is following - the [outbox pattern](https://debezium.io/blog/2019/02/19/reliable-microservices-data-exchange-with-the-outbox-pattern/). - This means that the application will put events into a column in the outbox table - in the structure of the target event type, and the CDC operator will capture them - shortly after the transaction is committed. Required. + Defines a map of table names and their properties (`eventType`, `idColumn` + and `payloadColumn`). The CDC operator is following the [outbox pattern](https://debezium.io/blog/2019/02/19/reliable-microservices-data-exchange-with-the-outbox-pattern/). + The application is responsible for putting events into a (JSON/B or VARCHAR) + payload column of the outbox table in the structure of the specified target + event type. The the CDC operator will consume them shortly after the + transaction is committed. The `idColumn` will be used in telemetry for the + CDC operator. The names for `idColumn` and `payloadColumn` can be configured. + Defaults are `id` and `payload`. The target `eventType` has to be defined. + Required. * **filter** Streamed events can be filtered by a jsonpath expression for each table. diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 485b37475..3924ddd28 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -200,8 +200,12 @@ spec: # streams: # - database: foo # tables: -# data.ta: event_type_a -# data.tb: event_type_b +# data.ta: +# eventType: event_type_a +# data.tb: +# eventType: event_type_b +# idColumn: tb_id +# payloadColumn: tb_payload # # Optional. Filter ignores events before a certain txnId and lsn. Can be used to skip bad events # filter: # data.ta: "[?(@.source.txId > 500 && @.source.lsn > 123456)]" diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index ad12ad6be..4b49494a8 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -487,7 +487,16 @@ spec: tables: type: object additionalProperties: - type: string + type: object + required: + - evenType + properties: + eventType: + type: string + idColumn: + type: string + payloadColumn: + type: string teamId: type: string tls: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index b0c88e264..82dde6b94 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -682,7 +682,19 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Type: "object", AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{ Schema: &apiextv1.JSONSchemaProps{ - Type: "string", + Type: "object", + Required: []string{"eventType"}, + Properties: map[string]apiextv1.JSONSchemaProps{ + "eventType": { + Type: "string", + }, + "idColumn": { + Type: "string", + }, + "payloadColumn": { + Type: "string", + }, + }, }, }, }, diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 0499763db..7a645e8fa 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -229,8 +229,14 @@ type ConnectionPooler struct { } type Stream struct { - Database string `json:"database"` - Tables map[string]string `json:"tables"` - Filter map[string]string `json:"filter,omitempty"` - BatchSize uint32 `json:"batchSize,omitempty"` + Database string `json:"database"` + Tables map[string]StreamTable `json:"tables"` + Filter map[string]string `json:"filter,omitempty"` + BatchSize uint32 `json:"batchSize,omitempty"` +} + +type StreamTable struct { + EventType string `json:"eventType"` + IdColumn string `json:"idColumn,omitempty" defaults:"id"` + PayloadColumn string `json:"payloadColumn,omitempty" defaults:"payload"` } diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 08cc7a3a9..b0f0dae25 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -1143,7 +1143,7 @@ func (in *Stream) DeepCopyInto(out *Stream) { *out = *in if in.Tables != nil { in, out := &in.Tables, &out.Tables - *out = make(map[string]string, len(*in)) + *out = make(map[string]StreamTable, len(*in)) for key, val := range *in { (*out)[key] = val } @@ -1168,6 +1168,22 @@ func (in *Stream) DeepCopy() *Stream { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StreamTable) DeepCopyInto(out *StreamTable) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StreamTable. +func (in *StreamTable) DeepCopy() *StreamTable { + if in == nil { + return nil + } + out := new(StreamTable) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TLSDescription) DeepCopyInto(out *TLSDescription) { *out = *in diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index b00b6d09c..cef86a05f 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -9,14 +9,11 @@ import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" zalandov1alpha1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1alpha1" "github.com/zalando/postgres-operator/pkg/util" - "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -var outboxTableNameTemplate config.StringTemplate = "{table}_{eventtype}_outbox" - func (c *Cluster) createStreams() error { c.setProcessName("creating streams") @@ -114,10 +111,10 @@ func (c *Cluster) generateFabricEventStream() *zalandov1alpha1.FabricEventStream eventStreams := make([]zalandov1alpha1.EventStream, 0) for _, stream := range c.Spec.Streams { - for table, eventType := range stream.Tables { - streamSource := c.getEventStreamSource(stream, table, eventType) - streamFlow := getEventStreamFlow(stream) - streamSink := getEventStreamSink(stream, eventType) + for tableName, table := range stream.Tables { + streamSource := c.getEventStreamSource(stream, tableName, table.IdColumn) + streamFlow := getEventStreamFlow(stream, table.PayloadColumn) + streamSink := getEventStreamSink(stream, table.EventType) eventStreams = append(eventStreams, zalandov1alpha1.EventStream{ EventStreamFlow: streamFlow, @@ -145,21 +142,22 @@ func (c *Cluster) generateFabricEventStream() *zalandov1alpha1.FabricEventStream } } -func (c *Cluster) getEventStreamSource(stream acidv1.Stream, table, eventType string) zalandov1alpha1.EventStreamSource { - _, schema := getTableSchema(table) - streamFilter := stream.Filter[table] +func (c *Cluster) getEventStreamSource(stream acidv1.Stream, tableName, idColumn string) zalandov1alpha1.EventStreamSource { + _, schema := getTableSchema(tableName) + streamFilter := stream.Filter[tableName] return zalandov1alpha1.EventStreamSource{ Type: constants.EventStreamSourcePGType, Schema: schema, - EventStreamTable: getOutboxTable(table, eventType), + EventStreamTable: getOutboxTable(tableName, idColumn), Filter: streamFilter, Connection: c.getStreamConnection(stream.Database, constants.EventStreamSourceSlotPrefix+constants.UserRoleNameSuffix), } } -func getEventStreamFlow(stream acidv1.Stream) zalandov1alpha1.EventStreamFlow { +func getEventStreamFlow(stream acidv1.Stream, payloadColumn string) zalandov1alpha1.EventStreamFlow { return zalandov1alpha1.EventStreamFlow{ - Type: constants.EventStreamFlowPgGenericType, + Type: constants.EventStreamFlowPgGenericType, + PayloadColumn: payloadColumn, } } @@ -182,9 +180,10 @@ func getTableSchema(fullTableName string) (tableName, schemaName string) { return tableName, schemaName } -func getOutboxTable(tableName, eventType string) zalandov1alpha1.EventStreamTable { +func getOutboxTable(tableName, idColumn string) zalandov1alpha1.EventStreamTable { return zalandov1alpha1.EventStreamTable{ - Name: outboxTableNameTemplate.Format("table", tableName, "eventtype", eventType), + Name: tableName, + IDColumn: idColumn, } } diff --git a/pkg/cluster/streams_test.go b/pkg/cluster/streams_test.go index 725c856b1..8a339b11a 100644 --- a/pkg/cluster/streams_test.go +++ b/pkg/cluster/streams_test.go @@ -48,8 +48,12 @@ var ( Streams: []acidv1.Stream{ { Database: "foo", - Tables: map[string]string{ - "bar": "stream_type_a", + Tables: map[string]acidv1.StreamTable{ + "bar": acidv1.StreamTable{ + EventType: "stream_type_a", + IdColumn: "b_id", + PayloadColumn: "b_payload", + }, }, BatchSize: uint32(100), }, @@ -124,8 +128,12 @@ func TestUpdateFabricEventStream(t *testing.T) { pgSpec.Streams = []acidv1.Stream{ { Database: "foo", - Tables: map[string]string{ - "bar": "stream_type_b", + Tables: map[string]acidv1.StreamTable{ + "bar": acidv1.StreamTable{ + EventType: "stream_type_b", + IdColumn: "b_id", + PayloadColumn: "b_payload", + }, }, BatchSize: uint32(250), }, From 50a8bf5fee8c384949c5abd7bbe958e09e4479cd Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 2 Dec 2021 17:46:57 +0100 Subject: [PATCH 24/44] fix typo in CRD schema --- charts/postgres-operator/crds/postgresqls.yaml | 2 +- manifests/postgresql.crd.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index f8da45c23..44187b6a9 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -491,7 +491,7 @@ spec: additionalProperties: type: object required: - - evenType + - eventType properties: eventType: type: string diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 4b49494a8..2b2509d1d 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -489,7 +489,7 @@ spec: additionalProperties: type: object required: - - evenType + - eventType properties: eventType: type: string From 8f2f70b4fea9afb9cc171fe46fab0092f8d1ab9a Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 3 Dec 2021 19:17:01 +0100 Subject: [PATCH 25/44] set applicationId of FES CRD by taking label from manifest --- pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go | 1 + pkg/cluster/streams.go | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index b0f0dae25..f5c669d9e 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index cef86a05f..301db97e4 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -108,8 +108,14 @@ func (c *Cluster) syncPostgresConfig() error { } func (c *Cluster) generateFabricEventStream() *zalandov1alpha1.FabricEventStream { + var applicationId string eventStreams := make([]zalandov1alpha1.EventStream, 0) + // take application label from manifest + if spec, err := c.GetSpec(); err == nil { + applicationId = spec.ObjectMeta.Labels["application"] + } + for _, stream := range c.Spec.Streams { for tableName, table := range stream.Tables { streamSource := c.getEventStreamSource(stream, tableName, table.IdColumn) @@ -136,7 +142,7 @@ func (c *Cluster) generateFabricEventStream() *zalandov1alpha1.FabricEventStream OwnerReferences: c.ownerReferences(), }, Spec: zalandov1alpha1.FabricEventStreamSpec{ - ApplicationId: "", + ApplicationId: applicationId, EventStreams: eventStreams, }, } From 90d6016dc844df56c23ac778599592498d32618f Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 6 Dec 2021 16:35:11 +0100 Subject: [PATCH 26/44] sync streams only when they are defined in manifest --- pkg/cluster/cluster.go | 14 +++++++++----- pkg/cluster/sync.go | 10 ++++++---- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index ac11f2750..e6a7c3aa8 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -361,8 +361,10 @@ func (c *Cluster) Create() error { // something fails, report warning c.createConnectionPooler(c.installLookupFunction) - if err = c.syncStreams(); err != nil { - c.logger.Errorf("could not create streams: %v", err) + if len(c.Spec.Streams) > 0 { + if err = c.syncStreams(); err != nil { + c.logger.Errorf("could not create streams: %v", err) + } } return nil @@ -859,9 +861,11 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { updateFailed = true } - if err := c.syncStreams(); err != nil { - c.logger.Errorf("could not sync streams: %v", err) - updateFailed = true + if len(c.Spec.Streams) > 0 { + if err := c.syncStreams(); err != nil { + c.logger.Errorf("could not sync streams: %v", err) + updateFailed = true + } } if !updateFailed { diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 9491fc194..55db338f1 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -129,10 +129,12 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { return fmt.Errorf("could not sync connection pooler: %v", err) } - c.logger.Debug("syncing streams") - if err = c.syncStreams(); err != nil { - err = fmt.Errorf("could not sync streams: %v", err) - return err + if len(c.Spec.Streams) > 0 { + c.logger.Debug("syncing streams") + if err = c.syncStreams(); err != nil { + err = fmt.Errorf("could not sync streams: %v", err) + return err + } } // Major version upgrade must only run after success of all earlier operations, must remain last item in sync From 96a2da1fcaa80c71bb1de8d2908a65d07f57ed33 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 7 Dec 2021 22:32:35 +0100 Subject: [PATCH 27/44] use defined fes CRD in unit test --- pkg/cluster/streams.go | 6 +-- pkg/cluster/streams_test.go | 93 ++++++++++++++++++++++++++++++++++--- 2 files changed, 89 insertions(+), 10 deletions(-) diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index 301db97e4..e3374f6e1 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -132,7 +132,7 @@ func (c *Cluster) generateFabricEventStream() *zalandov1alpha1.FabricEventStream return &zalandov1alpha1.FabricEventStream{ TypeMeta: metav1.TypeMeta{ Kind: constants.EventStreamSourceCRDKind, - APIVersion: "zalando.org/v1alphav1", + APIVersion: "zalando.org/v1alpha1", }, ObjectMeta: metav1.ObjectMeta{ Name: c.Name, @@ -149,12 +149,12 @@ func (c *Cluster) generateFabricEventStream() *zalandov1alpha1.FabricEventStream } func (c *Cluster) getEventStreamSource(stream acidv1.Stream, tableName, idColumn string) zalandov1alpha1.EventStreamSource { - _, schema := getTableSchema(tableName) + table, schema := getTableSchema(tableName) streamFilter := stream.Filter[tableName] return zalandov1alpha1.EventStreamSource{ Type: constants.EventStreamSourcePGType, Schema: schema, - EventStreamTable: getOutboxTable(tableName, idColumn), + EventStreamTable: getOutboxTable(table, idColumn), Filter: streamFilter, Connection: c.getStreamConnection(stream.Database, constants.EventStreamSourceSlotPrefix+constants.UserRoleNameSuffix), } diff --git a/pkg/cluster/streams_test.go b/pkg/cluster/streams_test.go index 8a339b11a..a18a00d67 100644 --- a/pkg/cluster/streams_test.go +++ b/pkg/cluster/streams_test.go @@ -2,6 +2,7 @@ package cluster import ( "encoding/json" + "fmt" "reflect" "context" @@ -9,8 +10,11 @@ import ( "github.com/stretchr/testify/assert" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1alpha1" fakezalandov1alpha1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake" + "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -26,13 +30,16 @@ func newFakeK8sStreamClient() (k8sutil.KubernetesClient, *fake.Clientset) { FabricEventStreamsGetter: zalandoClientSet.ZalandoV1alpha1(), PostgresqlsGetter: zalandoClientSet.AcidV1(), PodsGetter: clientSet.CoreV1(), + StatefulSetsGetter: clientSet.AppsV1(), }, clientSet } var ( clusterName string = "acid-test-cluster" namespace string = "default" - pg = acidv1.Postgresql{ + fesUser string = constants.EventStreamSourceSlotPrefix + constants.UserRoleNameSuffix + + pg = acidv1.Postgresql{ TypeMeta: metav1.TypeMeta{ Kind: "Postgresql", APIVersion: "acid.zalan.do/v1", @@ -40,6 +47,7 @@ var ( ObjectMeta: metav1.ObjectMeta{ Name: clusterName, Namespace: namespace, + Labels: map[string]string{"application": "test"}, }, Spec: acidv1.PostgresSpec{ Databases: map[string]string{ @@ -49,12 +57,15 @@ var ( { Database: "foo", Tables: map[string]acidv1.StreamTable{ - "bar": acidv1.StreamTable{ + "data.bar": acidv1.StreamTable{ EventType: "stream_type_a", IdColumn: "b_id", PayloadColumn: "b_payload", }, }, + Filter: map[string]string{ + "data.bar": "[?(@.source.txId > 500 && @.source.lsn > 123456)]", + }, BatchSize: uint32(100), }, }, @@ -66,6 +77,60 @@ var ( }, }, } + + fes = &v1alpha1.FabricEventStream{ + TypeMeta: metav1.TypeMeta{ + Kind: "FabricEventStream", + APIVersion: "zalando.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + OwnerReferences: []metav1.OwnerReference{ + metav1.OwnerReference{ + APIVersion: "apps/v1", + Kind: "StatefulSet", + Name: "acid-test-cluster", + Controller: util.True(), + }, + }, + }, + Spec: v1alpha1.FabricEventStreamSpec{ + ApplicationId: "test", + EventStreams: []v1alpha1.EventStream{ + { + EventStreamFlow: v1alpha1.EventStreamFlow{ + PayloadColumn: "b_payload", + Type: constants.EventStreamFlowPgGenericType, + }, + EventStreamSink: v1alpha1.EventStreamSink{ + EventType: "stream_type_a", + MaxBatchSize: uint32(100), + Type: constants.EventStreamSinkNakadiType, + }, + EventStreamSource: v1alpha1.EventStreamSource{ + Filter: "[?(@.source.txId > 500 && @.source.lsn > 123456)]", + Connection: v1alpha1.Connection{ + DBAuth: v1alpha1.DBAuth{ + Name: fmt.Sprintf("fes-user.%s.credentials.postgresql.acid.zalan.do", clusterName), + PasswordKey: "password", + Type: constants.EventStreamSourceAuthType, + UserKey: "username", + }, + Url: fmt.Sprintf("jdbc:postgresql://%s.%s/foo?user=%s&ssl=true&sslmode=require", clusterName, namespace, fesUser), + SlotName: "fes_foo", + }, + Schema: "data", + EventStreamTable: v1alpha1.EventStreamTable{ + IDColumn: "b_id", + Name: "bar", + }, + Type: constants.EventStreamSourcePGType, + }, + }, + }, + }, + } ) func TestGenerateFabricEventStream(t *testing.T) { @@ -74,6 +139,9 @@ func TestGenerateFabricEventStream(t *testing.T) { var cluster = New( Config{ OpConfig: config.Config{ + Auth: config.Auth{ + SecretNameTemplate: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}", + }, PodManagementPolicy: "ordered_ready", Resources: config.Resources{ ClusterLabels: map[string]string{"application": "spilo"}, @@ -87,15 +155,26 @@ func TestGenerateFabricEventStream(t *testing.T) { }, }, client, pg, logger, eventRecorder) - err := cluster.createOrUpdateStreams() + cluster.Name = clusterName + cluster.Namespace = namespace + + _, err := cluster.createStatefulSet() assert.NoError(t, err) - streamCRD, err := cluster.KubeClient.FabricEventStreams(namespace).Get(context.TODO(), cluster.Name, metav1.GetOptions{}) + err = cluster.createOrUpdateStreams() assert.NoError(t, err) result := cluster.generateFabricEventStream() - if !reflect.DeepEqual(result, streamCRD) { - t.Errorf("Malformed FabricEventStream, expected %#v, got %#v", streamCRD, result) + + if !reflect.DeepEqual(result, fes) { + t.Errorf("Malformed FabricEventStream, expected %#v, got %#v", fes, result) + } + + streamCRD, err := cluster.KubeClient.FabricEventStreams(namespace).Get(context.TODO(), cluster.Name, metav1.GetOptions{}) + assert.NoError(t, err) + + if !reflect.DeepEqual(streamCRD, fes) { + t.Errorf("Malformed FabricEventStream, expected %#v, got %#v", fes, streamCRD) } } @@ -129,7 +208,7 @@ func TestUpdateFabricEventStream(t *testing.T) { { Database: "foo", Tables: map[string]acidv1.StreamTable{ - "bar": acidv1.StreamTable{ + "data.bar": acidv1.StreamTable{ EventType: "stream_type_b", IdColumn: "b_id", PayloadColumn: "b_payload", From 74ee530a6cb5e616f058542b37d9206ec6cedf66 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 8 Dec 2021 17:11:18 +0100 Subject: [PATCH 28/44] introduce applicationId for separate stream CRDs --- .../postgres-operator/crds/postgresqls.yaml | 3 + docs/reference/cluster_manifest.md | 14 +- manifests/complete-postgres-manifest.yaml | 10 +- manifests/postgresql.crd.yaml | 3 + pkg/apis/acid.zalan.do/v1/crds.go | 5 +- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 9 +- pkg/cluster/streams.go | 124 ++++++++++-------- pkg/cluster/streams_test.go | 29 ++-- 8 files changed, 115 insertions(+), 82 deletions(-) diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 77546ca76..d94cc83e8 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -475,9 +475,12 @@ spec: items: type: object required: + - applicationId - database - tables properties: + applicationId: + type: string batchSize: type: integer database: diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 160ab2af4..8668cd72e 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -522,9 +522,17 @@ Those parameters are grouped under the `tls` top-level key. This sections enables change data capture (CDC) streams e.g. into Zalando’s distributed event broker [Nakadi](https://nakadi.io/). Parameters grouped -under the `streams` top-level key will be used by the operator to create a -CRD for Zalando's internal CDC operator named like the Postgres cluster. -Each stream object can have the following properties: +under the `streams` top-level key will be used by the operator to create +custom resources for Zalando's internal CDC operator. Each stream object can +have the following properties: + +* **applicationId** + The application name to which the database and CDC belongs to. For each + set of streams with a distinct `applicationId` a separate stream CR as well + as a separate logical replication slot will be created. This means there can + different streams in the same database and streams with the same + `applicationId` are bundled in one stream CR. The stream CR will be called + like the Postgres cluster plus "-" suffix. Required. * **database** Name of the database from where events will be published via Postgres' diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 3924ddd28..28d0a970d 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -3,6 +3,7 @@ kind: postgresql metadata: name: acid-test-cluster # labels: +# application: test-app # environment: demo # annotations: # "acid.zalan.do/controller": "second-operator" @@ -198,15 +199,16 @@ spec: # Enables change data capture streams for defined database tables # streams: -# - database: foo +# - applicationId: test-app +# database: foo # tables: -# data.ta: +# data.tab_a: # eventType: event_type_a -# data.tb: +# data.tab_b: # eventType: event_type_b # idColumn: tb_id # payloadColumn: tb_payload # # Optional. Filter ignores events before a certain txnId and lsn. Can be used to skip bad events # filter: -# data.ta: "[?(@.source.txId > 500 && @.source.lsn > 123456)]" +# data.tab_a: "[?(@.source.txId > 500 && @.source.lsn > 123456)]" # batchSize: 1000 diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 86408e564..29aa599df 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -473,9 +473,12 @@ spec: items: type: object required: + - applicationId - database - tables properties: + applicationId: + type: string batchSize: type: integer database: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 0f105fdd4..6277a4f73 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -664,8 +664,11 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ Items: &apiextv1.JSONSchemaPropsOrArray{ Schema: &apiextv1.JSONSchemaProps{ Type: "object", - Required: []string{"database", "tables"}, + Required: []string{"applicationId", "database", "tables"}, Properties: map[string]apiextv1.JSONSchemaProps{ + "applicationId": { + Type: "string", + }, "batchSize": { Type: "integer", }, diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 8b8067fb3..561dfcd05 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -230,10 +230,11 @@ type ConnectionPooler struct { } type Stream struct { - Database string `json:"database"` - Tables map[string]StreamTable `json:"tables"` - Filter map[string]string `json:"filter,omitempty"` - BatchSize uint32 `json:"batchSize,omitempty"` + ApplicationId string `json:"applicationId"` + Database string `json:"database"` + Tables map[string]StreamTable `json:"tables"` + Filter map[string]string `json:"filter,omitempty"` + BatchSize uint32 `json:"batchSize,omitempty"` } type StreamTable struct { diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index e3374f6e1..86e3687db 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -14,16 +14,24 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func (c *Cluster) createStreams() error { +func (c *Cluster) createStreams(appId string) { c.setProcessName("creating streams") - fes := c.generateFabricEventStream() - _, err := c.KubeClient.FabricEventStreams(c.Namespace).Create(context.TODO(), fes, metav1.CreateOptions{}) + var ( + fes *zalandov1alpha1.FabricEventStream + err error + ) + + msg := "could not create event stream custom resource with applicationId %s: %v" + + fes = c.generateFabricEventStream(appId) if err != nil { - return fmt.Errorf("could not create event stream custom resource: %v", err) + c.logger.Warningf(msg, appId, err) + } + _, err = c.KubeClient.FabricEventStreams(c.Namespace).Create(context.TODO(), fes, metav1.CreateOptions{}) + if err != nil { + c.logger.Warningf(msg, appId, err) } - - return nil } func (c *Cluster) updateStreams(newEventStreams *zalandov1alpha1.FabricEventStream) error { @@ -54,21 +62,34 @@ func (c *Cluster) deleteStreams() error { return nil } +func gatherApplicationIds(streams []acidv1.Stream) []string { + appIds := make([]string, 0) + for _, stream := range streams { + if !util.SliceContains(appIds, stream.ApplicationId) { + appIds = append(appIds, stream.ApplicationId) + } + } + + return appIds +} + func (c *Cluster) syncPostgresConfig() error { + slots := make(map[string]map[string]string) desiredPatroniConfig := c.Spec.Patroni - slots := desiredPatroniConfig.Slots + if len(desiredPatroniConfig.Slots) > 0 { + slots = desiredPatroniConfig.Slots + } for _, stream := range c.Spec.Streams { - slotName := c.getLogicalReplicationSlot(stream.Database) - - if slotName == "" { - slot := map[string]string{ - "database": stream.Database, - "plugin": "wal2json", - "type": "logical", - } - slots[constants.EventStreamSourceSlotPrefix+"_"+stream.Database] = slot + slot := map[string]string{ + "database": stream.Database, + "plugin": "wal2json", + "type": "logical", + } + slotName := constants.EventStreamSourceSlotPrefix + "_" + stream.Database + "_" + stream.ApplicationId + if _, exists := slots[slotName]; !exists { + slots[slotName] = slot } } @@ -107,16 +128,13 @@ func (c *Cluster) syncPostgresConfig() error { return nil } -func (c *Cluster) generateFabricEventStream() *zalandov1alpha1.FabricEventStream { - var applicationId string +func (c *Cluster) generateFabricEventStream(appId string) *zalandov1alpha1.FabricEventStream { eventStreams := make([]zalandov1alpha1.EventStream, 0) - // take application label from manifest - if spec, err := c.GetSpec(); err == nil { - applicationId = spec.ObjectMeta.Labels["application"] - } - for _, stream := range c.Spec.Streams { + if stream.ApplicationId != appId { + continue + } for tableName, table := range stream.Tables { streamSource := c.getEventStreamSource(stream, tableName, table.IdColumn) streamFlow := getEventStreamFlow(stream, table.PayloadColumn) @@ -135,14 +153,14 @@ func (c *Cluster) generateFabricEventStream() *zalandov1alpha1.FabricEventStream APIVersion: "zalando.org/v1alpha1", }, ObjectMeta: metav1.ObjectMeta{ - Name: c.Name, + Name: c.Name + "-" + appId, Namespace: c.Namespace, Annotations: c.AnnotationsToPropagate(c.annotationsSet(nil)), // make cluster StatefulSet the owner (like with connection pooler objects) OwnerReferences: c.ownerReferences(), }, Spec: zalandov1alpha1.FabricEventStreamSpec{ - ApplicationId: applicationId, + ApplicationId: appId, EventStreams: eventStreams, }, } @@ -156,7 +174,10 @@ func (c *Cluster) getEventStreamSource(stream acidv1.Stream, tableName, idColumn Schema: schema, EventStreamTable: getOutboxTable(table, idColumn), Filter: streamFilter, - Connection: c.getStreamConnection(stream.Database, constants.EventStreamSourceSlotPrefix+constants.UserRoleNameSuffix), + Connection: c.getStreamConnection( + stream.Database, + constants.EventStreamSourceSlotPrefix+constants.UserRoleNameSuffix, + stream.ApplicationId), } } @@ -193,10 +214,10 @@ func getOutboxTable(tableName, idColumn string) zalandov1alpha1.EventStreamTable } } -func (c *Cluster) getStreamConnection(database, user string) zalandov1alpha1.Connection { +func (c *Cluster) getStreamConnection(database, user, appId string) zalandov1alpha1.Connection { return zalandov1alpha1.Connection{ Url: fmt.Sprintf("jdbc:postgresql://%s.%s/%s?user=%s&ssl=true&sslmode=require", c.Name, c.Namespace, database, user), - SlotName: c.getLogicalReplicationSlot(database), + SlotName: constants.EventStreamSourceSlotPrefix + "_" + database + "_" + appId, DBAuth: zalandov1alpha1.DBAuth{ Type: constants.EventStreamSourceAuthType, Name: c.credentialSecretNameForCluster(user, c.Name), @@ -206,16 +227,6 @@ func (c *Cluster) getStreamConnection(database, user string) zalandov1alpha1.Con } } -func (c *Cluster) getLogicalReplicationSlot(database string) string { - for slotName, slot := range c.Spec.Patroni.Slots { - if slot["type"] == "logical" && slot["database"] == database && slot["plugin"] == "wal2json" { - return slotName - } - } - - return constants.EventStreamSourceSlotPrefix + "_" + database -} - func (c *Cluster) syncStreams() error { _, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), constants.EventStreamSourceCRDName, metav1.GetOptions{}) @@ -241,25 +252,26 @@ func (c *Cluster) createOrUpdateStreams() error { return fmt.Errorf("could not update Postgres config for event streaming: %v", err) } - effectiveStreams, err := c.KubeClient.FabricEventStreams(c.Namespace).Get(context.TODO(), c.Name, metav1.GetOptions{}) - if err != nil { - if !k8sutil.ResourceNotFound(err) { - return fmt.Errorf("error during reading of event streams: %v", err) - } - - c.logger.Infof("event streams do not exist, create it") - err := c.createStreams() + appIds := gatherApplicationIds(c.Spec.Streams) + for _, appId := range appIds { + fesName := c.Name + "-" + appId + effectiveStreams, err := c.KubeClient.FabricEventStreams(c.Namespace).Get(context.TODO(), fesName, metav1.GetOptions{}) if err != nil { - return fmt.Errorf("event streams creation failed: %v", err) - } - } else { - desiredStreams := c.generateFabricEventStream() - if !reflect.DeepEqual(effectiveStreams.Spec, desiredStreams.Spec) { - c.logger.Debug("updating event streams") - desiredStreams.ObjectMeta.ResourceVersion = effectiveStreams.ObjectMeta.ResourceVersion - err = c.updateStreams(desiredStreams) - if err != nil { - return fmt.Errorf("event streams update failed: %v", err) + if !k8sutil.ResourceNotFound(err) { + return fmt.Errorf("failed reading event stream %s: %v", fesName, err) + } + + c.logger.Infof("event streams do not exist, create it") + c.createStreams(appId) + } else { + desiredStreams := c.generateFabricEventStream(appId) + if !reflect.DeepEqual(effectiveStreams.Spec, desiredStreams.Spec) { + c.logger.Debug("updating event streams") + desiredStreams.ObjectMeta.ResourceVersion = effectiveStreams.ObjectMeta.ResourceVersion + err = c.updateStreams(desiredStreams) + if err != nil { + return fmt.Errorf("failed updating event stream %s: %v", fesName, err) + } } } } diff --git a/pkg/cluster/streams_test.go b/pkg/cluster/streams_test.go index a18a00d67..aa7ba8961 100644 --- a/pkg/cluster/streams_test.go +++ b/pkg/cluster/streams_test.go @@ -37,7 +37,10 @@ func newFakeK8sStreamClient() (k8sutil.KubernetesClient, *fake.Clientset) { var ( clusterName string = "acid-test-cluster" namespace string = "default" + appId string = "test-app" + dbName string = "foo" fesUser string = constants.EventStreamSourceSlotPrefix + constants.UserRoleNameSuffix + fesName string = clusterName + "-" + appId pg = acidv1.Postgresql{ TypeMeta: metav1.TypeMeta{ @@ -47,15 +50,15 @@ var ( ObjectMeta: metav1.ObjectMeta{ Name: clusterName, Namespace: namespace, - Labels: map[string]string{"application": "test"}, }, Spec: acidv1.PostgresSpec{ Databases: map[string]string{ - "foo": "foo_user", + dbName: dbName + constants.UserRoleNameSuffix, }, Streams: []acidv1.Stream{ { - Database: "foo", + ApplicationId: appId, + Database: "foo", Tables: map[string]acidv1.StreamTable{ "data.bar": acidv1.StreamTable{ EventType: "stream_type_a", @@ -69,9 +72,6 @@ var ( BatchSize: uint32(100), }, }, - Users: map[string]acidv1.UserFlags{ - "foo_user": []string{"replication"}, - }, Volume: acidv1.Volume{ Size: "1Gi", }, @@ -84,7 +84,7 @@ var ( APIVersion: "zalando.org/v1alpha1", }, ObjectMeta: metav1.ObjectMeta{ - Name: clusterName, + Name: fesName, Namespace: namespace, OwnerReferences: []metav1.OwnerReference{ metav1.OwnerReference{ @@ -96,7 +96,7 @@ var ( }, }, Spec: v1alpha1.FabricEventStreamSpec{ - ApplicationId: "test", + ApplicationId: appId, EventStreams: []v1alpha1.EventStream{ { EventStreamFlow: v1alpha1.EventStreamFlow{ @@ -118,7 +118,7 @@ var ( UserKey: "username", }, Url: fmt.Sprintf("jdbc:postgresql://%s.%s/foo?user=%s&ssl=true&sslmode=require", clusterName, namespace, fesUser), - SlotName: "fes_foo", + SlotName: fmt.Sprintf("%s_%s_%s", constants.EventStreamSourceSlotPrefix, dbName, appId), }, Schema: "data", EventStreamTable: v1alpha1.EventStreamTable{ @@ -164,13 +164,13 @@ func TestGenerateFabricEventStream(t *testing.T) { err = cluster.createOrUpdateStreams() assert.NoError(t, err) - result := cluster.generateFabricEventStream() + result := cluster.generateFabricEventStream(appId) if !reflect.DeepEqual(result, fes) { t.Errorf("Malformed FabricEventStream, expected %#v, got %#v", fes, result) } - streamCRD, err := cluster.KubeClient.FabricEventStreams(namespace).Get(context.TODO(), cluster.Name, metav1.GetOptions{}) + streamCRD, err := cluster.KubeClient.FabricEventStreams(namespace).Get(context.TODO(), fesName, metav1.GetOptions{}) assert.NoError(t, err) if !reflect.DeepEqual(streamCRD, fes) { @@ -206,7 +206,8 @@ func TestUpdateFabricEventStream(t *testing.T) { var pgSpec acidv1.PostgresSpec pgSpec.Streams = []acidv1.Stream{ { - Database: "foo", + ApplicationId: appId, + Database: dbName, Tables: map[string]acidv1.StreamTable{ "data.bar": acidv1.StreamTable{ EventType: "stream_type_b", @@ -230,10 +231,10 @@ func TestUpdateFabricEventStream(t *testing.T) { err = cluster.createOrUpdateStreams() assert.NoError(t, err) - streamCRD, err := cluster.KubeClient.FabricEventStreams(namespace).Get(context.TODO(), cluster.Name, metav1.GetOptions{}) + streamCRD, err := cluster.KubeClient.FabricEventStreams(namespace).Get(context.TODO(), fesName, metav1.GetOptions{}) assert.NoError(t, err) - result := cluster.generateFabricEventStream() + result := cluster.generateFabricEventStream(appId) if !reflect.DeepEqual(result, streamCRD) { t.Errorf("Malformed FabricEventStream, expected %#v, got %#v", streamCRD, result) } From 8690cc90e8c1f684e53ea6815520832024a787c0 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 9 Dec 2021 12:44:33 +0100 Subject: [PATCH 29/44] add FES to RBAC in chart --- .../postgres-operator/templates/clusterrole.yaml | 14 ++++++++++++++ docs/reference/cluster_manifest.md | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index 885bad3f7..aed736049 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -34,6 +34,20 @@ rules: - get - list - watch +# all verbs allowed for event streams +- apiGroups: + - zalando.org + resources: + - fabriceventstreams + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch # to create or get/update CRDs when starting up - apiGroups: - apiextensions.k8s.io diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 8668cd72e..0079d76d6 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -530,7 +530,7 @@ have the following properties: The application name to which the database and CDC belongs to. For each set of streams with a distinct `applicationId` a separate stream CR as well as a separate logical replication slot will be created. This means there can - different streams in the same database and streams with the same + be different streams in the same database and streams with the same `applicationId` are bundled in one stream CR. The stream CR will be called like the Postgres cluster plus "-" suffix. Required. From ff1ac1da4fd3771c34637ab1f88b068f3f89f3d7 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 14 Dec 2021 18:48:32 +0100 Subject: [PATCH 30/44] replace dash with underscroe for slot name --- pkg/cluster/streams.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index 86e3687db..101d350db 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -217,7 +217,7 @@ func getOutboxTable(tableName, idColumn string) zalandov1alpha1.EventStreamTable func (c *Cluster) getStreamConnection(database, user, appId string) zalandov1alpha1.Connection { return zalandov1alpha1.Connection{ Url: fmt.Sprintf("jdbc:postgresql://%s.%s/%s?user=%s&ssl=true&sslmode=require", c.Name, c.Namespace, database, user), - SlotName: constants.EventStreamSourceSlotPrefix + "_" + database + "_" + appId, + SlotName: constants.EventStreamSourceSlotPrefix + "_" + database + "_" + strings.Replace(appId, "-", "_", -1), DBAuth: zalandov1alpha1.DBAuth{ Type: constants.EventStreamSourceAuthType, Name: c.credentialSecretNameForCluster(user, c.Name), From 7079d0ef8013e58d4f0186de9a5ed7e3987860bc Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 15 Dec 2021 10:32:50 +0100 Subject: [PATCH 31/44] fix unit test, too --- pkg/cluster/streams_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/cluster/streams_test.go b/pkg/cluster/streams_test.go index aa7ba8961..99bf333a4 100644 --- a/pkg/cluster/streams_test.go +++ b/pkg/cluster/streams_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "reflect" + "strings" "context" "testing" @@ -118,7 +119,7 @@ var ( UserKey: "username", }, Url: fmt.Sprintf("jdbc:postgresql://%s.%s/foo?user=%s&ssl=true&sslmode=require", clusterName, namespace, fesUser), - SlotName: fmt.Sprintf("%s_%s_%s", constants.EventStreamSourceSlotPrefix, dbName, appId), + SlotName: fmt.Sprintf("%s_%s_%s", constants.EventStreamSourceSlotPrefix, dbName, strings.Replace(appId, "-", "_", -1)), }, Schema: "data", EventStreamTable: v1alpha1.EventStreamTable{ From dfec6fd6b0ac138574c58a27000a11ff89cfd87b Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 15 Dec 2021 10:48:49 +0100 Subject: [PATCH 32/44] fix codegen to support both apis --- hack/update-codegen.sh | 2 +- pkg/generated/clientset/versioned/clientset.go | 3 +++ pkg/generated/informers/externalversions/generic.go | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh index c5df6b375..eb7da7ced 100755 --- a/hack/update-codegen.sh +++ b/hack/update-codegen.sh @@ -17,7 +17,7 @@ trap "cleanup" EXIT SIGINT bash "${CODEGEN_PKG}/generate-groups.sh" all \ "${OPERATOR_PACKAGE_ROOT}/pkg/generated" "${OPERATOR_PACKAGE_ROOT}/pkg/apis" \ - "zalando.org:v1alpha1" \ + "acid.zalan.do:v1 zalando.org:v1alpha1" \ --go-header-file "${SCRIPT_ROOT}"/hack/custom-boilerplate.go.txt cp -r "${OPERATOR_PACKAGE_ROOT}"/pkg/* "${TARGET_CODE_DIR}" diff --git a/pkg/generated/clientset/versioned/clientset.go b/pkg/generated/clientset/versioned/clientset.go index 73a3863db..3d6282a0a 100644 --- a/pkg/generated/clientset/versioned/clientset.go +++ b/pkg/generated/clientset/versioned/clientset.go @@ -80,6 +80,9 @@ func NewForConfig(c *rest.Config) (*Clientset, error) { var cs Clientset var err error cs.acidV1, err = acidv1.NewForConfig(&configShallowCopy) + if err != nil { + return nil, err + } cs.zalandoV1alpha1, err = zalandov1alpha1.NewForConfig(&configShallowCopy) if err != nil { return nil, err diff --git a/pkg/generated/informers/externalversions/generic.go b/pkg/generated/informers/externalversions/generic.go index 23515c041..a58a2ec24 100644 --- a/pkg/generated/informers/externalversions/generic.go +++ b/pkg/generated/informers/externalversions/generic.go @@ -65,7 +65,7 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource case v1.SchemeGroupVersion.WithResource("postgresqls"): return &genericInformer{resource: resource.GroupResource(), informer: f.Acid().V1().Postgresqls().Informer()}, nil - // Group=zalando.org, Version=v1alpha1 + // Group=zalando.org, Version=v1alpha1 case v1alpha1.SchemeGroupVersion.WithResource("fabriceventstreams"): return &genericInformer{resource: resource.GroupResource(), informer: f.Zalando().V1alpha1().FabricEventStreams().Informer()}, nil From 284649500e308e206db4d6fb3290ee7c97e99b7d Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 15 Dec 2021 13:46:46 +0100 Subject: [PATCH 33/44] disable streams in chart --- .../templates/clusterrole.yaml | 2 ++ charts/postgres-operator/values.yaml | 3 ++ manifests/operator-service-account-rbac.yaml | 28 +++++++++---------- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index aed736049..55fead75f 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -35,6 +35,7 @@ rules: - list - watch # all verbs allowed for event streams +{{- if .Values.enableStreams }} - apiGroups: - zalando.org resources: @@ -48,6 +49,7 @@ rules: - patch - update - watch +{{- end }} # to create or get/update CRDs when starting up - apiGroups: - apiextensions.k8s.io diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 65619845a..21ba9f935 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -362,6 +362,9 @@ configConnectionPooler: connection_pooler_default_cpu_limit: "1" connection_pooler_default_memory_limit: 100Mi +# Zalando's internal CDC stream feature +enableStreams: false + rbac: # Specifies whether RBAC resources should be created create: true diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index e983d71a0..882ad7281 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -35,20 +35,20 @@ rules: - get - list - watch -# all verbs allowed for event streams -- apiGroups: - - zalando.org - resources: - - fabriceventstreams - verbs: - - create - - delete - - deletecollection - - get - - list - - patch - - update - - watch +# all verbs allowed for event streams (Zalando-internal feature) +#- apiGroups: +# - zalando.org +# resources: +# - fabriceventstreams +# verbs: +# - create +# - delete +# - deletecollection +# - get +# - list +# - patch +# - update +# - watch # to create or get/update CRDs when starting up - apiGroups: - apiextensions.k8s.io From a686824ab80e7939fdfde9df6b158fe5eeeb5e12 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 24 Jan 2022 17:19:07 +0100 Subject: [PATCH 34/44] move FES to v1 --- docs/reference/cluster_manifest.md | 2 +- hack/update-codegen.sh | 2 +- .../acid.zalan.do/v1/zz_generated.deepcopy.go | 2 +- .../{v1alpha1 => v1}/fabriceventstream.go | 2 +- .../zalando.org/{v1alpha1 => v1}/register.go | 4 +- .../{v1alpha1 => v1}/zz_generated.deepcopy.go | 2 +- pkg/cluster/streams.go | 42 +++++++-------- pkg/cluster/streams_test.go | 28 +++++----- .../clientset/versioned/clientset.go | 22 ++++---- pkg/generated/clientset/versioned/doc.go | 2 +- .../versioned/fake/clientset_generated.go | 12 ++--- pkg/generated/clientset/versioned/fake/doc.go | 2 +- .../clientset/versioned/fake/register.go | 6 +-- .../clientset/versioned/scheme/doc.go | 2 +- .../clientset/versioned/scheme/register.go | 6 +-- .../acid.zalan.do/v1/acid.zalan.do_client.go | 2 +- .../versioned/typed/acid.zalan.do/v1/doc.go | 2 +- .../typed/acid.zalan.do/v1/fake/doc.go | 2 +- .../v1/fake/fake_acid.zalan.do_client.go | 2 +- .../v1/fake/fake_operatorconfiguration.go | 2 +- .../acid.zalan.do/v1/fake/fake_postgresql.go | 2 +- .../v1/fake/fake_postgresteam.go | 2 +- .../acid.zalan.do/v1/generated_expansion.go | 2 +- .../acid.zalan.do/v1/operatorconfiguration.go | 2 +- .../typed/acid.zalan.do/v1/postgresql.go | 2 +- .../typed/acid.zalan.do/v1/postgresteam.go | 2 +- .../typed/zalando.org/{v1alpha1 => v1}/doc.go | 4 +- .../{v1alpha1 => v1}/fabriceventstream.go | 52 +++++++++---------- .../zalando.org/{v1alpha1 => v1}/fake/doc.go | 2 +- .../fake/fake_fabriceventstream.go | 46 ++++++++-------- .../fake/fake_zalando.org_client.go | 10 ++-- .../{v1alpha1 => v1}/generated_expansion.go | 4 +- .../{v1alpha1 => v1}/zalando.org_client.go | 34 ++++++------ .../acid.zalan.do/interface.go | 2 +- .../acid.zalan.do/v1/interface.go | 2 +- .../acid.zalan.do/v1/postgresql.go | 2 +- .../acid.zalan.do/v1/postgresteam.go | 2 +- .../informers/externalversions/factory.go | 2 +- .../informers/externalversions/generic.go | 10 ++-- .../internalinterfaces/factory_interfaces.go | 2 +- .../externalversions/zalando.org/interface.go | 14 ++--- .../{v1alpha1 => v1}/fabriceventstream.go | 28 +++++----- .../zalando.org/{v1alpha1 => v1}/interface.go | 4 +- .../acid.zalan.do/v1/expansion_generated.go | 2 +- .../listers/acid.zalan.do/v1/postgresql.go | 2 +- .../listers/acid.zalan.do/v1/postgresteam.go | 2 +- .../{v1alpha1 => v1}/expansion_generated.go | 4 +- .../{v1alpha1 => v1}/fabriceventstream.go | 26 +++++----- pkg/util/k8sutil/k8sutil.go | 14 ++--- pkg/util/patroni/patroni_test.go | 6 +-- 50 files changed, 217 insertions(+), 217 deletions(-) rename pkg/apis/zalando.org/{v1alpha1 => v1}/fabriceventstream.go (99%) rename pkg/apis/zalando.org/{v1alpha1 => v1}/register.go (96%) rename pkg/apis/zalando.org/{v1alpha1 => v1}/zz_generated.deepcopy.go (99%) rename pkg/generated/clientset/versioned/typed/zalando.org/{v1alpha1 => v1}/doc.go (95%) rename pkg/generated/clientset/versioned/typed/zalando.org/{v1alpha1 => v1}/fabriceventstream.go (71%) rename pkg/generated/clientset/versioned/typed/zalando.org/{v1alpha1 => v1}/fake/doc.go (97%) rename pkg/generated/clientset/versioned/typed/zalando.org/{v1alpha1 => v1}/fake/fake_fabriceventstream.go (73%) rename pkg/generated/clientset/versioned/typed/zalando.org/{v1alpha1 => v1}/fake/fake_zalando.org_client.go (80%) rename pkg/generated/clientset/versioned/typed/zalando.org/{v1alpha1 => v1}/generated_expansion.go (95%) rename pkg/generated/clientset/versioned/typed/zalando.org/{v1alpha1 => v1}/zalando.org_client.go (67%) rename pkg/generated/informers/externalversions/zalando.org/{v1alpha1 => v1}/fabriceventstream.go (78%) rename pkg/generated/informers/externalversions/zalando.org/{v1alpha1 => v1}/interface.go (97%) rename pkg/generated/listers/zalando.org/{v1alpha1 => v1}/expansion_generated.go (96%) rename pkg/generated/listers/zalando.org/{v1alpha1 => v1}/fabriceventstream.go (82%) diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 0079d76d6..07b03ec9f 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -538,7 +538,7 @@ have the following properties: Name of the database from where events will be published via Postgres' logical decoding feature. The operator will take care of updating the database configuration (setting `wal_level: logical`, creating logical - replication slots, using output plugin `wal2json` and creating a dedicated + replication slots, using output plugin `pgoutput` and creating a dedicated replication user). Required. * **tables** diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh index eb7da7ced..d34db9c45 100755 --- a/hack/update-codegen.sh +++ b/hack/update-codegen.sh @@ -17,7 +17,7 @@ trap "cleanup" EXIT SIGINT bash "${CODEGEN_PKG}/generate-groups.sh" all \ "${OPERATOR_PACKAGE_ROOT}/pkg/generated" "${OPERATOR_PACKAGE_ROOT}/pkg/apis" \ - "acid.zalan.do:v1 zalando.org:v1alpha1" \ + "acid.zalan.do:v1 zalando.org:v1" \ --go-header-file "${SCRIPT_ROOT}"/hack/custom-boilerplate.go.txt cp -r "${OPERATOR_PACKAGE_ROOT}"/pkg/* "${TARGET_CODE_DIR}" diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index f5c669d9e..42a701fd0 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -2,7 +2,7 @@ // +build !ignore_autogenerated /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/apis/zalando.org/v1alpha1/fabriceventstream.go b/pkg/apis/zalando.org/v1/fabriceventstream.go similarity index 99% rename from pkg/apis/zalando.org/v1alpha1/fabriceventstream.go rename to pkg/apis/zalando.org/v1/fabriceventstream.go index 2ef0b6405..7990d7700 100644 --- a/pkg/apis/zalando.org/v1alpha1/fabriceventstream.go +++ b/pkg/apis/zalando.org/v1/fabriceventstream.go @@ -1,4 +1,4 @@ -package v1alpha1 +package v1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/apis/zalando.org/v1alpha1/register.go b/pkg/apis/zalando.org/v1/register.go similarity index 96% rename from pkg/apis/zalando.org/v1alpha1/register.go rename to pkg/apis/zalando.org/v1/register.go index 0136a82ec..33a2c718b 100644 --- a/pkg/apis/zalando.org/v1alpha1/register.go +++ b/pkg/apis/zalando.org/v1/register.go @@ -1,4 +1,4 @@ -package v1alpha1 +package v1 import ( "github.com/zalando/postgres-operator/pkg/apis/zalando.org" @@ -10,7 +10,7 @@ import ( // APIVersion of the `fabriceventstream` CRD const ( - APIVersion = "v1alpha1" + APIVersion = "v1" ) var ( diff --git a/pkg/apis/zalando.org/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/zalando.org/v1/zz_generated.deepcopy.go similarity index 99% rename from pkg/apis/zalando.org/v1alpha1/zz_generated.deepcopy.go rename to pkg/apis/zalando.org/v1/zz_generated.deepcopy.go index 0327279e5..a44439a94 100644 --- a/pkg/apis/zalando.org/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/zalando.org/v1/zz_generated.deepcopy.go @@ -24,7 +24,7 @@ SOFTWARE. // Code generated by deepcopy-gen. DO NOT EDIT. -package v1alpha1 +package v1 import ( runtime "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index 101d350db..7053aeb20 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -7,7 +7,7 @@ import ( "strings" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - zalandov1alpha1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1alpha1" + zalandov1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" @@ -18,7 +18,7 @@ func (c *Cluster) createStreams(appId string) { c.setProcessName("creating streams") var ( - fes *zalandov1alpha1.FabricEventStream + fes *zalandov1.FabricEventStream err error ) @@ -34,7 +34,7 @@ func (c *Cluster) createStreams(appId string) { } } -func (c *Cluster) updateStreams(newEventStreams *zalandov1alpha1.FabricEventStream) error { +func (c *Cluster) updateStreams(newEventStreams *zalandov1.FabricEventStream) error { c.setProcessName("updating event streams") _, err := c.KubeClient.FabricEventStreams(newEventStreams.Namespace).Update(context.TODO(), newEventStreams, metav1.UpdateOptions{}) @@ -84,7 +84,7 @@ func (c *Cluster) syncPostgresConfig() error { for _, stream := range c.Spec.Streams { slot := map[string]string{ "database": stream.Database, - "plugin": "wal2json", + "plugin": "pgoutput", "type": "logical", } slotName := constants.EventStreamSourceSlotPrefix + "_" + stream.Database + "_" + stream.ApplicationId @@ -128,8 +128,8 @@ func (c *Cluster) syncPostgresConfig() error { return nil } -func (c *Cluster) generateFabricEventStream(appId string) *zalandov1alpha1.FabricEventStream { - eventStreams := make([]zalandov1alpha1.EventStream, 0) +func (c *Cluster) generateFabricEventStream(appId string) *zalandov1.FabricEventStream { + eventStreams := make([]zalandov1.EventStream, 0) for _, stream := range c.Spec.Streams { if stream.ApplicationId != appId { @@ -140,17 +140,17 @@ func (c *Cluster) generateFabricEventStream(appId string) *zalandov1alpha1.Fabri streamFlow := getEventStreamFlow(stream, table.PayloadColumn) streamSink := getEventStreamSink(stream, table.EventType) - eventStreams = append(eventStreams, zalandov1alpha1.EventStream{ + eventStreams = append(eventStreams, zalandov1.EventStream{ EventStreamFlow: streamFlow, EventStreamSink: streamSink, EventStreamSource: streamSource}) } } - return &zalandov1alpha1.FabricEventStream{ + return &zalandov1.FabricEventStream{ TypeMeta: metav1.TypeMeta{ Kind: constants.EventStreamSourceCRDKind, - APIVersion: "zalando.org/v1alpha1", + APIVersion: "zalando.org/v1", }, ObjectMeta: metav1.ObjectMeta{ Name: c.Name + "-" + appId, @@ -159,17 +159,17 @@ func (c *Cluster) generateFabricEventStream(appId string) *zalandov1alpha1.Fabri // make cluster StatefulSet the owner (like with connection pooler objects) OwnerReferences: c.ownerReferences(), }, - Spec: zalandov1alpha1.FabricEventStreamSpec{ + Spec: zalandov1.FabricEventStreamSpec{ ApplicationId: appId, EventStreams: eventStreams, }, } } -func (c *Cluster) getEventStreamSource(stream acidv1.Stream, tableName, idColumn string) zalandov1alpha1.EventStreamSource { +func (c *Cluster) getEventStreamSource(stream acidv1.Stream, tableName, idColumn string) zalandov1.EventStreamSource { table, schema := getTableSchema(tableName) streamFilter := stream.Filter[tableName] - return zalandov1alpha1.EventStreamSource{ + return zalandov1.EventStreamSource{ Type: constants.EventStreamSourcePGType, Schema: schema, EventStreamTable: getOutboxTable(table, idColumn), @@ -181,15 +181,15 @@ func (c *Cluster) getEventStreamSource(stream acidv1.Stream, tableName, idColumn } } -func getEventStreamFlow(stream acidv1.Stream, payloadColumn string) zalandov1alpha1.EventStreamFlow { - return zalandov1alpha1.EventStreamFlow{ +func getEventStreamFlow(stream acidv1.Stream, payloadColumn string) zalandov1.EventStreamFlow { + return zalandov1.EventStreamFlow{ Type: constants.EventStreamFlowPgGenericType, PayloadColumn: payloadColumn, } } -func getEventStreamSink(stream acidv1.Stream, eventType string) zalandov1alpha1.EventStreamSink { - return zalandov1alpha1.EventStreamSink{ +func getEventStreamSink(stream acidv1.Stream, eventType string) zalandov1.EventStreamSink { + return zalandov1.EventStreamSink{ Type: constants.EventStreamSinkNakadiType, EventType: eventType, MaxBatchSize: stream.BatchSize, @@ -207,18 +207,18 @@ func getTableSchema(fullTableName string) (tableName, schemaName string) { return tableName, schemaName } -func getOutboxTable(tableName, idColumn string) zalandov1alpha1.EventStreamTable { - return zalandov1alpha1.EventStreamTable{ +func getOutboxTable(tableName, idColumn string) zalandov1.EventStreamTable { + return zalandov1.EventStreamTable{ Name: tableName, IDColumn: idColumn, } } -func (c *Cluster) getStreamConnection(database, user, appId string) zalandov1alpha1.Connection { - return zalandov1alpha1.Connection{ +func (c *Cluster) getStreamConnection(database, user, appId string) zalandov1.Connection { + return zalandov1.Connection{ Url: fmt.Sprintf("jdbc:postgresql://%s.%s/%s?user=%s&ssl=true&sslmode=require", c.Name, c.Namespace, database, user), SlotName: constants.EventStreamSourceSlotPrefix + "_" + database + "_" + strings.Replace(appId, "-", "_", -1), - DBAuth: zalandov1alpha1.DBAuth{ + DBAuth: zalandov1.DBAuth{ Type: constants.EventStreamSourceAuthType, Name: c.credentialSecretNameForCluster(user, c.Name), UserKey: "username", diff --git a/pkg/cluster/streams_test.go b/pkg/cluster/streams_test.go index 99bf333a4..27d3e91f3 100644 --- a/pkg/cluster/streams_test.go +++ b/pkg/cluster/streams_test.go @@ -11,8 +11,8 @@ import ( "github.com/stretchr/testify/assert" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1alpha1" - fakezalandov1alpha1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake" + v1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" + fakezalandov1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/fake" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" @@ -24,11 +24,11 @@ import ( ) func newFakeK8sStreamClient() (k8sutil.KubernetesClient, *fake.Clientset) { - zalandoClientSet := fakezalandov1alpha1.NewSimpleClientset() + zalandoClientSet := fakezalandov1.NewSimpleClientset() clientSet := fake.NewSimpleClientset() return k8sutil.KubernetesClient{ - FabricEventStreamsGetter: zalandoClientSet.ZalandoV1alpha1(), + FabricEventStreamsGetter: zalandoClientSet.ZalandoV1(), PostgresqlsGetter: zalandoClientSet.AcidV1(), PodsGetter: clientSet.CoreV1(), StatefulSetsGetter: clientSet.AppsV1(), @@ -79,10 +79,10 @@ var ( }, } - fes = &v1alpha1.FabricEventStream{ + fes = &v1.FabricEventStream{ TypeMeta: metav1.TypeMeta{ Kind: "FabricEventStream", - APIVersion: "zalando.org/v1alpha1", + APIVersion: "zalando.org/v1", }, ObjectMeta: metav1.ObjectMeta{ Name: fesName, @@ -96,23 +96,23 @@ var ( }, }, }, - Spec: v1alpha1.FabricEventStreamSpec{ + Spec: v1.FabricEventStreamSpec{ ApplicationId: appId, - EventStreams: []v1alpha1.EventStream{ + EventStreams: []v1.EventStream{ { - EventStreamFlow: v1alpha1.EventStreamFlow{ + EventStreamFlow: v1.EventStreamFlow{ PayloadColumn: "b_payload", Type: constants.EventStreamFlowPgGenericType, }, - EventStreamSink: v1alpha1.EventStreamSink{ + EventStreamSink: v1.EventStreamSink{ EventType: "stream_type_a", MaxBatchSize: uint32(100), Type: constants.EventStreamSinkNakadiType, }, - EventStreamSource: v1alpha1.EventStreamSource{ + EventStreamSource: v1.EventStreamSource{ Filter: "[?(@.source.txId > 500 && @.source.lsn > 123456)]", - Connection: v1alpha1.Connection{ - DBAuth: v1alpha1.DBAuth{ + Connection: v1.Connection{ + DBAuth: v1.DBAuth{ Name: fmt.Sprintf("fes-user.%s.credentials.postgresql.acid.zalan.do", clusterName), PasswordKey: "password", Type: constants.EventStreamSourceAuthType, @@ -122,7 +122,7 @@ var ( SlotName: fmt.Sprintf("%s_%s_%s", constants.EventStreamSourceSlotPrefix, dbName, strings.Replace(appId, "-", "_", -1)), }, Schema: "data", - EventStreamTable: v1alpha1.EventStreamTable{ + EventStreamTable: v1.EventStreamTable{ IDColumn: "b_id", Name: "bar", }, diff --git a/pkg/generated/clientset/versioned/clientset.go b/pkg/generated/clientset/versioned/clientset.go index 3d6282a0a..50f0ac841 100644 --- a/pkg/generated/clientset/versioned/clientset.go +++ b/pkg/generated/clientset/versioned/clientset.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -28,7 +28,7 @@ import ( "fmt" acidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" - zalandov1alpha1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1" + zalandov1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1" discovery "k8s.io/client-go/discovery" rest "k8s.io/client-go/rest" flowcontrol "k8s.io/client-go/util/flowcontrol" @@ -37,15 +37,15 @@ import ( type Interface interface { Discovery() discovery.DiscoveryInterface AcidV1() acidv1.AcidV1Interface - ZalandoV1alpha1() zalandov1alpha1.ZalandoV1alpha1Interface + ZalandoV1() zalandov1.ZalandoV1Interface } // Clientset contains the clients for groups. Each group has exactly one // version included in a Clientset. type Clientset struct { *discovery.DiscoveryClient - acidV1 *acidv1.AcidV1Client - zalandoV1alpha1 *zalandov1alpha1.ZalandoV1alpha1Client + acidV1 *acidv1.AcidV1Client + zalandoV1 *zalandov1.ZalandoV1Client } // AcidV1 retrieves the AcidV1Client @@ -53,9 +53,9 @@ func (c *Clientset) AcidV1() acidv1.AcidV1Interface { return c.acidV1 } -// ZalandoV1alpha1 retrieves the ZalandoV1alpha1Client -func (c *Clientset) ZalandoV1alpha1() zalandov1alpha1.ZalandoV1alpha1Interface { - return c.zalandoV1alpha1 +// ZalandoV1 retrieves the ZalandoV1Client +func (c *Clientset) ZalandoV1() zalandov1.ZalandoV1Interface { + return c.zalandoV1 } // Discovery retrieves the DiscoveryClient @@ -83,7 +83,7 @@ func NewForConfig(c *rest.Config) (*Clientset, error) { if err != nil { return nil, err } - cs.zalandoV1alpha1, err = zalandov1alpha1.NewForConfig(&configShallowCopy) + cs.zalandoV1, err = zalandov1.NewForConfig(&configShallowCopy) if err != nil { return nil, err } @@ -100,7 +100,7 @@ func NewForConfig(c *rest.Config) (*Clientset, error) { func NewForConfigOrDie(c *rest.Config) *Clientset { var cs Clientset cs.acidV1 = acidv1.NewForConfigOrDie(c) - cs.zalandoV1alpha1 = zalandov1alpha1.NewForConfigOrDie(c) + cs.zalandoV1 = zalandov1.NewForConfigOrDie(c) cs.DiscoveryClient = discovery.NewDiscoveryClientForConfigOrDie(c) return &cs @@ -110,7 +110,7 @@ func NewForConfigOrDie(c *rest.Config) *Clientset { func New(c rest.Interface) *Clientset { var cs Clientset cs.acidV1 = acidv1.New(c) - cs.zalandoV1alpha1 = zalandov1alpha1.New(c) + cs.zalandoV1 = zalandov1.New(c) cs.DiscoveryClient = discovery.NewDiscoveryClient(c) return &cs diff --git a/pkg/generated/clientset/versioned/doc.go b/pkg/generated/clientset/versioned/doc.go index ae87609f6..ebb4ab535 100644 --- a/pkg/generated/clientset/versioned/doc.go +++ b/pkg/generated/clientset/versioned/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/fake/clientset_generated.go b/pkg/generated/clientset/versioned/fake/clientset_generated.go index 49e891b58..4c94d23a6 100644 --- a/pkg/generated/clientset/versioned/fake/clientset_generated.go +++ b/pkg/generated/clientset/versioned/fake/clientset_generated.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -28,8 +28,8 @@ import ( clientset "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned" acidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" fakeacidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake" - zalandov1alpha1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1" - fakezalandov1alpha1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fake" + zalandov1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1" + fakezalandov1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/discovery" @@ -92,7 +92,7 @@ func (c *Clientset) AcidV1() acidv1.AcidV1Interface { return &fakeacidv1.FakeAcidV1{Fake: &c.Fake} } -// ZalandoV1alpha1 retrieves the ZalandoV1alpha1Client -func (c *Clientset) ZalandoV1alpha1() zalandov1alpha1.ZalandoV1alpha1Interface { - return &fakezalandov1alpha1.FakeZalandoV1alpha1{Fake: &c.Fake} +// ZalandoV1 retrieves the ZalandoV1Client +func (c *Clientset) ZalandoV1() zalandov1.ZalandoV1Interface { + return &fakezalandov1.FakeZalandoV1{Fake: &c.Fake} } diff --git a/pkg/generated/clientset/versioned/fake/doc.go b/pkg/generated/clientset/versioned/fake/doc.go index bc1c91a11..b9e99ecf2 100644 --- a/pkg/generated/clientset/versioned/fake/doc.go +++ b/pkg/generated/clientset/versioned/fake/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/fake/register.go b/pkg/generated/clientset/versioned/fake/register.go index b022f3e76..313eeacc2 100644 --- a/pkg/generated/clientset/versioned/fake/register.go +++ b/pkg/generated/clientset/versioned/fake/register.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -26,7 +26,7 @@ package fake import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - zalandov1alpha1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1alpha1" + zalandov1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -39,7 +39,7 @@ var codecs = serializer.NewCodecFactory(scheme) var localSchemeBuilder = runtime.SchemeBuilder{ acidv1.AddToScheme, - zalandov1alpha1.AddToScheme, + zalandov1.AddToScheme, } // AddToScheme adds all types of this clientset into the given scheme. This allows composition diff --git a/pkg/generated/clientset/versioned/scheme/doc.go b/pkg/generated/clientset/versioned/scheme/doc.go index cd594164b..387784624 100644 --- a/pkg/generated/clientset/versioned/scheme/doc.go +++ b/pkg/generated/clientset/versioned/scheme/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/scheme/register.go b/pkg/generated/clientset/versioned/scheme/register.go index 5fc5886ea..823909bcb 100644 --- a/pkg/generated/clientset/versioned/scheme/register.go +++ b/pkg/generated/clientset/versioned/scheme/register.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -26,7 +26,7 @@ package scheme import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - zalandov1alpha1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1alpha1" + zalandov1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -39,7 +39,7 @@ var Codecs = serializer.NewCodecFactory(Scheme) var ParameterCodec = runtime.NewParameterCodec(Scheme) var localSchemeBuilder = runtime.SchemeBuilder{ acidv1.AddToScheme, - zalandov1alpha1.AddToScheme, + zalandov1.AddToScheme, } // AddToScheme adds all types of this clientset into the given scheme. This allows composition diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go index 5666201d4..d36078922 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/acid.zalan.do_client.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go index eb8fcf1f4..ba729b8ca 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go index c5fd1c04b..380725443 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go index 03e7dda94..c235b2761 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_acid.zalan.do_client.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go index c03ea7d94..fbea4930c 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_operatorconfiguration.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go index 01a0ed7a4..cb39ea290 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresql.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresteam.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresteam.go index b333ae046..59cf600aa 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresteam.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/fake/fake_postgresteam.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go index b4e99cbc8..a40f2bdb0 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/generated_expansion.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go index be22e075d..a73fb0df6 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/operatorconfiguration.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go index 5241cfb54..c58ed03ed 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresql.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresteam.go b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresteam.go index 96fbb882a..002254ef1 100644 --- a/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresteam.go +++ b/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1/postgresteam.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/doc.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/doc.go similarity index 95% rename from pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/doc.go rename to pkg/generated/clientset/versioned/typed/zalando.org/v1/doc.go index a9896a348..ba729b8ca 100644 --- a/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/doc.go +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -23,4 +23,4 @@ SOFTWARE. // Code generated by client-gen. DO NOT EDIT. // This package has the automatically generated typed clients. -package v1alpha1 +package v1 diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fabriceventstream.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fabriceventstream.go similarity index 71% rename from pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fabriceventstream.go rename to pkg/generated/clientset/versioned/typed/zalando.org/v1/fabriceventstream.go index ec93fe65a..26c3431f2 100644 --- a/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fabriceventstream.go +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fabriceventstream.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -22,15 +22,15 @@ SOFTWARE. // Code generated by client-gen. DO NOT EDIT. -package v1alpha1 +package v1 import ( "context" "time" - v1alpha1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1alpha1" + v1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" scheme "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" rest "k8s.io/client-go/rest" @@ -44,14 +44,14 @@ type FabricEventStreamsGetter interface { // FabricEventStreamInterface has methods to work with FabricEventStream resources. type FabricEventStreamInterface interface { - Create(ctx context.Context, fabricEventStream *v1alpha1.FabricEventStream, opts v1.CreateOptions) (*v1alpha1.FabricEventStream, error) - Update(ctx context.Context, fabricEventStream *v1alpha1.FabricEventStream, opts v1.UpdateOptions) (*v1alpha1.FabricEventStream, error) - Delete(ctx context.Context, name string, opts v1.DeleteOptions) error - DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error - Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.FabricEventStream, error) - List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.FabricEventStreamList, error) - Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) - Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.FabricEventStream, err error) + Create(ctx context.Context, fabricEventStream *v1.FabricEventStream, opts metav1.CreateOptions) (*v1.FabricEventStream, error) + Update(ctx context.Context, fabricEventStream *v1.FabricEventStream, opts metav1.UpdateOptions) (*v1.FabricEventStream, error) + Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error + Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.FabricEventStream, error) + List(ctx context.Context, opts metav1.ListOptions) (*v1.FabricEventStreamList, error) + Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.FabricEventStream, err error) FabricEventStreamExpansion } @@ -62,7 +62,7 @@ type fabricEventStreams struct { } // newFabricEventStreams returns a FabricEventStreams -func newFabricEventStreams(c *ZalandoV1alpha1Client, namespace string) *fabricEventStreams { +func newFabricEventStreams(c *ZalandoV1Client, namespace string) *fabricEventStreams { return &fabricEventStreams{ client: c.RESTClient(), ns: namespace, @@ -70,8 +70,8 @@ func newFabricEventStreams(c *ZalandoV1alpha1Client, namespace string) *fabricEv } // Get takes name of the fabricEventStream, and returns the corresponding fabricEventStream object, and an error if there is any. -func (c *fabricEventStreams) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.FabricEventStream, err error) { - result = &v1alpha1.FabricEventStream{} +func (c *fabricEventStreams) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.FabricEventStream, err error) { + result = &v1.FabricEventStream{} err = c.client.Get(). Namespace(c.ns). Resource("fabriceventstreams"). @@ -83,12 +83,12 @@ func (c *fabricEventStreams) Get(ctx context.Context, name string, options v1.Ge } // List takes label and field selectors, and returns the list of FabricEventStreams that match those selectors. -func (c *fabricEventStreams) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.FabricEventStreamList, err error) { +func (c *fabricEventStreams) List(ctx context.Context, opts metav1.ListOptions) (result *v1.FabricEventStreamList, err error) { var timeout time.Duration if opts.TimeoutSeconds != nil { timeout = time.Duration(*opts.TimeoutSeconds) * time.Second } - result = &v1alpha1.FabricEventStreamList{} + result = &v1.FabricEventStreamList{} err = c.client.Get(). Namespace(c.ns). Resource("fabriceventstreams"). @@ -100,7 +100,7 @@ func (c *fabricEventStreams) List(ctx context.Context, opts v1.ListOptions) (res } // Watch returns a watch.Interface that watches the requested fabricEventStreams. -func (c *fabricEventStreams) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { +func (c *fabricEventStreams) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { var timeout time.Duration if opts.TimeoutSeconds != nil { timeout = time.Duration(*opts.TimeoutSeconds) * time.Second @@ -115,8 +115,8 @@ func (c *fabricEventStreams) Watch(ctx context.Context, opts v1.ListOptions) (wa } // Create takes the representation of a fabricEventStream and creates it. Returns the server's representation of the fabricEventStream, and an error, if there is any. -func (c *fabricEventStreams) Create(ctx context.Context, fabricEventStream *v1alpha1.FabricEventStream, opts v1.CreateOptions) (result *v1alpha1.FabricEventStream, err error) { - result = &v1alpha1.FabricEventStream{} +func (c *fabricEventStreams) Create(ctx context.Context, fabricEventStream *v1.FabricEventStream, opts metav1.CreateOptions) (result *v1.FabricEventStream, err error) { + result = &v1.FabricEventStream{} err = c.client.Post(). Namespace(c.ns). Resource("fabriceventstreams"). @@ -128,8 +128,8 @@ func (c *fabricEventStreams) Create(ctx context.Context, fabricEventStream *v1al } // Update takes the representation of a fabricEventStream and updates it. Returns the server's representation of the fabricEventStream, and an error, if there is any. -func (c *fabricEventStreams) Update(ctx context.Context, fabricEventStream *v1alpha1.FabricEventStream, opts v1.UpdateOptions) (result *v1alpha1.FabricEventStream, err error) { - result = &v1alpha1.FabricEventStream{} +func (c *fabricEventStreams) Update(ctx context.Context, fabricEventStream *v1.FabricEventStream, opts metav1.UpdateOptions) (result *v1.FabricEventStream, err error) { + result = &v1.FabricEventStream{} err = c.client.Put(). Namespace(c.ns). Resource("fabriceventstreams"). @@ -142,7 +142,7 @@ func (c *fabricEventStreams) Update(ctx context.Context, fabricEventStream *v1al } // Delete takes name of the fabricEventStream and deletes it. Returns an error if one occurs. -func (c *fabricEventStreams) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { +func (c *fabricEventStreams) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { return c.client.Delete(). Namespace(c.ns). Resource("fabriceventstreams"). @@ -153,7 +153,7 @@ func (c *fabricEventStreams) Delete(ctx context.Context, name string, opts v1.De } // DeleteCollection deletes a collection of objects. -func (c *fabricEventStreams) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { +func (c *fabricEventStreams) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error { var timeout time.Duration if listOpts.TimeoutSeconds != nil { timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second @@ -169,8 +169,8 @@ func (c *fabricEventStreams) DeleteCollection(ctx context.Context, opts v1.Delet } // Patch applies the patch and returns the patched fabricEventStream. -func (c *fabricEventStreams) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.FabricEventStream, err error) { - result = &v1alpha1.FabricEventStream{} +func (c *fabricEventStreams) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.FabricEventStream, err error) { + result = &v1.FabricEventStream{} err = c.client.Patch(pt). Namespace(c.ns). Resource("fabriceventstreams"). diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fake/doc.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/doc.go similarity index 97% rename from pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fake/doc.go rename to pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/doc.go index c5fd1c04b..380725443 100644 --- a/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fake/doc.go +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fake/fake_fabriceventstream.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_fabriceventstream.go similarity index 73% rename from pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fake/fake_fabriceventstream.go rename to pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_fabriceventstream.go index 9fd2485de..8d76e06de 100644 --- a/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fake/fake_fabriceventstream.go +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_fabriceventstream.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -27,7 +27,7 @@ package fake import ( "context" - v1alpha1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1alpha1" + zalandoorgv1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" labels "k8s.io/apimachinery/pkg/labels" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -38,29 +38,29 @@ import ( // FakeFabricEventStreams implements FabricEventStreamInterface type FakeFabricEventStreams struct { - Fake *FakeZalandoV1alpha1 + Fake *FakeZalandoV1 ns string } -var fabriceventstreamsResource = schema.GroupVersionResource{Group: "zalando.org", Version: "v1alpha1", Resource: "fabriceventstreams"} +var fabriceventstreamsResource = schema.GroupVersionResource{Group: "zalando.org", Version: "v1", Resource: "fabriceventstreams"} -var fabriceventstreamsKind = schema.GroupVersionKind{Group: "zalando.org", Version: "v1alpha1", Kind: "FabricEventStream"} +var fabriceventstreamsKind = schema.GroupVersionKind{Group: "zalando.org", Version: "v1", Kind: "FabricEventStream"} // Get takes name of the fabricEventStream, and returns the corresponding fabricEventStream object, and an error if there is any. -func (c *FakeFabricEventStreams) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.FabricEventStream, err error) { +func (c *FakeFabricEventStreams) Get(ctx context.Context, name string, options v1.GetOptions) (result *zalandoorgv1.FabricEventStream, err error) { obj, err := c.Fake. - Invokes(testing.NewGetAction(fabriceventstreamsResource, c.ns, name), &v1alpha1.FabricEventStream{}) + Invokes(testing.NewGetAction(fabriceventstreamsResource, c.ns, name), &zalandoorgv1.FabricEventStream{}) if obj == nil { return nil, err } - return obj.(*v1alpha1.FabricEventStream), err + return obj.(*zalandoorgv1.FabricEventStream), err } // List takes label and field selectors, and returns the list of FabricEventStreams that match those selectors. -func (c *FakeFabricEventStreams) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.FabricEventStreamList, err error) { +func (c *FakeFabricEventStreams) List(ctx context.Context, opts v1.ListOptions) (result *zalandoorgv1.FabricEventStreamList, err error) { obj, err := c.Fake. - Invokes(testing.NewListAction(fabriceventstreamsResource, fabriceventstreamsKind, c.ns, opts), &v1alpha1.FabricEventStreamList{}) + Invokes(testing.NewListAction(fabriceventstreamsResource, fabriceventstreamsKind, c.ns, opts), &zalandoorgv1.FabricEventStreamList{}) if obj == nil { return nil, err @@ -70,8 +70,8 @@ func (c *FakeFabricEventStreams) List(ctx context.Context, opts v1.ListOptions) if label == nil { label = labels.Everything() } - list := &v1alpha1.FabricEventStreamList{ListMeta: obj.(*v1alpha1.FabricEventStreamList).ListMeta} - for _, item := range obj.(*v1alpha1.FabricEventStreamList).Items { + list := &zalandoorgv1.FabricEventStreamList{ListMeta: obj.(*zalandoorgv1.FabricEventStreamList).ListMeta} + for _, item := range obj.(*zalandoorgv1.FabricEventStreamList).Items { if label.Matches(labels.Set(item.Labels)) { list.Items = append(list.Items, item) } @@ -87,31 +87,31 @@ func (c *FakeFabricEventStreams) Watch(ctx context.Context, opts v1.ListOptions) } // Create takes the representation of a fabricEventStream and creates it. Returns the server's representation of the fabricEventStream, and an error, if there is any. -func (c *FakeFabricEventStreams) Create(ctx context.Context, fabricEventStream *v1alpha1.FabricEventStream, opts v1.CreateOptions) (result *v1alpha1.FabricEventStream, err error) { +func (c *FakeFabricEventStreams) Create(ctx context.Context, fabricEventStream *zalandoorgv1.FabricEventStream, opts v1.CreateOptions) (result *zalandoorgv1.FabricEventStream, err error) { obj, err := c.Fake. - Invokes(testing.NewCreateAction(fabriceventstreamsResource, c.ns, fabricEventStream), &v1alpha1.FabricEventStream{}) + Invokes(testing.NewCreateAction(fabriceventstreamsResource, c.ns, fabricEventStream), &zalandoorgv1.FabricEventStream{}) if obj == nil { return nil, err } - return obj.(*v1alpha1.FabricEventStream), err + return obj.(*zalandoorgv1.FabricEventStream), err } // Update takes the representation of a fabricEventStream and updates it. Returns the server's representation of the fabricEventStream, and an error, if there is any. -func (c *FakeFabricEventStreams) Update(ctx context.Context, fabricEventStream *v1alpha1.FabricEventStream, opts v1.UpdateOptions) (result *v1alpha1.FabricEventStream, err error) { +func (c *FakeFabricEventStreams) Update(ctx context.Context, fabricEventStream *zalandoorgv1.FabricEventStream, opts v1.UpdateOptions) (result *zalandoorgv1.FabricEventStream, err error) { obj, err := c.Fake. - Invokes(testing.NewUpdateAction(fabriceventstreamsResource, c.ns, fabricEventStream), &v1alpha1.FabricEventStream{}) + Invokes(testing.NewUpdateAction(fabriceventstreamsResource, c.ns, fabricEventStream), &zalandoorgv1.FabricEventStream{}) if obj == nil { return nil, err } - return obj.(*v1alpha1.FabricEventStream), err + return obj.(*zalandoorgv1.FabricEventStream), err } // Delete takes name of the fabricEventStream and deletes it. Returns an error if one occurs. func (c *FakeFabricEventStreams) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { _, err := c.Fake. - Invokes(testing.NewDeleteAction(fabriceventstreamsResource, c.ns, name), &v1alpha1.FabricEventStream{}) + Invokes(testing.NewDeleteAction(fabriceventstreamsResource, c.ns, name), &zalandoorgv1.FabricEventStream{}) return err } @@ -120,17 +120,17 @@ func (c *FakeFabricEventStreams) Delete(ctx context.Context, name string, opts v func (c *FakeFabricEventStreams) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { action := testing.NewDeleteCollectionAction(fabriceventstreamsResource, c.ns, listOpts) - _, err := c.Fake.Invokes(action, &v1alpha1.FabricEventStreamList{}) + _, err := c.Fake.Invokes(action, &zalandoorgv1.FabricEventStreamList{}) return err } // Patch applies the patch and returns the patched fabricEventStream. -func (c *FakeFabricEventStreams) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.FabricEventStream, err error) { +func (c *FakeFabricEventStreams) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *zalandoorgv1.FabricEventStream, err error) { obj, err := c.Fake. - Invokes(testing.NewPatchSubresourceAction(fabriceventstreamsResource, c.ns, name, pt, data, subresources...), &v1alpha1.FabricEventStream{}) + Invokes(testing.NewPatchSubresourceAction(fabriceventstreamsResource, c.ns, name, pt, data, subresources...), &zalandoorgv1.FabricEventStream{}) if obj == nil { return nil, err } - return obj.(*v1alpha1.FabricEventStream), err + return obj.(*zalandoorgv1.FabricEventStream), err } diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fake/fake_zalando.org_client.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_zalando.org_client.go similarity index 80% rename from pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fake/fake_zalando.org_client.go rename to pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_zalando.org_client.go index f8583cae9..17786c9a7 100644 --- a/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/fake/fake_zalando.org_client.go +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/fake/fake_zalando.org_client.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -25,22 +25,22 @@ SOFTWARE. package fake import ( - v1alpha1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1" + v1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1" rest "k8s.io/client-go/rest" testing "k8s.io/client-go/testing" ) -type FakeZalandoV1alpha1 struct { +type FakeZalandoV1 struct { *testing.Fake } -func (c *FakeZalandoV1alpha1) FabricEventStreams(namespace string) v1alpha1.FabricEventStreamInterface { +func (c *FakeZalandoV1) FabricEventStreams(namespace string) v1.FabricEventStreamInterface { return &FakeFabricEventStreams{c, namespace} } // RESTClient returns a RESTClient that is used to communicate // with API server by this client implementation. -func (c *FakeZalandoV1alpha1) RESTClient() rest.Interface { +func (c *FakeZalandoV1) RESTClient() rest.Interface { var ret *rest.RESTClient return ret } diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/generated_expansion.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/generated_expansion.go similarity index 95% rename from pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/generated_expansion.go rename to pkg/generated/clientset/versioned/typed/zalando.org/v1/generated_expansion.go index 89c4f1d5d..6a776af84 100644 --- a/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/generated_expansion.go +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/generated_expansion.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -22,6 +22,6 @@ SOFTWARE. // Code generated by client-gen. DO NOT EDIT. -package v1alpha1 +package v1 type FabricEventStreamExpansion interface{} diff --git a/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/zalando.org_client.go b/pkg/generated/clientset/versioned/typed/zalando.org/v1/zalando.org_client.go similarity index 67% rename from pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/zalando.org_client.go rename to pkg/generated/clientset/versioned/typed/zalando.org/v1/zalando.org_client.go index 56436b2f4..937fd6b7e 100644 --- a/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1/zalando.org_client.go +++ b/pkg/generated/clientset/versioned/typed/zalando.org/v1/zalando.org_client.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -22,30 +22,30 @@ SOFTWARE. // Code generated by client-gen. DO NOT EDIT. -package v1alpha1 +package v1 import ( - v1alpha1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1alpha1" + v1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/scheme" rest "k8s.io/client-go/rest" ) -type ZalandoV1alpha1Interface interface { +type ZalandoV1Interface interface { RESTClient() rest.Interface FabricEventStreamsGetter } -// ZalandoV1alpha1Client is used to interact with features provided by the zalando.org group. -type ZalandoV1alpha1Client struct { +// ZalandoV1Client is used to interact with features provided by the zalando.org group. +type ZalandoV1Client struct { restClient rest.Interface } -func (c *ZalandoV1alpha1Client) FabricEventStreams(namespace string) FabricEventStreamInterface { +func (c *ZalandoV1Client) FabricEventStreams(namespace string) FabricEventStreamInterface { return newFabricEventStreams(c, namespace) } -// NewForConfig creates a new ZalandoV1alpha1Client for the given config. -func NewForConfig(c *rest.Config) (*ZalandoV1alpha1Client, error) { +// NewForConfig creates a new ZalandoV1Client for the given config. +func NewForConfig(c *rest.Config) (*ZalandoV1Client, error) { config := *c if err := setConfigDefaults(&config); err != nil { return nil, err @@ -54,12 +54,12 @@ func NewForConfig(c *rest.Config) (*ZalandoV1alpha1Client, error) { if err != nil { return nil, err } - return &ZalandoV1alpha1Client{client}, nil + return &ZalandoV1Client{client}, nil } -// NewForConfigOrDie creates a new ZalandoV1alpha1Client for the given config and +// NewForConfigOrDie creates a new ZalandoV1Client for the given config and // panics if there is an error in the config. -func NewForConfigOrDie(c *rest.Config) *ZalandoV1alpha1Client { +func NewForConfigOrDie(c *rest.Config) *ZalandoV1Client { client, err := NewForConfig(c) if err != nil { panic(err) @@ -67,13 +67,13 @@ func NewForConfigOrDie(c *rest.Config) *ZalandoV1alpha1Client { return client } -// New creates a new ZalandoV1alpha1Client for the given RESTClient. -func New(c rest.Interface) *ZalandoV1alpha1Client { - return &ZalandoV1alpha1Client{c} +// New creates a new ZalandoV1Client for the given RESTClient. +func New(c rest.Interface) *ZalandoV1Client { + return &ZalandoV1Client{c} } func setConfigDefaults(config *rest.Config) error { - gv := v1alpha1.SchemeGroupVersion + gv := v1.SchemeGroupVersion config.GroupVersion = &gv config.APIPath = "/apis" config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() @@ -87,7 +87,7 @@ func setConfigDefaults(config *rest.Config) error { // RESTClient returns a RESTClient that is used to communicate // with API server by this client implementation. -func (c *ZalandoV1alpha1Client) RESTClient() rest.Interface { +func (c *ZalandoV1Client) RESTClient() rest.Interface { if c == nil { return nil } diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/interface.go b/pkg/generated/informers/externalversions/acid.zalan.do/interface.go index 6f77564fa..b2b85d87d 100644 --- a/pkg/generated/informers/externalversions/acid.zalan.do/interface.go +++ b/pkg/generated/informers/externalversions/acid.zalan.do/interface.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go b/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go index 5c05e6d68..347c53f99 100644 --- a/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go +++ b/pkg/generated/informers/externalversions/acid.zalan.do/v1/interface.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go index 1453af276..61d88fb22 100644 --- a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go +++ b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresql.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresteam.go b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresteam.go index a19e4726f..eb15e4fad 100644 --- a/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresteam.go +++ b/pkg/generated/informers/externalversions/acid.zalan.do/v1/postgresteam.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/informers/externalversions/factory.go b/pkg/generated/informers/externalversions/factory.go index b3938ec74..0842aa5ae 100644 --- a/pkg/generated/informers/externalversions/factory.go +++ b/pkg/generated/informers/externalversions/factory.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/informers/externalversions/generic.go b/pkg/generated/informers/externalversions/generic.go index a58a2ec24..afa4c6ac2 100644 --- a/pkg/generated/informers/externalversions/generic.go +++ b/pkg/generated/informers/externalversions/generic.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -28,7 +28,7 @@ import ( "fmt" v1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" - v1alpha1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1alpha1" + zalandoorgv1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" schema "k8s.io/apimachinery/pkg/runtime/schema" cache "k8s.io/client-go/tools/cache" ) @@ -65,9 +65,9 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource case v1.SchemeGroupVersion.WithResource("postgresqls"): return &genericInformer{resource: resource.GroupResource(), informer: f.Acid().V1().Postgresqls().Informer()}, nil - // Group=zalando.org, Version=v1alpha1 - case v1alpha1.SchemeGroupVersion.WithResource("fabriceventstreams"): - return &genericInformer{resource: resource.GroupResource(), informer: f.Zalando().V1alpha1().FabricEventStreams().Informer()}, nil + // Group=zalando.org, Version=v1 + case zalandoorgv1.SchemeGroupVersion.WithResource("fabriceventstreams"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Zalando().V1().FabricEventStreams().Informer()}, nil } diff --git a/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go b/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go index 6d1b334bf..1046f8f48 100644 --- a/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go +++ b/pkg/generated/informers/externalversions/internalinterfaces/factory_interfaces.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/informers/externalversions/zalando.org/interface.go b/pkg/generated/informers/externalversions/zalando.org/interface.go index c8cb52a09..afece8ac3 100644 --- a/pkg/generated/informers/externalversions/zalando.org/interface.go +++ b/pkg/generated/informers/externalversions/zalando.org/interface.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -26,13 +26,13 @@ package zalando import ( internalinterfaces "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/internalinterfaces" - v1alpha1 "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/zalando.org/v1alpha1" + v1 "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/zalando.org/v1" ) // Interface provides access to each of this group's versions. type Interface interface { - // V1alpha1 provides access to shared informers for resources in V1alpha1. - V1alpha1() v1alpha1.Interface + // V1 provides access to shared informers for resources in V1. + V1() v1.Interface } type group struct { @@ -46,7 +46,7 @@ func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakList return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} } -// V1alpha1 returns a new v1alpha1.Interface. -func (g *group) V1alpha1() v1alpha1.Interface { - return v1alpha1.New(g.factory, g.namespace, g.tweakListOptions) +// V1 returns a new v1.Interface. +func (g *group) V1() v1.Interface { + return v1.New(g.factory, g.namespace, g.tweakListOptions) } diff --git a/pkg/generated/informers/externalversions/zalando.org/v1alpha1/fabriceventstream.go b/pkg/generated/informers/externalversions/zalando.org/v1/fabriceventstream.go similarity index 78% rename from pkg/generated/informers/externalversions/zalando.org/v1alpha1/fabriceventstream.go rename to pkg/generated/informers/externalversions/zalando.org/v1/fabriceventstream.go index 7d29d716d..13733e1d1 100644 --- a/pkg/generated/informers/externalversions/zalando.org/v1alpha1/fabriceventstream.go +++ b/pkg/generated/informers/externalversions/zalando.org/v1/fabriceventstream.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -22,17 +22,17 @@ SOFTWARE. // Code generated by informer-gen. DO NOT EDIT. -package v1alpha1 +package v1 import ( "context" time "time" - zalandoorgv1alpha1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1alpha1" + zalandoorgv1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" versioned "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned" internalinterfaces "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/internalinterfaces" - v1alpha1 "github.com/zalando/postgres-operator/pkg/generated/listers/zalando.org/v1alpha1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "github.com/zalando/postgres-operator/pkg/generated/listers/zalando.org/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" watch "k8s.io/apimachinery/pkg/watch" cache "k8s.io/client-go/tools/cache" @@ -42,7 +42,7 @@ import ( // FabricEventStreams. type FabricEventStreamInformer interface { Informer() cache.SharedIndexInformer - Lister() v1alpha1.FabricEventStreamLister + Lister() v1.FabricEventStreamLister } type fabricEventStreamInformer struct { @@ -64,20 +64,20 @@ func NewFabricEventStreamInformer(client versioned.Interface, namespace string, func NewFilteredFabricEventStreamInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { return cache.NewSharedIndexInformer( &cache.ListWatch{ - ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { if tweakListOptions != nil { tweakListOptions(&options) } - return client.ZalandoV1alpha1().FabricEventStreams(namespace).List(context.TODO(), options) + return client.ZalandoV1().FabricEventStreams(namespace).List(context.TODO(), options) }, - WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { if tweakListOptions != nil { tweakListOptions(&options) } - return client.ZalandoV1alpha1().FabricEventStreams(namespace).Watch(context.TODO(), options) + return client.ZalandoV1().FabricEventStreams(namespace).Watch(context.TODO(), options) }, }, - &zalandoorgv1alpha1.FabricEventStream{}, + &zalandoorgv1.FabricEventStream{}, resyncPeriod, indexers, ) @@ -88,9 +88,9 @@ func (f *fabricEventStreamInformer) defaultInformer(client versioned.Interface, } func (f *fabricEventStreamInformer) Informer() cache.SharedIndexInformer { - return f.factory.InformerFor(&zalandoorgv1alpha1.FabricEventStream{}, f.defaultInformer) + return f.factory.InformerFor(&zalandoorgv1.FabricEventStream{}, f.defaultInformer) } -func (f *fabricEventStreamInformer) Lister() v1alpha1.FabricEventStreamLister { - return v1alpha1.NewFabricEventStreamLister(f.Informer().GetIndexer()) +func (f *fabricEventStreamInformer) Lister() v1.FabricEventStreamLister { + return v1.NewFabricEventStreamLister(f.Informer().GetIndexer()) } diff --git a/pkg/generated/informers/externalversions/zalando.org/v1alpha1/interface.go b/pkg/generated/informers/externalversions/zalando.org/v1/interface.go similarity index 97% rename from pkg/generated/informers/externalversions/zalando.org/v1alpha1/interface.go rename to pkg/generated/informers/externalversions/zalando.org/v1/interface.go index 7815569a2..329f15503 100644 --- a/pkg/generated/informers/externalversions/zalando.org/v1alpha1/interface.go +++ b/pkg/generated/informers/externalversions/zalando.org/v1/interface.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -22,7 +22,7 @@ SOFTWARE. // Code generated by informer-gen. DO NOT EDIT. -package v1alpha1 +package v1 import ( internalinterfaces "github.com/zalando/postgres-operator/pkg/generated/informers/externalversions/internalinterfaces" diff --git a/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go b/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go index cc3e578b2..3189f5948 100644 --- a/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go +++ b/pkg/generated/listers/acid.zalan.do/v1/expansion_generated.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/listers/acid.zalan.do/v1/postgresql.go b/pkg/generated/listers/acid.zalan.do/v1/postgresql.go index d2258bd01..650f8d579 100644 --- a/pkg/generated/listers/acid.zalan.do/v1/postgresql.go +++ b/pkg/generated/listers/acid.zalan.do/v1/postgresql.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/listers/acid.zalan.do/v1/postgresteam.go b/pkg/generated/listers/acid.zalan.do/v1/postgresteam.go index 38073e92d..c63b8da58 100644 --- a/pkg/generated/listers/acid.zalan.do/v1/postgresteam.go +++ b/pkg/generated/listers/acid.zalan.do/v1/postgresteam.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pkg/generated/listers/zalando.org/v1alpha1/expansion_generated.go b/pkg/generated/listers/zalando.org/v1/expansion_generated.go similarity index 96% rename from pkg/generated/listers/zalando.org/v1alpha1/expansion_generated.go rename to pkg/generated/listers/zalando.org/v1/expansion_generated.go index 16438da8b..5a90e7828 100644 --- a/pkg/generated/listers/zalando.org/v1alpha1/expansion_generated.go +++ b/pkg/generated/listers/zalando.org/v1/expansion_generated.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -22,7 +22,7 @@ SOFTWARE. // Code generated by lister-gen. DO NOT EDIT. -package v1alpha1 +package v1 // FabricEventStreamListerExpansion allows custom methods to be added to // FabricEventStreamLister. diff --git a/pkg/generated/listers/zalando.org/v1alpha1/fabriceventstream.go b/pkg/generated/listers/zalando.org/v1/fabriceventstream.go similarity index 82% rename from pkg/generated/listers/zalando.org/v1alpha1/fabriceventstream.go rename to pkg/generated/listers/zalando.org/v1/fabriceventstream.go index 244d11bbf..b4c895c74 100644 --- a/pkg/generated/listers/zalando.org/v1alpha1/fabriceventstream.go +++ b/pkg/generated/listers/zalando.org/v1/fabriceventstream.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Compose, Zalando SE +Copyright 2022 Compose, Zalando SE Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -22,10 +22,10 @@ SOFTWARE. // Code generated by lister-gen. DO NOT EDIT. -package v1alpha1 +package v1 import ( - v1alpha1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1alpha1" + v1 "github.com/zalando/postgres-operator/pkg/apis/zalando.org/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" @@ -36,7 +36,7 @@ import ( type FabricEventStreamLister interface { // List lists all FabricEventStreams in the indexer. // Objects returned here must be treated as read-only. - List(selector labels.Selector) (ret []*v1alpha1.FabricEventStream, err error) + List(selector labels.Selector) (ret []*v1.FabricEventStream, err error) // FabricEventStreams returns an object that can list and get FabricEventStreams. FabricEventStreams(namespace string) FabricEventStreamNamespaceLister FabricEventStreamListerExpansion @@ -53,9 +53,9 @@ func NewFabricEventStreamLister(indexer cache.Indexer) FabricEventStreamLister { } // List lists all FabricEventStreams in the indexer. -func (s *fabricEventStreamLister) List(selector labels.Selector) (ret []*v1alpha1.FabricEventStream, err error) { +func (s *fabricEventStreamLister) List(selector labels.Selector) (ret []*v1.FabricEventStream, err error) { err = cache.ListAll(s.indexer, selector, func(m interface{}) { - ret = append(ret, m.(*v1alpha1.FabricEventStream)) + ret = append(ret, m.(*v1.FabricEventStream)) }) return ret, err } @@ -70,10 +70,10 @@ func (s *fabricEventStreamLister) FabricEventStreams(namespace string) FabricEve type FabricEventStreamNamespaceLister interface { // List lists all FabricEventStreams in the indexer for a given namespace. // Objects returned here must be treated as read-only. - List(selector labels.Selector) (ret []*v1alpha1.FabricEventStream, err error) + List(selector labels.Selector) (ret []*v1.FabricEventStream, err error) // Get retrieves the FabricEventStream from the indexer for a given namespace and name. // Objects returned here must be treated as read-only. - Get(name string) (*v1alpha1.FabricEventStream, error) + Get(name string) (*v1.FabricEventStream, error) FabricEventStreamNamespaceListerExpansion } @@ -85,21 +85,21 @@ type fabricEventStreamNamespaceLister struct { } // List lists all FabricEventStreams in the indexer for a given namespace. -func (s fabricEventStreamNamespaceLister) List(selector labels.Selector) (ret []*v1alpha1.FabricEventStream, err error) { +func (s fabricEventStreamNamespaceLister) List(selector labels.Selector) (ret []*v1.FabricEventStream, err error) { err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { - ret = append(ret, m.(*v1alpha1.FabricEventStream)) + ret = append(ret, m.(*v1.FabricEventStream)) }) return ret, err } // Get retrieves the FabricEventStream from the indexer for a given namespace and name. -func (s fabricEventStreamNamespaceLister) Get(name string) (*v1alpha1.FabricEventStream, error) { +func (s fabricEventStreamNamespaceLister) Get(name string) (*v1.FabricEventStream, error) { obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) if err != nil { return nil, err } if !exists { - return nil, errors.NewNotFound(v1alpha1.Resource("fabriceventstream"), name) + return nil, errors.NewNotFound(v1.Resource("fabriceventstream"), name) } - return obj.(*v1alpha1.FabricEventStream), nil + return obj.(*v1.FabricEventStream), nil } diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index 67192dd21..0897777ee 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -14,7 +14,7 @@ import ( apiacidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" zalandoclient "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned" acidv1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/acid.zalan.do/v1" - zalandov1alpha1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1alpha1" + zalandov1 "github.com/zalando/postgres-operator/pkg/generated/clientset/versioned/typed/zalando.org/v1" "github.com/zalando/postgres-operator/pkg/spec" apiappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -59,11 +59,11 @@ type KubernetesClient struct { acidv1.OperatorConfigurationsGetter acidv1.PostgresTeamsGetter acidv1.PostgresqlsGetter - zalandov1alpha1.FabricEventStreamsGetter + zalandov1.FabricEventStreamsGetter - RESTClient rest.Interface - AcidV1ClientSet *zalandoclient.Clientset - ZalandoV1Alpha1ClientSet *zalandoclient.Clientset + RESTClient rest.Interface + AcidV1ClientSet *zalandoclient.Clientset + Zalandov1ClientSet *zalandoclient.Clientset } type mockSecret struct { @@ -165,7 +165,7 @@ func NewFromConfig(cfg *rest.Config) (KubernetesClient, error) { if err != nil { return kubeClient, fmt.Errorf("could not create acid.zalan.do clientset: %v", err) } - kubeClient.ZalandoV1Alpha1ClientSet = zalandoclient.NewForConfigOrDie(cfg) + kubeClient.Zalandov1ClientSet = zalandoclient.NewForConfigOrDie(cfg) if err != nil { return kubeClient, fmt.Errorf("could not create zalando.org clientset: %v", err) } @@ -173,7 +173,7 @@ func NewFromConfig(cfg *rest.Config) (KubernetesClient, error) { kubeClient.OperatorConfigurationsGetter = kubeClient.AcidV1ClientSet.AcidV1() kubeClient.PostgresTeamsGetter = kubeClient.AcidV1ClientSet.AcidV1() kubeClient.PostgresqlsGetter = kubeClient.AcidV1ClientSet.AcidV1() - kubeClient.FabricEventStreamsGetter = kubeClient.ZalandoV1Alpha1ClientSet.ZalandoV1alpha1() + kubeClient.FabricEventStreamsGetter = kubeClient.Zalandov1ClientSet.ZalandoV1() return kubeClient, nil } diff --git a/pkg/util/patroni/patroni_test.go b/pkg/util/patroni/patroni_test.go index 5a6b2657c..aa9d1e732 100644 --- a/pkg/util/patroni/patroni_test.go +++ b/pkg/util/patroni/patroni_test.go @@ -136,7 +136,7 @@ func TestGetConfig(t *testing.T) { Slots: map[string]map[string]string{ "cdc": { "database": "foo", - "plugin": "wal2json", + "plugin": "pgoutput", "type": "logical", }, }, @@ -169,7 +169,7 @@ func TestGetConfig(t *testing.T) { "wal_log_hints": "on", } - configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 100, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "hot_standby", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "wal2json", "type": "logical"}}, "ttl": 30}` + configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 100, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "hot_standby", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "pgoutput", "type": "logical"}}, "ttl": 30}` r := ioutil.NopCloser(bytes.NewReader([]byte(configJson))) response := http.Response{ @@ -204,7 +204,7 @@ func TestSetPostgresParameters(t *testing.T) { "wal_level": "logical", } - configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 50, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "logical", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "wal2json", "type": "logical"}}, "ttl": 30}` + configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 50, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "logical", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "pgoutput", "type": "logical"}}, "ttl": 30}` r := ioutil.NopCloser(bytes.NewReader([]byte(configJson))) response := http.Response{ From 179fcb1126f851721bdd19f8230083f7e1b2ac5c Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 25 Jan 2022 17:18:49 +0100 Subject: [PATCH 35/44] switch to pgoutput plugin and let operator create publications --- docs/reference/cluster_manifest.md | 12 +++-- pkg/apis/zalando.org/v1/fabriceventstream.go | 7 +-- pkg/cluster/database.go | 22 ++++++++ pkg/cluster/streams.go | 53 ++++++++++++++++---- pkg/cluster/streams_test.go | 6 ++- 5 files changed, 79 insertions(+), 21 deletions(-) diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 07b03ec9f..398480f0a 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -546,11 +546,13 @@ have the following properties: and `payloadColumn`). The CDC operator is following the [outbox pattern](https://debezium.io/blog/2019/02/19/reliable-microservices-data-exchange-with-the-outbox-pattern/). The application is responsible for putting events into a (JSON/B or VARCHAR) payload column of the outbox table in the structure of the specified target - event type. The the CDC operator will consume them shortly after the - transaction is committed. The `idColumn` will be used in telemetry for the - CDC operator. The names for `idColumn` and `payloadColumn` can be configured. - Defaults are `id` and `payload`. The target `eventType` has to be defined. - Required. + event type. The operator will create a [PUBLICATION](https://www.postgresql.org/docs/14/logical-replication-publication.html) + in Postgres for all tables specified for one `database` and `applicationId`. + The CDC operator will consume from it shortly after transactions are + committed to the outbox table. The `idColumn` will be used in telemetry for + the CDC operator. The names for `idColumn` and `payloadColumn` can be + configured. Defaults are `id` and `payload`. The target `eventType` has to + be defined. Required. * **filter** Streamed events can be filtered by a jsonpath expression for each table. diff --git a/pkg/apis/zalando.org/v1/fabriceventstream.go b/pkg/apis/zalando.org/v1/fabriceventstream.go index 7990d7700..81706802d 100644 --- a/pkg/apis/zalando.org/v1/fabriceventstream.go +++ b/pkg/apis/zalando.org/v1/fabriceventstream.go @@ -68,9 +68,10 @@ type EventStreamTable struct { // Connection to be used for allowing the FES operator to connect to a database type Connection struct { - Url string `json:"jdbcUrl"` - SlotName string `json:"slotName"` - DBAuth DBAuth `json:"databaseAuthentication"` + Url string `json:"jdbcUrl"` + SlotName string `json:"slotName"` + PublicationName string `json:"publicationName"` + DBAuth DBAuth `json:"databaseAuthentication"` } // DBAuth specifies the credentials to be used for connecting with the database diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index aa3a5e3be..ff08c6c24 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -39,6 +39,8 @@ const ( createExtensionSQL = `CREATE EXTENSION IF NOT EXISTS "%s" SCHEMA "%s"` alterExtensionSQL = `ALTER EXTENSION "%s" SET SCHEMA "%s"` + createPublicationSQL = `CREATE PUBLICATION "%s" FOR TABLE %s WITH (publish = 'insert, update');` + globalDefaultPrivilegesSQL = `SET ROLE TO "%s"; ALTER DEFAULT PRIVILEGES GRANT USAGE ON SCHEMAS TO "%s","%s"; ALTER DEFAULT PRIVILEGES GRANT SELECT ON TABLES TO "%s"; @@ -610,3 +612,23 @@ func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string) error { return nil } + +// getExtension returns the list of current database extensions +// The caller is responsible for opening and closing the database connection +func (c *Cluster) createPublication(dbName, publication, tables string) (err error) { + + if err := c.initDbConnWithName(dbName); err != nil { + return fmt.Errorf("could not init connection to database %q", dbName) + } + defer func() { + if err = c.closeDbConn(); err != nil { + err = fmt.Errorf("could not close connection to database %q: %v", dbName, err) + } + }() + + if _, err := c.pgDb.Exec(fmt.Sprintf(createPublicationSQL, publication, tables)); err != nil { + return fmt.Errorf("could not create publication %s: %v", publication, err) + } + + return err +} diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index 7053aeb20..dd126df0f 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -76,6 +76,7 @@ func gatherApplicationIds(streams []acidv1.Stream) []string { func (c *Cluster) syncPostgresConfig() error { slots := make(map[string]map[string]string) + publications := make(map[string]map[string]acidv1.StreamTable) desiredPatroniConfig := c.Spec.Patroni if len(desiredPatroniConfig.Slots) > 0 { slots = desiredPatroniConfig.Slots @@ -87,9 +88,18 @@ func (c *Cluster) syncPostgresConfig() error { "plugin": "pgoutput", "type": "logical", } - slotName := constants.EventStreamSourceSlotPrefix + "_" + stream.Database + "_" + stream.ApplicationId + slotName := getSlotName(stream.Database, stream.ApplicationId) if _, exists := slots[slotName]; !exists { slots[slotName] = slot + publications[slotName] = stream.Tables + } else { + streamTables := publications[slotName] + for tableName, table := range stream.Tables { + if _, exists := streamTables[tableName]; !exists { + streamTables[tableName] = table + } + } + publications[slotName] = streamTables } } @@ -108,7 +118,7 @@ func (c *Cluster) syncPostgresConfig() error { pods, err := c.listPods() if err != nil || len(pods) == 0 { - c.logger.Warnf("could not list pods of the statefulset: %v", err) + c.logger.Warningf("could not list pods of the statefulset: %v", err) } for i, pod := range pods { podName := util.NameFromMeta(pods[i].ObjectMeta) @@ -125,6 +135,22 @@ func (c *Cluster) syncPostgresConfig() error { } } + // next create publications to each created slot + for publication, tables := range publications { + dbName := slots[publication]["database"] + tableNames := make([]string, len(tables)) + i := 0 + for t := range tables { + tableNames[i] = fmt.Sprintf("%q", t) + i++ + } + tableList := strings.Join(tableNames, ", ") + c.logger.Debugf("creating publication %q in database %q for tables %s", publication, dbName, tableList) + if err := c.createPublication(dbName, publication, tableList); err != nil { + c.logger.Warningf("%v", err) + } + } + return nil } @@ -214,10 +240,15 @@ func getOutboxTable(tableName, idColumn string) zalandov1.EventStreamTable { } } +func getSlotName(dbName, appId string) string { + return constants.EventStreamSourceSlotPrefix + "_" + dbName + "_" + strings.Replace(appId, "-", "_", -1) +} + func (c *Cluster) getStreamConnection(database, user, appId string) zalandov1.Connection { return zalandov1.Connection{ - Url: fmt.Sprintf("jdbc:postgresql://%s.%s/%s?user=%s&ssl=true&sslmode=require", c.Name, c.Namespace, database, user), - SlotName: constants.EventStreamSourceSlotPrefix + "_" + database + "_" + strings.Replace(appId, "-", "_", -1), + Url: fmt.Sprintf("jdbc:postgresql://%s.%s/%s?user=%s&ssl=true&sslmode=require", c.Name, c.Namespace, database, user), + SlotName: getSlotName(database, appId), + PublicationName: getSlotName(database, appId), DBAuth: zalandov1.DBAuth{ Type: constants.EventStreamSourceAuthType, Name: c.credentialSecretNameForCluster(user, c.Name), @@ -229,12 +260,19 @@ func (c *Cluster) getStreamConnection(database, user, appId string) zalandov1.Co func (c *Cluster) syncStreams() error { + c.setProcessName("syncing streams") + _, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), constants.EventStreamSourceCRDName, metav1.GetOptions{}) if k8sutil.ResourceNotFound(err) { c.logger.Debugf("event stream CRD not installed, skipping") return nil } + err = c.syncPostgresConfig() + if err != nil { + return fmt.Errorf("could not update Postgres config for event streaming: %v", err) + } + err = c.createOrUpdateStreams() if err != nil { return err @@ -245,13 +283,6 @@ func (c *Cluster) syncStreams() error { func (c *Cluster) createOrUpdateStreams() error { - c.setProcessName("syncing streams") - - err := c.syncPostgresConfig() - if err != nil { - return fmt.Errorf("could not update Postgres config for event streaming: %v", err) - } - appIds := gatherApplicationIds(c.Spec.Streams) for _, appId := range appIds { fesName := c.Name + "-" + appId diff --git a/pkg/cluster/streams_test.go b/pkg/cluster/streams_test.go index 27d3e91f3..f1c615e04 100644 --- a/pkg/cluster/streams_test.go +++ b/pkg/cluster/streams_test.go @@ -42,6 +42,7 @@ var ( dbName string = "foo" fesUser string = constants.EventStreamSourceSlotPrefix + constants.UserRoleNameSuffix fesName string = clusterName + "-" + appId + slotName string = fmt.Sprintf("%s_%s_%s", constants.EventStreamSourceSlotPrefix, dbName, strings.Replace(appId, "-", "_", -1)) pg = acidv1.Postgresql{ TypeMeta: metav1.TypeMeta{ @@ -118,8 +119,9 @@ var ( Type: constants.EventStreamSourceAuthType, UserKey: "username", }, - Url: fmt.Sprintf("jdbc:postgresql://%s.%s/foo?user=%s&ssl=true&sslmode=require", clusterName, namespace, fesUser), - SlotName: fmt.Sprintf("%s_%s_%s", constants.EventStreamSourceSlotPrefix, dbName, strings.Replace(appId, "-", "_", -1)), + Url: fmt.Sprintf("jdbc:postgresql://%s.%s/foo?user=%s&ssl=true&sslmode=require", clusterName, namespace, fesUser), + SlotName: slotName, + PublicationName: slotName, }, Schema: "data", EventStreamTable: v1.EventStreamTable{ From eb51428a4491e0f989583ec7c030bada3a233ad3 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 25 Jan 2022 18:57:55 +0100 Subject: [PATCH 36/44] quotes for schema and table --- pkg/cluster/streams.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index dd126df0f..ea8381eee 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -141,7 +141,8 @@ func (c *Cluster) syncPostgresConfig() error { tableNames := make([]string, len(tables)) i := 0 for t := range tables { - tableNames[i] = fmt.Sprintf("%q", t) + tableName, schemaName := getTableSchema(t) + tableNames[i] = fmt.Sprintf("%q.%q", schemaName, tableName) i++ } tableList := strings.Join(tableNames, ", ") From 31ec300f87bf6f571083c2da75da242be253bfe7 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 26 Jan 2022 11:38:19 +0100 Subject: [PATCH 37/44] alter publications if needed --- pkg/cluster/database.go | 86 +++++++++++++++++++++++++++++++---------- pkg/cluster/streams.go | 47 ++++++++++++++++++++-- 2 files changed, 109 insertions(+), 24 deletions(-) diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index ff08c6c24..bb640ade0 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -39,7 +39,12 @@ const ( createExtensionSQL = `CREATE EXTENSION IF NOT EXISTS "%s" SCHEMA "%s"` alterExtensionSQL = `ALTER EXTENSION "%s" SET SCHEMA "%s"` + getPublicationsSQL = `SELECT p.pubname, string_agg(pt.schemaname || . || pt.tablename, ', ' ORDER BY pt.schemaname, pt.tablename) + FROM pg_publication p + JOIN pg_publication_tables pt ON pt.pubname = p.pubname + GROUP BY p.pubname;` createPublicationSQL = `CREATE PUBLICATION "%s" FOR TABLE %s WITH (publish = 'insert, update');` + alterPublicationSQL = `ALTER PUBLICATION "%s" SET TABLE %s;` globalDefaultPrivilegesSQL = `SET ROLE TO "%s"; ALTER DEFAULT PRIVILEGES GRANT USAGE ON SCHEMAS TO "%s","%s"; @@ -508,6 +513,67 @@ func (c *Cluster) execCreateOrAlterExtension(extName, schemaName, statement, doi return nil } +// getPublications returns the list of current database publications with tables +// The caller is responsible for opening and closing the database connection +func (c *Cluster) getPublications() (publications map[string]string, err error) { + var ( + rows *sql.Rows + dbPublications map[string]string + ) + + if rows, err = c.pgDb.Query(getPublicationsSQL); err != nil { + return nil, fmt.Errorf("could not query database publications: %v", err) + } + + defer func() { + if err2 := rows.Close(); err2 != nil { + if err != nil { + err = fmt.Errorf("error when closing query cursor: %v, previous error: %v", err2, err) + } else { + err = fmt.Errorf("error when closing query cursor: %v", err2) + } + } + }() + + for rows.Next() { + var ( + dbPublication string + dbPublicationTables string + ) + + if err = rows.Scan(&dbPublication, &dbPublicationTables); err != nil { + return nil, fmt.Errorf("error when processing row: %v", err) + } + dbPublications[dbPublication] = dbPublicationTables + } + + return dbPublications, err +} + +// executeCreatePublication creates new publication for given tables +// The caller is responsible for opening and closing the database connection. +func (c *Cluster) executeCreatePublication(pubName, tableList string) error { + return c.execCreateOrAlterPublication(pubName, tableList, createPublicationSQL, + "creating publication", "create publication") +} + +// executeAlterExtension changes the table list of the given publication. +// The caller is responsible for opening and closing the database connection. +func (c *Cluster) executeAlterPublication(pubName, tableList string) error { + return c.execCreateOrAlterPublication(pubName, tableList, alterPublicationSQL, + "changing table list of publication", "alter publication tables") +} + +func (c *Cluster) execCreateOrAlterPublication(pubName, tableList, statement, doing, operation string) error { + + c.logger.Infof("%s %q table list %q", doing, pubName, tableList) + if _, err := c.pgDb.Exec(fmt.Sprintf(statement, pubName, tableList)); err != nil { + return fmt.Errorf("could not execute %s: %v", operation, err) + } + + return nil +} + // Creates a connection pool credentials lookup function in every database to // perform remote authentication. func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string) error { @@ -612,23 +678,3 @@ func (c *Cluster) installLookupFunction(poolerSchema, poolerUser string) error { return nil } - -// getExtension returns the list of current database extensions -// The caller is responsible for opening and closing the database connection -func (c *Cluster) createPublication(dbName, publication, tables string) (err error) { - - if err := c.initDbConnWithName(dbName); err != nil { - return fmt.Errorf("could not init connection to database %q", dbName) - } - defer func() { - if err = c.closeDbConn(); err != nil { - err = fmt.Errorf("could not close connection to database %q: %v", dbName, err) - } - }() - - if _, err := c.pgDb.Exec(fmt.Sprintf(createPublicationSQL, publication, tables)); err != nil { - return fmt.Errorf("could not create publication %s: %v", publication, err) - } - - return err -} diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index ea8381eee..ebd110c32 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "reflect" + "sort" "strings" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" @@ -77,6 +78,15 @@ func (c *Cluster) syncPostgresConfig() error { slots := make(map[string]map[string]string) publications := make(map[string]map[string]acidv1.StreamTable) + createPublications := make(map[string]string) + alterPublications := make(map[string]string) + + defer func() { + if err := c.closeDbConn(); err != nil { + c.logger.Errorf("could not close database connection: %v", err) + } + }() + desiredPatroniConfig := c.Spec.Patroni if len(desiredPatroniConfig.Slots) > 0 { slots = desiredPatroniConfig.Slots @@ -135,9 +145,19 @@ func (c *Cluster) syncPostgresConfig() error { } } - // next create publications to each created slot + // next, create publications to each created slot for publication, tables := range publications { + // but first check for existing publications dbName := slots[publication]["database"] + if err := c.initDbConnWithName(dbName); err != nil { + return fmt.Errorf("could not init database connection") + } + + currentPublications, err := c.getPublications() + if err != nil { + return fmt.Errorf("could not get current publications: %v", err) + } + tableNames := make([]string, len(tables)) i := 0 for t := range tables { @@ -145,10 +165,29 @@ func (c *Cluster) syncPostgresConfig() error { tableNames[i] = fmt.Sprintf("%q.%q", schemaName, tableName) i++ } + sort.Strings(tableNames) tableList := strings.Join(tableNames, ", ") - c.logger.Debugf("creating publication %q in database %q for tables %s", publication, dbName, tableList) - if err := c.createPublication(dbName, publication, tableList); err != nil { - c.logger.Warningf("%v", err) + + currentTables, exists := currentPublications[publication] + if !exists { + createPublications[publication] = tableList + } else if currentTables != tableList { + alterPublications[publication] = tableList + } + + if len(createPublications)+len(alterPublications) == 0 { + return nil + } + + for publicationName, tables := range createPublications { + if err = c.executeCreatePublication(publicationName, tables); err != nil { + c.logger.Warningf("%v", err) + } + } + for publicationName, tables := range alterPublications { + if err = c.executeAlterPublication(publicationName, tables); err != nil { + c.logger.Warningf("%v", err) + } } } From 2630d8eff3cda3772a86fbe8555f3cb9405715d4 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 26 Jan 2022 14:04:17 +0100 Subject: [PATCH 38/44] minor fixes --- manifests/operator-service-account-rbac.yaml | 26 ++++++++++---------- pkg/cluster/database.go | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index 882ad7281..c10dc5fd7 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -36,19 +36,19 @@ rules: - list - watch # all verbs allowed for event streams (Zalando-internal feature) -#- apiGroups: -# - zalando.org -# resources: -# - fabriceventstreams -# verbs: -# - create -# - delete -# - deletecollection -# - get -# - list -# - patch -# - update -# - watch +# - apiGroups: +# - zalando.org +# resources: +# - fabriceventstreams +# verbs: +# - create +# - delete +# - deletecollection +# - get +# - list +# - patch +# - update +# - watch # to create or get/update CRDs when starting up - apiGroups: - apiextensions.k8s.io diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index bb640ade0..6121fac70 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -39,7 +39,7 @@ const ( createExtensionSQL = `CREATE EXTENSION IF NOT EXISTS "%s" SCHEMA "%s"` alterExtensionSQL = `ALTER EXTENSION "%s" SET SCHEMA "%s"` - getPublicationsSQL = `SELECT p.pubname, string_agg(pt.schemaname || . || pt.tablename, ', ' ORDER BY pt.schemaname, pt.tablename) + getPublicationsSQL = `SELECT p.pubname, string_agg(pt.schemaname || '.' || pt.tablename, ', ' ORDER BY pt.schemaname, pt.tablename) FROM pg_publication p JOIN pg_publication_tables pt ON pt.pubname = p.pubname GROUP BY p.pubname;` From 3f0ed2682866e65ed15637795d50099d29451e4b Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 26 Jan 2022 23:15:28 +0100 Subject: [PATCH 39/44] fix sync of publications --- manifests/complete-postgres-manifest.yaml | 2 +- pkg/apis/acid.zalan.do/v1/util_test.go | 6 +++--- pkg/apis/zalando.org/v1/fabriceventstream.go | 3 ++- pkg/cluster/database.go | 7 ++++--- pkg/cluster/k8sres_test.go | 4 ++-- pkg/cluster/streams.go | 20 +++++++++----------- pkg/cluster/streams_test.go | 6 +++--- pkg/util/constants/streams.go | 1 + pkg/util/patroni/patroni_test.go | 6 +++--- 9 files changed, 28 insertions(+), 27 deletions(-) diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 28d0a970d..8252227d1 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -111,7 +111,7 @@ spec: # permanent_logical_1: # type: logical # database: foo -# plugin: pgoutput +# plugin: wal2json ttl: 30 loop_wait: &loop_wait 10 retry_timeout: 10 diff --git a/pkg/apis/acid.zalan.do/v1/util_test.go b/pkg/apis/acid.zalan.do/v1/util_test.go index bf6875a82..56ede36c4 100644 --- a/pkg/apis/acid.zalan.do/v1/util_test.go +++ b/pkg/apis/acid.zalan.do/v1/util_test.go @@ -252,7 +252,7 @@ var unmarshalCluster = []struct { "permanent_logical_1" : { "type" : "logical", "database" : "foo", - "plugin" : "pgoutput" + "plugin" : "wal2json" } } }, @@ -298,7 +298,7 @@ var unmarshalCluster = []struct { LoopWait: 10, RetryTimeout: 10, MaximumLagOnFailover: 33554432, - Slots: map[string]map[string]string{"permanent_logical_1": {"type": "logical", "database": "foo", "plugin": "pgoutput"}}, + Slots: map[string]map[string]string{"permanent_logical_1": {"type": "logical", "database": "foo", "plugin": "wal2json"}}, }, Resources: Resources{ ResourceRequests: ResourceDescription{CPU: "10m", Memory: "50Mi"}, @@ -334,7 +334,7 @@ var unmarshalCluster = []struct { }, Error: "", }, - marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"9.6","parameters":{"log_statement":"all","max_connections":"10","shared_buffers":"32MB"}},"pod_priority_class_name":"spilo-pod-priority","volume":{"size":"5Gi","storageClass":"SSD", "subPath": "subdir"},"enableShmVolume":false,"patroni":{"initdb":{"data-checksums":"true","encoding":"UTF8","locale":"en_US.UTF-8"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"],"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}},"resources":{"requests":{"cpu":"10m","memory":"50Mi"},"limits":{"cpu":"300m","memory":"3000Mi"}},"teamId":"acid","allowedSourceRanges":["127.0.0.1/32"],"numberOfInstances":2,"users":{"zalando":["superuser","createdb"]},"maintenanceWindows":["Mon:01:00-06:00","Sat:00:00-04:00","05:00-05:15"],"clone":{"cluster":"acid-batman"}},"status":{"PostgresClusterStatus":""}}`), + marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"9.6","parameters":{"log_statement":"all","max_connections":"10","shared_buffers":"32MB"}},"pod_priority_class_name":"spilo-pod-priority","volume":{"size":"5Gi","storageClass":"SSD", "subPath": "subdir"},"enableShmVolume":false,"patroni":{"initdb":{"data-checksums":"true","encoding":"UTF8","locale":"en_US.UTF-8"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"],"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"slots":{"permanent_logical_1":{"database":"foo","plugin":"wal2json","type":"logical"}}},"resources":{"requests":{"cpu":"10m","memory":"50Mi"},"limits":{"cpu":"300m","memory":"3000Mi"}},"teamId":"acid","allowedSourceRanges":["127.0.0.1/32"],"numberOfInstances":2,"users":{"zalando":["superuser","createdb"]},"maintenanceWindows":["Mon:01:00-06:00","Sat:00:00-04:00","05:00-05:15"],"clone":{"cluster":"acid-batman"}},"status":{"PostgresClusterStatus":""}}`), err: nil}, { about: "example with teamId set in input", diff --git a/pkg/apis/zalando.org/v1/fabriceventstream.go b/pkg/apis/zalando.org/v1/fabriceventstream.go index 81706802d..65d082c1e 100644 --- a/pkg/apis/zalando.org/v1/fabriceventstream.go +++ b/pkg/apis/zalando.org/v1/fabriceventstream.go @@ -70,7 +70,8 @@ type EventStreamTable struct { type Connection struct { Url string `json:"jdbcUrl"` SlotName string `json:"slotName"` - PublicationName string `json:"publicationName"` + PluginType string `json:"pluginType,omitempty" defaults:"wal2json"` + PublicationName string `json:"publicationName,omitempty"` DBAuth DBAuth `json:"databaseAuthentication"` } diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index 6121fac70..18563b0a9 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -517,8 +517,7 @@ func (c *Cluster) execCreateOrAlterExtension(extName, schemaName, statement, doi // The caller is responsible for opening and closing the database connection func (c *Cluster) getPublications() (publications map[string]string, err error) { var ( - rows *sql.Rows - dbPublications map[string]string + rows *sql.Rows ) if rows, err = c.pgDb.Query(getPublicationsSQL); err != nil { @@ -535,6 +534,8 @@ func (c *Cluster) getPublications() (publications map[string]string, err error) } }() + dbPublications := make(map[string]string) + for rows.Next() { var ( dbPublication string @@ -566,7 +567,7 @@ func (c *Cluster) executeAlterPublication(pubName, tableList string) error { func (c *Cluster) execCreateOrAlterPublication(pubName, tableList, statement, doing, operation string) error { - c.logger.Infof("%s %q table list %q", doing, pubName, tableList) + c.logger.Infof("%s %q with table list %q", doing, pubName, tableList) if _, err := c.pgDb.Exec(fmt.Sprintf(statement, pubName, tableList)); err != nil { return fmt.Errorf("could not execute %s: %v", operation, err) } diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index c3c4a56bd..fede3fca8 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -91,11 +91,11 @@ func TestGenerateSpiloJSONConfiguration(t *testing.T) { MaximumLagOnFailover: 33554432, SynchronousMode: true, SynchronousModeStrict: true, - Slots: map[string]map[string]string{"permanent_logical_1": {"type": "logical", "database": "foo", "plugin": "pgoutput"}}, + Slots: map[string]map[string]string{"permanent_logical_1": {"type": "logical", "database": "foo", "plugin": "wal2json"}}, }, role: "zalandos", opConfig: config.Config{}, - result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/11/bin","pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"zalandos":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"synchronous_mode":true,"synchronous_mode_strict":true,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}}}}`, + result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/11/bin","pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"zalandos":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"synchronous_mode":true,"synchronous_mode_strict":true,"slots":{"permanent_logical_1":{"database":"foo","plugin":"wal2json","type":"logical"}}}}}`, }, } for _, tt := range tests { diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index ebd110c32..3bcaf6b3e 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -75,7 +75,6 @@ func gatherApplicationIds(streams []acidv1.Stream) []string { } func (c *Cluster) syncPostgresConfig() error { - slots := make(map[string]map[string]string) publications := make(map[string]map[string]acidv1.StreamTable) createPublications := make(map[string]string) @@ -92,10 +91,11 @@ func (c *Cluster) syncPostgresConfig() error { slots = desiredPatroniConfig.Slots } + // define extra logical slots for Patroni config for _, stream := range c.Spec.Streams { slot := map[string]string{ "database": stream.Database, - "plugin": "pgoutput", + "plugin": constants.EventStreamSourcePluginType, "type": "logical", } slotName := getSlotName(stream.Database, stream.ApplicationId) @@ -114,18 +114,15 @@ func (c *Cluster) syncPostgresConfig() error { } if len(slots) > 0 { - c.logger.Debugf("setting wal level to 'logical' in Postgres configuration to allow for decoding changes") - for slotName, slot := range slots { - c.logger.Debugf("creating logical replication slot %q in database %q", slotName, slot["database"]) - } desiredPatroniConfig.Slots = slots } else { return nil } - // if streams are defined wal_level must be switched to logical and slots have to be defined + // if streams are defined wal_level must be switched to logical desiredPgParameters := map[string]string{"wal_level": "logical"} + // apply config changes in pods pods, err := c.listPods() if err != nil || len(pods) == 0 { c.logger.Warningf("could not list pods of the statefulset: %v", err) @@ -146,6 +143,7 @@ func (c *Cluster) syncPostgresConfig() error { } // next, create publications to each created slot + c.logger.Debug("syncing database publications") for publication, tables := range publications { // but first check for existing publications dbName := slots[publication]["database"] @@ -162,7 +160,7 @@ func (c *Cluster) syncPostgresConfig() error { i := 0 for t := range tables { tableName, schemaName := getTableSchema(t) - tableNames[i] = fmt.Sprintf("%q.%q", schemaName, tableName) + tableNames[i] = fmt.Sprintf("%s.%s", schemaName, tableName) i++ } sort.Strings(tableNames) @@ -286,9 +284,9 @@ func getSlotName(dbName, appId string) string { func (c *Cluster) getStreamConnection(database, user, appId string) zalandov1.Connection { return zalandov1.Connection{ - Url: fmt.Sprintf("jdbc:postgresql://%s.%s/%s?user=%s&ssl=true&sslmode=require", c.Name, c.Namespace, database, user), - SlotName: getSlotName(database, appId), - PublicationName: getSlotName(database, appId), + Url: fmt.Sprintf("jdbc:postgresql://%s.%s/%s?user=%s&ssl=true&sslmode=require", c.Name, c.Namespace, database, user), + SlotName: getSlotName(database, appId), + PluginType: constants.EventStreamSourcePluginType, DBAuth: zalandov1.DBAuth{ Type: constants.EventStreamSourceAuthType, Name: c.credentialSecretNameForCluster(user, c.Name), diff --git a/pkg/cluster/streams_test.go b/pkg/cluster/streams_test.go index f1c615e04..0068364c3 100644 --- a/pkg/cluster/streams_test.go +++ b/pkg/cluster/streams_test.go @@ -119,9 +119,9 @@ var ( Type: constants.EventStreamSourceAuthType, UserKey: "username", }, - Url: fmt.Sprintf("jdbc:postgresql://%s.%s/foo?user=%s&ssl=true&sslmode=require", clusterName, namespace, fesUser), - SlotName: slotName, - PublicationName: slotName, + Url: fmt.Sprintf("jdbc:postgresql://%s.%s/foo?user=%s&ssl=true&sslmode=require", clusterName, namespace, fesUser), + SlotName: slotName, + PluginType: constants.EventStreamSourcePluginType, }, Schema: "data", EventStreamTable: v1.EventStreamTable{ diff --git a/pkg/util/constants/streams.go b/pkg/util/constants/streams.go index 82d61c5c2..a636abf1c 100644 --- a/pkg/util/constants/streams.go +++ b/pkg/util/constants/streams.go @@ -6,6 +6,7 @@ const ( EventStreamSourceCRDName = "fabriceventstreams.zalando.org" EventStreamSourcePGType = "PostgresLogicalReplication" EventStreamSourceSlotPrefix = "fes" + EventStreamSourcePluginType = "pgoutput" EventStreamSourceAuthType = "DatabaseAuthenticationSecret" EventStreamFlowPgGenericType = "PostgresWalToGenericNakadiEvent" EventStreamSinkNakadiType = "Nakadi" diff --git a/pkg/util/patroni/patroni_test.go b/pkg/util/patroni/patroni_test.go index 60f289c6f..614f77828 100644 --- a/pkg/util/patroni/patroni_test.go +++ b/pkg/util/patroni/patroni_test.go @@ -185,7 +185,7 @@ func TestGetConfig(t *testing.T) { Slots: map[string]map[string]string{ "cdc": { "database": "foo", - "plugin": "pgoutput", + "plugin": "wal2json", "type": "logical", }, }, @@ -218,7 +218,7 @@ func TestGetConfig(t *testing.T) { "wal_log_hints": "on", } - configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 100, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "hot_standby", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "pgoutput", "type": "logical"}}, "ttl": 30}` + configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 100, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "hot_standby", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "wal2json", "type": "logical"}}, "ttl": 30}` r := ioutil.NopCloser(bytes.NewReader([]byte(configJson))) response := http.Response{ @@ -253,7 +253,7 @@ func TestSetPostgresParameters(t *testing.T) { "wal_level": "logical", } - configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 50, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "logical", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "pgoutput", "type": "logical"}}, "ttl": 30}` + configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 50, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "logical", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "wal2json", "type": "logical"}}, "ttl": 30}` r := ioutil.NopCloser(bytes.NewReader([]byte(configJson))) response := http.Response{ From 66824076e59db093fe3263264b50c5018d6dffdc Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 28 Jan 2022 14:21:23 +0100 Subject: [PATCH 40/44] cosmetic changes --- manifests/complete-postgres-manifest.yaml | 2 +- pkg/apis/acid.zalan.do/v1/util_test.go | 6 +++--- pkg/cluster/database.go | 2 +- pkg/cluster/k8sres_test.go | 4 ++-- pkg/cluster/streams.go | 6 +++--- pkg/cluster/streams_test.go | 2 +- pkg/util/patroni/patroni_test.go | 6 +++--- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 8252227d1..28d0a970d 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -111,7 +111,7 @@ spec: # permanent_logical_1: # type: logical # database: foo -# plugin: wal2json +# plugin: pgoutput ttl: 30 loop_wait: &loop_wait 10 retry_timeout: 10 diff --git a/pkg/apis/acid.zalan.do/v1/util_test.go b/pkg/apis/acid.zalan.do/v1/util_test.go index 56ede36c4..bf6875a82 100644 --- a/pkg/apis/acid.zalan.do/v1/util_test.go +++ b/pkg/apis/acid.zalan.do/v1/util_test.go @@ -252,7 +252,7 @@ var unmarshalCluster = []struct { "permanent_logical_1" : { "type" : "logical", "database" : "foo", - "plugin" : "wal2json" + "plugin" : "pgoutput" } } }, @@ -298,7 +298,7 @@ var unmarshalCluster = []struct { LoopWait: 10, RetryTimeout: 10, MaximumLagOnFailover: 33554432, - Slots: map[string]map[string]string{"permanent_logical_1": {"type": "logical", "database": "foo", "plugin": "wal2json"}}, + Slots: map[string]map[string]string{"permanent_logical_1": {"type": "logical", "database": "foo", "plugin": "pgoutput"}}, }, Resources: Resources{ ResourceRequests: ResourceDescription{CPU: "10m", Memory: "50Mi"}, @@ -334,7 +334,7 @@ var unmarshalCluster = []struct { }, Error: "", }, - marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"9.6","parameters":{"log_statement":"all","max_connections":"10","shared_buffers":"32MB"}},"pod_priority_class_name":"spilo-pod-priority","volume":{"size":"5Gi","storageClass":"SSD", "subPath": "subdir"},"enableShmVolume":false,"patroni":{"initdb":{"data-checksums":"true","encoding":"UTF8","locale":"en_US.UTF-8"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"],"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"slots":{"permanent_logical_1":{"database":"foo","plugin":"wal2json","type":"logical"}}},"resources":{"requests":{"cpu":"10m","memory":"50Mi"},"limits":{"cpu":"300m","memory":"3000Mi"}},"teamId":"acid","allowedSourceRanges":["127.0.0.1/32"],"numberOfInstances":2,"users":{"zalando":["superuser","createdb"]},"maintenanceWindows":["Mon:01:00-06:00","Sat:00:00-04:00","05:00-05:15"],"clone":{"cluster":"acid-batman"}},"status":{"PostgresClusterStatus":""}}`), + marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"9.6","parameters":{"log_statement":"all","max_connections":"10","shared_buffers":"32MB"}},"pod_priority_class_name":"spilo-pod-priority","volume":{"size":"5Gi","storageClass":"SSD", "subPath": "subdir"},"enableShmVolume":false,"patroni":{"initdb":{"data-checksums":"true","encoding":"UTF8","locale":"en_US.UTF-8"},"pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"],"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}},"resources":{"requests":{"cpu":"10m","memory":"50Mi"},"limits":{"cpu":"300m","memory":"3000Mi"}},"teamId":"acid","allowedSourceRanges":["127.0.0.1/32"],"numberOfInstances":2,"users":{"zalando":["superuser","createdb"]},"maintenanceWindows":["Mon:01:00-06:00","Sat:00:00-04:00","05:00-05:15"],"clone":{"cluster":"acid-batman"}},"status":{"PostgresClusterStatus":""}}`), err: nil}, { about: "example with teamId set in input", diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index 18563b0a9..e7553754d 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -562,7 +562,7 @@ func (c *Cluster) executeCreatePublication(pubName, tableList string) error { // The caller is responsible for opening and closing the database connection. func (c *Cluster) executeAlterPublication(pubName, tableList string) error { return c.execCreateOrAlterPublication(pubName, tableList, alterPublicationSQL, - "changing table list of publication", "alter publication tables") + "changing publication", "alter publication tables") } func (c *Cluster) execCreateOrAlterPublication(pubName, tableList, statement, doing, operation string) error { diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index fede3fca8..c3c4a56bd 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -91,11 +91,11 @@ func TestGenerateSpiloJSONConfiguration(t *testing.T) { MaximumLagOnFailover: 33554432, SynchronousMode: true, SynchronousModeStrict: true, - Slots: map[string]map[string]string{"permanent_logical_1": {"type": "logical", "database": "foo", "plugin": "wal2json"}}, + Slots: map[string]map[string]string{"permanent_logical_1": {"type": "logical", "database": "foo", "plugin": "pgoutput"}}, }, role: "zalandos", opConfig: config.Config{}, - result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/11/bin","pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"zalandos":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"synchronous_mode":true,"synchronous_mode_strict":true,"slots":{"permanent_logical_1":{"database":"foo","plugin":"wal2json","type":"logical"}}}}}`, + result: `{"postgresql":{"bin_dir":"/usr/lib/postgresql/11/bin","pg_hba":["hostssl all all 0.0.0.0/0 md5","host all all 0.0.0.0/0 md5"]},"bootstrap":{"initdb":[{"auth-host":"md5"},{"auth-local":"trust"},"data-checksums",{"encoding":"UTF8"},{"locale":"en_US.UTF-8"}],"users":{"zalandos":{"password":"","options":["CREATEDB","NOLOGIN"]}},"dcs":{"ttl":30,"loop_wait":10,"retry_timeout":10,"maximum_lag_on_failover":33554432,"synchronous_mode":true,"synchronous_mode_strict":true,"slots":{"permanent_logical_1":{"database":"foo","plugin":"pgoutput","type":"logical"}}}}}`, }, } for _, tt := range tests { diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index 3bcaf6b3e..75675d490 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -217,7 +217,7 @@ func (c *Cluster) generateFabricEventStream(appId string) *zalandov1.FabricEvent APIVersion: "zalando.org/v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: c.Name + "-" + appId, + Name: fmt.Sprintf("%s-%s", c.Name, appId), Namespace: c.Namespace, Annotations: c.AnnotationsToPropagate(c.annotationsSet(nil)), // make cluster StatefulSet the owner (like with connection pooler objects) @@ -279,7 +279,7 @@ func getOutboxTable(tableName, idColumn string) zalandov1.EventStreamTable { } func getSlotName(dbName, appId string) string { - return constants.EventStreamSourceSlotPrefix + "_" + dbName + "_" + strings.Replace(appId, "-", "_", -1) + return fmt.Sprintf("%s_%s_%s", constants.EventStreamSourceSlotPrefix, dbName, strings.Replace(appId, "-", "_", -1)) } func (c *Cluster) getStreamConnection(database, user, appId string) zalandov1.Connection { @@ -323,7 +323,7 @@ func (c *Cluster) createOrUpdateStreams() error { appIds := gatherApplicationIds(c.Spec.Streams) for _, appId := range appIds { - fesName := c.Name + "-" + appId + fesName := fmt.Sprintf("%s-%s", c.Name, appId) effectiveStreams, err := c.KubeClient.FabricEventStreams(c.Namespace).Get(context.TODO(), fesName, metav1.GetOptions{}) if err != nil { if !k8sutil.ResourceNotFound(err) { diff --git a/pkg/cluster/streams_test.go b/pkg/cluster/streams_test.go index 0068364c3..283b956e9 100644 --- a/pkg/cluster/streams_test.go +++ b/pkg/cluster/streams_test.go @@ -41,7 +41,7 @@ var ( appId string = "test-app" dbName string = "foo" fesUser string = constants.EventStreamSourceSlotPrefix + constants.UserRoleNameSuffix - fesName string = clusterName + "-" + appId + fesName string = fmt.Sprintf("%s-%s", clusterName, appId) slotName string = fmt.Sprintf("%s_%s_%s", constants.EventStreamSourceSlotPrefix, dbName, strings.Replace(appId, "-", "_", -1)) pg = acidv1.Postgresql{ diff --git a/pkg/util/patroni/patroni_test.go b/pkg/util/patroni/patroni_test.go index 614f77828..60f289c6f 100644 --- a/pkg/util/patroni/patroni_test.go +++ b/pkg/util/patroni/patroni_test.go @@ -185,7 +185,7 @@ func TestGetConfig(t *testing.T) { Slots: map[string]map[string]string{ "cdc": { "database": "foo", - "plugin": "wal2json", + "plugin": "pgoutput", "type": "logical", }, }, @@ -218,7 +218,7 @@ func TestGetConfig(t *testing.T) { "wal_log_hints": "on", } - configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 100, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "hot_standby", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "wal2json", "type": "logical"}}, "ttl": 30}` + configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 100, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "hot_standby", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "pgoutput", "type": "logical"}}, "ttl": 30}` r := ioutil.NopCloser(bytes.NewReader([]byte(configJson))) response := http.Response{ @@ -253,7 +253,7 @@ func TestSetPostgresParameters(t *testing.T) { "wal_level": "logical", } - configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 50, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "logical", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "wal2json", "type": "logical"}}, "ttl": 30}` + configJson := `{"loop_wait": 10, "maximum_lag_on_failover": 33554432, "postgresql": {"parameters": {"archive_mode": "on", "archive_timeout": "1800s", "autovacuum_analyze_scale_factor": 0.02, "autovacuum_max_workers": 5, "autovacuum_vacuum_scale_factor": 0.05, "checkpoint_completion_target": 0.9, "hot_standby": "on", "log_autovacuum_min_duration": 0, "log_checkpoints": "on", "log_connections": "on", "log_disconnections": "on", "log_line_prefix": "%t [%p]: [%l-1] %c %x %d %u %a %h ", "log_lock_waits": "on", "log_min_duration_statement": 500, "log_statement": "ddl", "log_temp_files": 0, "max_connections": 50, "max_replication_slots": 10, "max_wal_senders": 10, "tcp_keepalives_idle": 900, "tcp_keepalives_interval": 100, "track_functions": "all", "wal_level": "logical", "wal_log_hints": "on"}, "use_pg_rewind": true, "use_slots": true}, "retry_timeout": 10, "slots": {"cdc": {"database": "foo", "plugin": "pgoutput", "type": "logical"}}, "ttl": 30}` r := ioutil.NopCloser(bytes.NewReader([]byte(configJson))) response := http.Response{ From 9007475ed83beaa30e11c6f383da0547a2986290 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 21 Feb 2022 17:01:58 +0100 Subject: [PATCH 41/44] reflect code review and additional refactoring --- docs/reference/cluster_manifest.md | 17 +- pkg/cluster/streams.go | 240 +++++++++++++++-------------- pkg/cluster/streams_test.go | 4 +- pkg/util/constants/streams.go | 5 +- 4 files changed, 144 insertions(+), 122 deletions(-) diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 0e1cad1af..fe1f6dfd7 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -536,11 +536,18 @@ Those parameters are grouped under the `tls` top-level key. ## Change data capture streams -This sections enables change data capture (CDC) streams e.g. into Zalando’s -distributed event broker [Nakadi](https://nakadi.io/). Parameters grouped -under the `streams` top-level key will be used by the operator to create -custom resources for Zalando's internal CDC operator. Each stream object can -have the following properties: +This sections enables change data capture (CDC) streams via Postgres' +[logical decoding](https://www.postgresql.org/docs/14/logicaldecoding.html) +feature and `pgoutput` plugin. While the Postgres operator takes responsibility +for providing the setup to publish change events, it relies on external tools +to consume them. At Zalando, we are using a workflow based on +[Debezium Connector](https://debezium.io/documentation/reference/stable/connectors/postgresql.html) +which can feed streams into Zalando’s distributed event broker [Nakadi](https://nakadi.io/) +among others. + +The Postgres Operator creates custom resources for Zalando's internal CDC +operator which will be used to set up the consumer part. Each stream object +can have the following properties: * **applicationId** The application name to which the database and CDC belongs to. For each diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index 75675d490..a6d498d83 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -15,32 +15,22 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func (c *Cluster) createStreams(appId string) { +func (c *Cluster) createStreams(appId string) error { c.setProcessName("creating streams") - var ( - fes *zalandov1.FabricEventStream - err error - ) - - msg := "could not create event stream custom resource with applicationId %s: %v" - - fes = c.generateFabricEventStream(appId) - if err != nil { - c.logger.Warningf(msg, appId, err) - } - _, err = c.KubeClient.FabricEventStreams(c.Namespace).Create(context.TODO(), fes, metav1.CreateOptions{}) - if err != nil { - c.logger.Warningf(msg, appId, err) + fes := c.generateFabricEventStream(appId) + if _, err := c.KubeClient.FabricEventStreams(c.Namespace).Create(context.TODO(), fes, metav1.CreateOptions{}); err != nil { + return err } + + return nil } func (c *Cluster) updateStreams(newEventStreams *zalandov1.FabricEventStream) error { c.setProcessName("updating event streams") - _, err := c.KubeClient.FabricEventStreams(newEventStreams.Namespace).Update(context.TODO(), newEventStreams, metav1.UpdateOptions{}) - if err != nil { - return fmt.Errorf("could not update event stream custom resource: %v", err) + if _, err := c.KubeClient.FabricEventStreams(newEventStreams.Namespace).Update(context.TODO(), newEventStreams, metav1.UpdateOptions{}); err != nil { + return err } return nil @@ -50,7 +40,7 @@ func (c *Cluster) deleteStreams() error { c.setProcessName("deleting event streams") // check if stream CRD is installed before trying a delete - _, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), constants.EventStreamSourceCRDName, metav1.GetOptions{}) + _, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), constants.EventStreamCRDName, metav1.GetOptions{}) if k8sutil.ResourceNotFound(err) { return nil } @@ -74,118 +64,87 @@ func gatherApplicationIds(streams []acidv1.Stream) []string { return appIds } -func (c *Cluster) syncPostgresConfig() error { - slots := make(map[string]map[string]string) - publications := make(map[string]map[string]acidv1.StreamTable) - createPublications := make(map[string]string) - alterPublications := make(map[string]string) - - defer func() { - if err := c.closeDbConn(); err != nil { - c.logger.Errorf("could not close database connection: %v", err) - } - }() - - desiredPatroniConfig := c.Spec.Patroni - if len(desiredPatroniConfig.Slots) > 0 { - slots = desiredPatroniConfig.Slots - } - - // define extra logical slots for Patroni config - for _, stream := range c.Spec.Streams { - slot := map[string]string{ - "database": stream.Database, - "plugin": constants.EventStreamSourcePluginType, - "type": "logical", - } - slotName := getSlotName(stream.Database, stream.ApplicationId) - if _, exists := slots[slotName]; !exists { - slots[slotName] = slot - publications[slotName] = stream.Tables - } else { - streamTables := publications[slotName] - for tableName, table := range stream.Tables { - if _, exists := streamTables[tableName]; !exists { - streamTables[tableName] = table - } - } - publications[slotName] = streamTables - } - } - - if len(slots) > 0 { - desiredPatroniConfig.Slots = slots - } else { - return nil - } +func (c *Cluster) syncPostgresConfig(requiredPatroniConfig acidv1.Patroni) error { + errorMsg := "no pods found to update config" // if streams are defined wal_level must be switched to logical - desiredPgParameters := map[string]string{"wal_level": "logical"} + requiredPgParameters := map[string]string{"wal_level": "logical"} // apply config changes in pods pods, err := c.listPods() - if err != nil || len(pods) == 0 { - c.logger.Warningf("could not list pods of the statefulset: %v", err) + if err != nil { + errorMsg = fmt.Sprintf("could not list pods of the statefulset: %v", err) } for i, pod := range pods { podName := util.NameFromMeta(pods[i].ObjectMeta) effectivePatroniConfig, effectivePgParameters, err := c.patroni.GetConfig(&pod) if err != nil { - c.logger.Warningf("could not get Postgres config from pod %s: %v", podName, err) + errorMsg = fmt.Sprintf("could not get Postgres config from pod %s: %v", podName, err) continue } - _, err = c.checkAndSetGlobalPostgreSQLConfiguration(&pod, effectivePatroniConfig, desiredPatroniConfig, effectivePgParameters, desiredPgParameters) + _, err = c.checkAndSetGlobalPostgreSQLConfiguration(&pod, effectivePatroniConfig, requiredPatroniConfig, effectivePgParameters, requiredPgParameters) if err != nil { - c.logger.Warningf("could not set PostgreSQL configuration options for pod %s: %v", podName, err) + errorMsg = fmt.Sprintf("could not set PostgreSQL configuration options for pod %s: %v", podName, err) continue } + + // Patroni's config endpoint is just a "proxy" to DCS. It is enough to patch it only once and it doesn't matter which pod is used + return nil } - // next, create publications to each created slot - c.logger.Debug("syncing database publications") - for publication, tables := range publications { - // but first check for existing publications - dbName := slots[publication]["database"] - if err := c.initDbConnWithName(dbName); err != nil { - return fmt.Errorf("could not init database connection") - } + return fmt.Errorf(errorMsg) +} - currentPublications, err := c.getPublications() - if err != nil { - return fmt.Errorf("could not get current publications: %v", err) - } +func (c *Cluster) syncPublication(publication, dbName string, tables map[string]acidv1.StreamTable) error { + createPublications := make(map[string]string) + alterPublications := make(map[string]string) - tableNames := make([]string, len(tables)) - i := 0 - for t := range tables { - tableName, schemaName := getTableSchema(t) - tableNames[i] = fmt.Sprintf("%s.%s", schemaName, tableName) - i++ - } - sort.Strings(tableNames) - tableList := strings.Join(tableNames, ", ") - - currentTables, exists := currentPublications[publication] - if !exists { - createPublications[publication] = tableList - } else if currentTables != tableList { - alterPublications[publication] = tableList + defer func() { + if err := c.closeDbConn(); err != nil { + c.logger.Errorf("could not close database connection: %v", err) } + }() - if len(createPublications)+len(alterPublications) == 0 { - return nil - } + // check for existing publications + if err := c.initDbConnWithName(dbName); err != nil { + return fmt.Errorf("could not init database connection") + } - for publicationName, tables := range createPublications { - if err = c.executeCreatePublication(publicationName, tables); err != nil { - c.logger.Warningf("%v", err) - } + currentPublications, err := c.getPublications() + if err != nil { + return fmt.Errorf("could not get current publications: %v", err) + } + + tableNames := make([]string, len(tables)) + i := 0 + for t := range tables { + tableName, schemaName := getTableSchema(t) + tableNames[i] = fmt.Sprintf("%s.%s", schemaName, tableName) + i++ + } + sort.Strings(tableNames) + tableList := strings.Join(tableNames, ", ") + + currentTables, exists := currentPublications[publication] + if !exists { + createPublications[publication] = tableList + } else if currentTables != tableList { + alterPublications[publication] = tableList + } + + if len(createPublications)+len(alterPublications) == 0 { + return nil + } + + for publicationName, tables := range createPublications { + if err = c.executeCreatePublication(publicationName, tables); err != nil { + return fmt.Errorf("creation of publication %q failed: %v", publicationName, err) } - for publicationName, tables := range alterPublications { - if err = c.executeAlterPublication(publicationName, tables); err != nil { - c.logger.Warningf("%v", err) - } + } + for publicationName, tables := range alterPublications { + if err = c.executeAlterPublication(publicationName, tables); err != nil { + return fmt.Errorf("update of publication %q failed: %v", publicationName, err) } } @@ -213,8 +172,8 @@ func (c *Cluster) generateFabricEventStream(appId string) *zalandov1.FabricEvent return &zalandov1.FabricEventStream{ TypeMeta: metav1.TypeMeta{ - Kind: constants.EventStreamSourceCRDKind, - APIVersion: "zalando.org/v1", + APIVersion: constants.EventStreamCRDApiVersion, + Kind: constants.EventStreamCRDKind, }, ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-%s", c.Name, appId), @@ -300,15 +259,65 @@ func (c *Cluster) syncStreams() error { c.setProcessName("syncing streams") - _, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), constants.EventStreamSourceCRDName, metav1.GetOptions{}) + _, err := c.KubeClient.CustomResourceDefinitions().Get(context.TODO(), constants.EventStreamCRDName, metav1.GetOptions{}) if k8sutil.ResourceNotFound(err) { c.logger.Debugf("event stream CRD not installed, skipping") return nil } - err = c.syncPostgresConfig() + slots := make(map[string]map[string]string) + publications := make(map[string]map[string]acidv1.StreamTable) + + requiredPatroniConfig := c.Spec.Patroni + if len(requiredPatroniConfig.Slots) > 0 { + slots = requiredPatroniConfig.Slots + } + + // gather list of required slots and publications + for _, stream := range c.Spec.Streams { + slot := map[string]string{ + "database": stream.Database, + "plugin": constants.EventStreamSourcePluginType, + "type": "logical", + } + slotName := getSlotName(stream.Database, stream.ApplicationId) + if _, exists := slots[slotName]; !exists { + slots[slotName] = slot + publications[slotName] = stream.Tables + } else { + streamTables := publications[slotName] + for tableName, table := range stream.Tables { + if _, exists := streamTables[tableName]; !exists { + streamTables[tableName] = table + } + } + publications[slotName] = streamTables + } + } + + // no slots = no streams defined + if len(slots) > 0 { + requiredPatroniConfig.Slots = slots + } else { + return nil + } + + // add extra logical slots to Patroni config + c.logger.Debug("syncing Postgres config for logical decoding") + err = c.syncPostgresConfig(requiredPatroniConfig) if err != nil { - return fmt.Errorf("could not update Postgres config for event streaming: %v", err) + return fmt.Errorf("failed to snyc Postgres config for event streaming: %v", err) + } + + // next, create publications to each created slot + c.logger.Debug("syncing database publications") + for publication, tables := range publications { + // but first check for existing publications + dbName := slots[publication]["database"] + err = c.syncPublication(publication, dbName, tables) + if err != nil { + c.logger.Warningf("could not sync publication %q in database %d: %v", publication, dbName, err) + } } err = c.createOrUpdateStreams() @@ -331,7 +340,11 @@ func (c *Cluster) createOrUpdateStreams() error { } c.logger.Infof("event streams do not exist, create it") - c.createStreams(appId) + err = c.createStreams(appId) + if err != nil { + return fmt.Errorf("failed creating event stream %s: %v", fesName, err) + } + c.logger.Infof("event stream %q has been successfully created", fesName) } else { desiredStreams := c.generateFabricEventStream(appId) if !reflect.DeepEqual(effectiveStreams.Spec, desiredStreams.Spec) { @@ -341,6 +354,7 @@ func (c *Cluster) createOrUpdateStreams() error { if err != nil { return fmt.Errorf("failed updating event stream %s: %v", fesName, err) } + c.logger.Infof("event stream %q has been successfully updated", fesName) } } } diff --git a/pkg/cluster/streams_test.go b/pkg/cluster/streams_test.go index 283b956e9..89dd294ca 100644 --- a/pkg/cluster/streams_test.go +++ b/pkg/cluster/streams_test.go @@ -82,8 +82,8 @@ var ( fes = &v1.FabricEventStream{ TypeMeta: metav1.TypeMeta{ - Kind: "FabricEventStream", - APIVersion: "zalando.org/v1", + APIVersion: constants.EventStreamCRDApiVersion, + Kind: constants.EventStreamCRDKind, }, ObjectMeta: metav1.ObjectMeta{ Name: fesName, diff --git a/pkg/util/constants/streams.go b/pkg/util/constants/streams.go index a636abf1c..bd70b719b 100644 --- a/pkg/util/constants/streams.go +++ b/pkg/util/constants/streams.go @@ -2,8 +2,9 @@ package constants // PostgreSQL specific constants const ( - EventStreamSourceCRDKind = "FabricEventStream" - EventStreamSourceCRDName = "fabriceventstreams.zalando.org" + EventStreamCRDApiVersion = "zalando.org/v1" + EventStreamCRDKind = "FabricEventStream" + EventStreamCRDName = "fabriceventstreams.zalando.org" EventStreamSourcePGType = "PostgresLogicalReplication" EventStreamSourceSlotPrefix = "fes" EventStreamSourcePluginType = "pgoutput" From 34cad4601d169c725a0cc296dd584630c93f3e48 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 21 Feb 2022 17:08:36 +0100 Subject: [PATCH 42/44] fix warning formatting --- pkg/cluster/streams.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cluster/streams.go b/pkg/cluster/streams.go index a6d498d83..a1a1b4d4a 100644 --- a/pkg/cluster/streams.go +++ b/pkg/cluster/streams.go @@ -316,7 +316,7 @@ func (c *Cluster) syncStreams() error { dbName := slots[publication]["database"] err = c.syncPublication(publication, dbName, tables) if err != nil { - c.logger.Warningf("could not sync publication %q in database %d: %v", publication, dbName, err) + c.logger.Warningf("could not sync publication %q in database %q: %v", publication, dbName, err) } } From 795cd5d4ec3bf5f612aff62b3c52e8a2d50997ba Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 21 Feb 2022 17:27:10 +0100 Subject: [PATCH 43/44] sync publication with debug messages --- manifests/complete-postgres-manifest.yaml | 16 +++++++++------- pkg/cluster/database.go | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 74b3c7f85..514288e2e 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -205,13 +205,15 @@ spec: # - applicationId: test-app # database: foo # tables: -# data.tab_a: -# eventType: event_type_a -# data.tab_b: -# eventType: event_type_b -# idColumn: tb_id -# payloadColumn: tb_payload +# data.state_pending_outbox: +# eventType: test-app.status-pending +# data.state_approved_outbox: +# eventType: test-app.status-approved +# data.orders_outbox: +# eventType: test-app.order-completed +# idColumn: o_id +# payloadColumn: o_payload # # Optional. Filter ignores events before a certain txnId and lsn. Can be used to skip bad events # filter: -# data.tab_a: "[?(@.source.txId > 500 && @.source.lsn > 123456)]" +# data.orders_outbox: "[?(@.source.txId > 500 && @.source.lsn > 123456)]" # batchSize: 1000 diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index c30acd115..652f0d0ae 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -633,7 +633,7 @@ func (c *Cluster) executeAlterPublication(pubName, tableList string) error { func (c *Cluster) execCreateOrAlterPublication(pubName, tableList, statement, doing, operation string) error { - c.logger.Infof("%s %q with table list %q", doing, pubName, tableList) + c.logger.Debugf("%s %q with table list %q", doing, pubName, tableList) if _, err := c.pgDb.Exec(fmt.Sprintf(statement, pubName, tableList)); err != nil { return fmt.Errorf("could not execute %s: %v", operation, err) } From d6ed3c21a40036ca4633d2aa094584bffe2b4bab Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 21 Feb 2022 17:51:12 +0100 Subject: [PATCH 44/44] fix manifest example for password rotation --- manifests/complete-postgres-manifest.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 514288e2e..80918457c 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -18,8 +18,12 @@ spec: - superuser - createdb foo_user: [] -# usersWithSecretRotation: "foo_user" -# usersWithInPlaceSecretRotation: "flyway,bar_owner_user" +# flyway: [] +# usersWithSecretRotation: +# - foo_user +# usersWithInPlaceSecretRotation: +# - flyway +# - bar_owner_user enableMasterLoadBalancer: false enableReplicaLoadBalancer: false enableConnectionPooler: false # enable/disable connection pooler deployment