From 88735a798a8c9ef98897ac94efa09a14527de83b Mon Sep 17 00:00:00 2001 From: Igor Yanchenko <1504692+yanchenko-igor@users.noreply.github.com> Date: Fri, 3 Jul 2020 11:53:37 +0300 Subject: [PATCH 01/35] Resize volume by changing pvc size if enabled in config. (#958) * Try to resize pvc if resizing pv has failed * added config option to switch between storage resize strategies * changes according to requests * Update pkg/controller/operator_config.go Co-authored-by: Felix Kunde * enable_storage_resize documented added examples to the default configuration and helm value files * enable_storage_resize renamed to volume_resize_mode, off by default * volume_resize_mode renamed to storage_resize_mode * Update pkg/apis/acid.zalan.do/v1/crds.go * pkg/cluster/volumes.go updated * Update docs/reference/operator_parameters.md * Update manifests/postgresql-operator-default-configuration.yaml * Update pkg/controller/operator_config.go * Update pkg/util/config/config.go * Update charts/postgres-operator/values-crd.yaml * Update charts/postgres-operator/values.yaml * Update docs/reference/operator_parameters.md * added logging if no changes required Co-authored-by: Felix Kunde --- charts/postgres-operator/values-crd.yaml | 2 + charts/postgres-operator/values.yaml | 2 + docs/reference/operator_parameters.md | 6 +++ manifests/configmap.yaml | 1 + manifests/operatorconfiguration.crd.yaml | 6 +++ ...gresql-operator-default-configuration.yaml | 1 + pkg/apis/acid.zalan.do/v1/crds.go | 14 +++++ .../v1/operator_configuration_type.go | 1 + pkg/cluster/sync.go | 51 +++++++++++++++---- pkg/cluster/volumes.go | 50 +++++++++++++++++- pkg/controller/operator_config.go | 1 + pkg/util/config/config.go | 1 + 12 files changed, 125 insertions(+), 11 deletions(-) diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 14287fdaf..db26e6d98 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -124,6 +124,8 @@ configKubernetes: # whether the Spilo container should run in privileged mode spilo_privileged: false + # storage resize strategy, available options are: ebs, pvc, off + storage_resize_mode: ebs # operator watches for postgres objects in the given namespace watched_namespace: "*" # listen to all namespaces diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index cce0b79c8..eb5c10b0e 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -115,6 +115,8 @@ configKubernetes: # whether the Spilo container should run in privileged mode spilo_privileged: "false" + # storage resize strategy, available options are: ebs, pvc, off + storage_resize_mode: ebs # operator watches for postgres objects in the given namespace watched_namespace: "*" # listen to all namespaces diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index f8189f913..7e5196d56 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -333,6 +333,12 @@ configuration they are grouped under the `kubernetes` key. of stateful sets of PG clusters. The default is `ordered_ready`, the second possible value is `parallel`. +* **storage_resize_mode** + defines how operator handels the difference between requested volume size and + actual size. Available options are: ebs - tries to resize EBS volume, pvc - + changes PVC definition, off - disables resize of the volumes. Default is "ebs". + When using OpenShift please use one of the other available options. + ## Kubernetes resource requests This group allows you to configure resource requests for the Postgres pods. diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 963aea96b..d666b0383 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -97,6 +97,7 @@ data: # set_memory_request_to_limit: "false" # spilo_fsgroup: 103 spilo_privileged: "false" + # storage_resize_mode: "off" super_username: postgres # team_admin_role: "admin" # team_api_role_configuration: "log_statement:all" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 364ea6d5a..346eabb4a 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -168,6 +168,12 @@ spec: type: integer spilo_privileged: type: boolean + storage_resize_mode: + type: string + enum: + - "ebs" + - "pvc" + - "off" toleration: type: object additionalProperties: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index ab6a23113..cb7b1ed11 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -59,6 +59,7 @@ configuration: secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}" # spilo_fsgroup: 103 spilo_privileged: false + storage_resize_mode: ebs # toleration: {} # watched_namespace: "" postgres_pod_resources: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 43410ed3b..bc38d6dfd 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -980,6 +980,20 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "spilo_privileged": { Type: "boolean", }, + "storage_resize_mode": { + Type: "string", + Enum: []apiextv1beta1.JSON{ + { + Raw: []byte(`"ebs"`), + }, + { + Raw: []byte(`"pvc"`), + }, + { + Raw: []byte(`"off"`), + }, + }, + }, "toleration": { Type: "object", AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 2dd0bbb50..5ac5a4677 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -53,6 +53,7 @@ type KubernetesMetaConfiguration struct { WatchedNamespace string `json:"watched_namespace,omitempty"` PDBNameFormat config.StringTemplate `json:"pdb_name_format,omitempty"` EnablePodDisruptionBudget *bool `json:"enable_pod_disruption_budget,omitempty"` + StorageResizeMode string `json:"storage_resize_mode,omitempty"` EnableInitContainers *bool `json:"enable_init_containers,omitempty"` EnableSidecars *bool `json:"enable_sidecars,omitempty"` SecretNameTemplate config.StringTemplate `json:"secret_name_template,omitempty"` diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index e49bd4537..b03b5d494 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -57,16 +57,26 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { return err } - // potentially enlarge volumes before changing the statefulset. By doing that - // in this order we make sure the operator is not stuck waiting for a pod that - // cannot start because it ran out of disk space. - // TODO: handle the case of the cluster that is downsized and enlarged again - // (there will be a volume from the old pod for which we can't act before the - // the statefulset modification is concluded) - c.logger.Debugf("syncing persistent volumes") - if err = c.syncVolumes(); err != nil { - err = fmt.Errorf("could not sync persistent volumes: %v", err) - return err + if c.OpConfig.StorageResizeMode == "pvc" { + c.logger.Debugf("syncing persistent volume claims") + if err = c.syncVolumeClaims(); err != nil { + err = fmt.Errorf("could not sync persistent volume claims: %v", err) + return err + } + } else if c.OpConfig.StorageResizeMode == "ebs" { + // potentially enlarge volumes before changing the statefulset. By doing that + // in this order we make sure the operator is not stuck waiting for a pod that + // cannot start because it ran out of disk space. + // TODO: handle the case of the cluster that is downsized and enlarged again + // (there will be a volume from the old pod for which we can't act before the + // the statefulset modification is concluded) + c.logger.Debugf("syncing persistent volumes") + if err = c.syncVolumes(); err != nil { + err = fmt.Errorf("could not sync persistent volumes: %v", err) + return err + } + } else { + c.logger.Infof("Storage resize is disabled (storage_resize_mode is off). Skipping volume sync.") } if err = c.enforceMinResourceLimits(&c.Spec); err != nil { @@ -571,6 +581,27 @@ func (c *Cluster) syncRoles() (err error) { return nil } +// syncVolumeClaims reads all persistent volume claims and checks that their size matches the one declared in the statefulset. +func (c *Cluster) syncVolumeClaims() error { + c.setProcessName("syncing volume claims") + + act, err := c.volumeClaimsNeedResizing(c.Spec.Volume) + if err != nil { + return fmt.Errorf("could not compare size of the volume claims: %v", err) + } + if !act { + c.logger.Infof("volume claims don't require changes") + return nil + } + if err := c.resizeVolumeClaims(c.Spec.Volume); err != nil { + return fmt.Errorf("could not sync volume claims: %v", err) + } + + c.logger.Infof("volume claims have been synced successfully") + + return nil +} + // syncVolumes reads all persistent volumes and checks that their size matches the one declared in the statefulset. func (c *Cluster) syncVolumes() error { c.setProcessName("syncing volumes") diff --git a/pkg/cluster/volumes.go b/pkg/cluster/volumes.go index a5bfe6c2d..d5c08c2e2 100644 --- a/pkg/cluster/volumes.go +++ b/pkg/cluster/volumes.go @@ -52,6 +52,35 @@ func (c *Cluster) deletePersistentVolumeClaims() error { return nil } +func (c *Cluster) resizeVolumeClaims(newVolume acidv1.Volume) error { + c.logger.Debugln("resizing PVCs") + pvcs, err := c.listPersistentVolumeClaims() + if err != nil { + return err + } + newQuantity, err := resource.ParseQuantity(newVolume.Size) + if err != nil { + return fmt.Errorf("could not parse volume size: %v", err) + } + _, newSize, err := c.listVolumesWithManifestSize(newVolume) + for _, pvc := range pvcs { + volumeSize := quantityToGigabyte(pvc.Spec.Resources.Requests[v1.ResourceStorage]) + if volumeSize >= newSize { + if volumeSize > newSize { + c.logger.Warningf("cannot shrink persistent volume") + } + continue + } + pvc.Spec.Resources.Requests[v1.ResourceStorage] = newQuantity + c.logger.Debugf("updating persistent volume claim definition for volume %q", pvc.Name) + if _, err := c.KubeClient.PersistentVolumeClaims(pvc.Namespace).Update(context.TODO(), &pvc, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("could not update persistent volume claim: %q", err) + } + c.logger.Debugf("successfully updated persistent volume claim %q", pvc.Name) + } + return nil +} + func (c *Cluster) listPersistentVolumes() ([]*v1.PersistentVolume, error) { result := make([]*v1.PersistentVolume, 0) @@ -150,7 +179,7 @@ func (c *Cluster) resizeVolumes(newVolume acidv1.Volume, resizers []volumes.Volu c.logger.Debugf("successfully updated persistent volume %q", pv.Name) } if !compatible { - c.logger.Warningf("volume %q is incompatible with all available resizing providers", pv.Name) + c.logger.Warningf("volume %q is incompatible with all available resizing providers, consider switching storage_resize_mode to pvc or off", pv.Name) totalIncompatible++ } } @@ -160,6 +189,25 @@ func (c *Cluster) resizeVolumes(newVolume acidv1.Volume, resizers []volumes.Volu return nil } +func (c *Cluster) volumeClaimsNeedResizing(newVolume acidv1.Volume) (bool, error) { + newSize, err := resource.ParseQuantity(newVolume.Size) + manifestSize := quantityToGigabyte(newSize) + if err != nil { + return false, fmt.Errorf("could not parse volume size from the manifest: %v", err) + } + pvcs, err := c.listPersistentVolumeClaims() + if err != nil { + return false, fmt.Errorf("could not receive persistent volume claims: %v", err) + } + for _, pvc := range pvcs { + currentSize := quantityToGigabyte(pvc.Spec.Resources.Requests[v1.ResourceStorage]) + if currentSize != manifestSize { + return true, nil + } + } + return false, nil +} + func (c *Cluster) volumesNeedResizing(newVolume acidv1.Volume) (bool, error) { vols, manifestSize, err := c.listVolumesWithManifestSize(newVolume) if err != nil { diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index bfb0e6dcc..a5a91dba7 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -65,6 +65,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.WatchedNamespace = fromCRD.Kubernetes.WatchedNamespace result.PDBNameFormat = fromCRD.Kubernetes.PDBNameFormat result.EnablePodDisruptionBudget = util.CoalesceBool(fromCRD.Kubernetes.EnablePodDisruptionBudget, util.True()) + result.StorageResizeMode = util.Coalesce(fromCRD.Kubernetes.StorageResizeMode, "ebs") result.EnableInitContainers = util.CoalesceBool(fromCRD.Kubernetes.EnableInitContainers, util.True()) result.EnableSidecars = util.CoalesceBool(fromCRD.Kubernetes.EnableSidecars, util.True()) result.SecretNameTemplate = fromCRD.Kubernetes.SecretNameTemplate diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 01057f236..bf1f5b70a 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -142,6 +142,7 @@ type Config struct { CustomPodAnnotations map[string]string `name:"custom_pod_annotations"` EnablePodAntiAffinity bool `name:"enable_pod_antiaffinity" default:"false"` PodAntiAffinityTopologyKey string `name:"pod_antiaffinity_topology_key" default:"kubernetes.io/hostname"` + StorageResizeMode string `name:"storage_resize_mode" default:"ebs"` // deprecated and kept for backward compatibility EnableLoadBalancer *bool `name:"enable_load_balancer"` MasterDNSNameFormat StringTemplate `name:"master_dns_name_format" default:"{cluster}.{team}.{hostedzone}"` From c10d30903e049bc75ce29e0a9342ff45434deeb5 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 8 Jul 2020 11:56:58 +0200 Subject: [PATCH 02/35] bump pgBouncer image (#1050) Co-authored-by: Felix Kunde --- charts/postgres-operator/values-crd.yaml | 2 +- charts/postgres-operator/values.yaml | 2 +- manifests/configmap.yaml | 2 +- manifests/postgresql-operator-default-configuration.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index db26e6d98..2652d02e1 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -273,7 +273,7 @@ configConnectionPooler: # db user for pooler to use connection_pooler_user: "pooler" # docker image - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-8" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-9" # max db connections the pooler should hold connection_pooler_max_db_connections: 60 # default pooling mode diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index eb5c10b0e..7e83a32fa 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -265,7 +265,7 @@ configConnectionPooler: # db user for pooler to use connection_pooler_user: "pooler" # docker image - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-8" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-9" # max db connections the pooler should hold connection_pooler_max_db_connections: "60" # default pooling mode diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index d666b0383..2af4c8f8b 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -15,7 +15,7 @@ data: # connection_pooler_default_cpu_request: "500m" # connection_pooler_default_memory_limit: 100Mi # connection_pooler_default_memory_request: 100Mi - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-8" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-9" # connection_pooler_max_db_connections: 60 # connection_pooler_mode: "transaction" # connection_pooler_number_of_instances: 2 diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index cb7b1ed11..2cd71fff3 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -129,7 +129,7 @@ configuration: connection_pooler_default_cpu_request: "500m" connection_pooler_default_memory_limit: 100Mi connection_pooler_default_memory_request: 100Mi - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-8" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-9" # connection_pooler_max_db_connections: 60 connection_pooler_mode: "transaction" connection_pooler_number_of_instances: 2 From b80f9767d1982bd863172f7cbf5bd2786f8b920e Mon Sep 17 00:00:00 2001 From: Igor Yanchenko <1504692+yanchenko-igor@users.noreply.github.com> Date: Fri, 10 Jul 2020 10:07:25 +0300 Subject: [PATCH 03/35] test coverage (#1055) --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 091c875ba..a52769c91 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,5 +18,6 @@ install: script: - hack/verify-codegen.sh - - travis_wait 20 goveralls -service=travis-ci -package ./pkg/... -v + - travis_wait 20 go test -race -covermode atomic -coverprofile=profile.cov ./pkg/... -v + - goveralls -coverprofile=profile.cov -service=travis-ci -v - make e2e From 375963424d1e8b78f6fed2cacfce6b2e27656b3b Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 10 Jul 2020 15:07:42 +0200 Subject: [PATCH 04/35] delete secrets the right way (#1054) * delete secrets the right way * make a one function * continue deleting secrets even if one delete fails Co-authored-by: Felix Kunde --- pkg/cluster/cluster.go | 6 ++---- pkg/cluster/resources.go | 27 ++++++++++++++++++--------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 44c3e9b62..ef728a728 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -797,10 +797,8 @@ func (c *Cluster) Delete() { c.logger.Warningf("could not delete statefulset: %v", err) } - for _, obj := range c.Secrets { - if err := c.deleteSecret(obj); err != nil { - c.logger.Warningf("could not delete secret: %v", err) - } + if err := c.deleteSecrets(); err != nil { + c.logger.Warningf("could not delete secrets: %v", err) } if err := c.deletePodDisruptionBudget(); err != nil { diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index 5c35058c2..c75457a5a 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -725,17 +725,26 @@ func (c *Cluster) deleteEndpoint(role PostgresRole) error { return nil } -func (c *Cluster) deleteSecret(secret *v1.Secret) error { - c.setProcessName("deleting secret %q", util.NameFromMeta(secret.ObjectMeta)) - c.logger.Debugf("deleting secret %q", util.NameFromMeta(secret.ObjectMeta)) - err := c.KubeClient.Secrets(secret.Namespace).Delete(context.TODO(), secret.Name, c.deleteOptions) - if err != nil { - return err +func (c *Cluster) deleteSecrets() error { + c.setProcessName("deleting secrets") + var errors []string + errorCount := 0 + for uid, secret := range c.Secrets { + c.logger.Debugf("deleting secret %q", util.NameFromMeta(secret.ObjectMeta)) + err := c.KubeClient.Secrets(secret.Namespace).Delete(context.TODO(), secret.Name, c.deleteOptions) + if err != nil { + errors = append(errors, fmt.Sprintf("could not delete secret %q: %v", util.NameFromMeta(secret.ObjectMeta), err)) + errorCount++ + } + c.logger.Infof("secret %q has been deleted", util.NameFromMeta(secret.ObjectMeta)) + c.Secrets[uid] = nil } - c.logger.Infof("secret %q has been deleted", util.NameFromMeta(secret.ObjectMeta)) - delete(c.Secrets, secret.UID) - return err + if errorCount > 0 { + return fmt.Errorf("could not delete all secrets: %v", errors) + } + + return nil } func (c *Cluster) createRoles() (err error) { From ec932f88d826861d8b7d2512a92e55ffdb29bb13 Mon Sep 17 00:00:00 2001 From: Toon Sevrin Date: Wed, 15 Jul 2020 13:53:10 +0200 Subject: [PATCH 05/35] Port-forward service instead of pod (#1040) --- docs/quickstart.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/quickstart.md b/docs/quickstart.md index d2c88b9a4..034d32e39 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -160,7 +160,7 @@ You can now access the web interface by port forwarding the UI pod (mind the label selector) and enter `localhost:8081` in your browser: ```bash -kubectl port-forward "$(kubectl get pod -l name=postgres-operator-ui --output='name')" 8081 +kubectl port-forward svc/postgres-operator-ui 8081:8081 ``` Available option are explained in detail in the [UI docs](operator-ui.md). From 002b47ec3248685080762db0c25ee314bf50c060 Mon Sep 17 00:00:00 2001 From: Igor Yanchenko <1504692+yanchenko-igor@users.noreply.github.com> Date: Thu, 16 Jul 2020 15:43:57 +0300 Subject: [PATCH 06/35] Use scram-sha-256 hash if postgresql parameter password_encryption set to do so. (#995) * Use scram-sha-256 hash if postgresql parameter password_encryption set to do so. * test fixed * Refactoring * code style --- go.mod | 1 + pkg/cluster/cluster.go | 6 +++- pkg/util/users/users.go | 11 ++++---- pkg/util/util.go | 61 ++++++++++++++++++++++++++++++++++++++--- pkg/util/util_test.go | 28 ++++++++++++++----- 5 files changed, 90 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index 041f90706..6e8cd8ef4 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/sirupsen/logrus v1.6.0 github.com/stretchr/testify v1.5.1 golang.org/x/mod v0.3.0 // indirect + golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 golang.org/x/tools v0.0.0-20200615222825-6aa8f57aacd9 // indirect gopkg.in/yaml.v2 v2.2.8 k8s.io/api v0.18.3 diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index ef728a728..a88cde53e 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -124,6 +124,10 @@ func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgres return fmt.Sprintf("%s-%s", e.PodName, e.ResourceVersion), nil }) + password_encryption, ok := pgSpec.Spec.PostgresqlParam.Parameters["password_encryption"] + if !ok { + password_encryption = "md5" + } cluster := &Cluster{ Config: cfg, @@ -135,7 +139,7 @@ func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgres Secrets: make(map[types.UID]*v1.Secret), Services: make(map[PostgresRole]*v1.Service), Endpoints: make(map[PostgresRole]*v1.Endpoints)}, - userSyncStrategy: users.DefaultUserSyncStrategy{}, + userSyncStrategy: users.DefaultUserSyncStrategy{password_encryption}, deleteOptions: metav1.DeleteOptions{PropagationPolicy: &deletePropagationPolicy}, podEventsQueue: podEventsQueue, KubeClient: kubeClient, diff --git a/pkg/util/users/users.go b/pkg/util/users/users.go index 345caa001..166e90264 100644 --- a/pkg/util/users/users.go +++ b/pkg/util/users/users.go @@ -28,6 +28,7 @@ const ( // an existing roles of another role membership, nor it removes the already assigned flag // (except for the NOLOGIN). TODO: process other NOflags, i.e. NOSUPERUSER correctly. type DefaultUserSyncStrategy struct { + PasswordEncryption string } // ProduceSyncRequests figures out the types of changes that need to happen with the given users. @@ -45,7 +46,7 @@ func (strategy DefaultUserSyncStrategy) ProduceSyncRequests(dbUsers spec.PgUserM } } else { r := spec.PgSyncUserRequest{} - newMD5Password := util.PGUserPassword(newUser) + newMD5Password := util.NewEncryptor(strategy.PasswordEncryption).PGUserPassword(newUser) if dbUser.Password != newMD5Password { r.User.Password = newMD5Password @@ -140,7 +141,7 @@ func (strategy DefaultUserSyncStrategy) createPgUser(user spec.PgUser, db *sql.D if user.Password == "" { userPassword = "PASSWORD NULL" } else { - userPassword = fmt.Sprintf(passwordTemplate, util.PGUserPassword(user)) + userPassword = fmt.Sprintf(passwordTemplate, util.NewEncryptor(strategy.PasswordEncryption).PGUserPassword(user)) } query := fmt.Sprintf(createUserSQL, user.Name, strings.Join(userFlags, " "), userPassword) @@ -155,7 +156,7 @@ func (strategy DefaultUserSyncStrategy) alterPgUser(user spec.PgUser, db *sql.DB var resultStmt []string if user.Password != "" || len(user.Flags) > 0 { - alterStmt := produceAlterStmt(user) + alterStmt := produceAlterStmt(user, strategy.PasswordEncryption) resultStmt = append(resultStmt, alterStmt) } if len(user.MemberOf) > 0 { @@ -174,14 +175,14 @@ func (strategy DefaultUserSyncStrategy) alterPgUser(user spec.PgUser, db *sql.DB return nil } -func produceAlterStmt(user spec.PgUser) string { +func produceAlterStmt(user spec.PgUser, encryption string) string { // ALTER ROLE ... LOGIN ENCRYPTED PASSWORD .. result := make([]string, 0) password := user.Password flags := user.Flags if password != "" { - result = append(result, fmt.Sprintf(passwordTemplate, util.PGUserPassword(user))) + result = append(result, fmt.Sprintf(passwordTemplate, util.NewEncryptor(encryption).PGUserPassword(user))) } if len(flags) != 0 { result = append(result, strings.Join(flags, " ")) diff --git a/pkg/util/util.go b/pkg/util/util.go index ff1be4e68..abb9be01f 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -1,8 +1,11 @@ package util import ( + "crypto/hmac" "crypto/md5" // #nosec we need it to for PostgreSQL md5 passwords cryptoRand "crypto/rand" + "crypto/sha256" + "encoding/base64" "encoding/hex" "fmt" "math/big" @@ -16,10 +19,14 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/zalando/postgres-operator/pkg/spec" + "golang.org/x/crypto/pbkdf2" ) const ( - md5prefix = "md5" + md5prefix = "md5" + scramsha256prefix = "SCRAM-SHA-256" + saltlength = 16 + iterations = 4096 ) var passwordChars = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") @@ -61,16 +68,62 @@ func NameFromMeta(meta metav1.ObjectMeta) spec.NamespacedName { } } -// PGUserPassword is used to generate md5 password hash for a given user. It does nothing for already hashed passwords. -func PGUserPassword(user spec.PgUser) string { - if (len(user.Password) == md5.Size*2+len(md5prefix) && user.Password[:3] == md5prefix) || user.Password == "" { +type Hasher func(user spec.PgUser) string +type Random func(n int) string + +type Encryptor struct { + encrypt Hasher + random Random +} + +func NewEncryptor(encryption string) *Encryptor { + e := Encryptor{random: RandomPassword} + m := map[string]Hasher{ + "md5": e.PGUserPasswordMD5, + "scram-sha-256": e.PGUserPasswordScramSHA256, + } + hasher, ok := m[encryption] + if !ok { + hasher = e.PGUserPasswordMD5 + } + e.encrypt = hasher + return &e +} + +func (e *Encryptor) PGUserPassword(user spec.PgUser) string { + if (len(user.Password) == md5.Size*2+len(md5prefix) && user.Password[:3] == md5prefix) || + (len(user.Password) > len(scramsha256prefix) && user.Password[:len(scramsha256prefix)] == scramsha256prefix) || user.Password == "" { // Avoid processing already encrypted or empty passwords return user.Password } + return e.encrypt(user) +} + +func (e *Encryptor) PGUserPasswordMD5(user spec.PgUser) string { s := md5.Sum([]byte(user.Password + user.Name)) // #nosec, using md5 since PostgreSQL uses it for hashing passwords. return md5prefix + hex.EncodeToString(s[:]) } +func (e *Encryptor) PGUserPasswordScramSHA256(user spec.PgUser) string { + salt := []byte(e.random(saltlength)) + key := pbkdf2.Key([]byte(user.Password), salt, iterations, 32, sha256.New) + mac := hmac.New(sha256.New, key) + mac.Write([]byte("Server Key")) + serverKey := mac.Sum(nil) + mac = hmac.New(sha256.New, key) + mac.Write([]byte("Client Key")) + clientKey := mac.Sum(nil) + storedKey := sha256.Sum256(clientKey) + pass := fmt.Sprintf("%s$%v:%s$%s:%s", + scramsha256prefix, + iterations, + base64.StdEncoding.EncodeToString(salt), + base64.StdEncoding.EncodeToString(storedKey[:]), + base64.StdEncoding.EncodeToString(serverKey), + ) + return pass +} + // Diff returns diffs between 2 objects func Diff(a, b interface{}) []string { return pretty.Diff(a, b) diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index 1f86ea1b4..a9d25112b 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -12,20 +12,27 @@ import ( ) var pgUsers = []struct { - in spec.PgUser - out string + in spec.PgUser + outmd5 string + outscramsha256 string }{{spec.PgUser{ Name: "test", Password: "password", Flags: []string{}, MemberOf: []string{}}, - "md587f77988ccb5aa917c93201ba314fcd4"}, + "md587f77988ccb5aa917c93201ba314fcd4", "SCRAM-SHA-256$4096:c2FsdA==$lF4cRm/Jky763CN4HtxdHnjV4Q8AWTNlKvGmEFFU8IQ=:ub8OgRsftnk2ccDMOt7ffHXNcikRkQkq1lh4xaAqrSw="}, {spec.PgUser{ Name: "test", Password: "md592f413f3974bdf3799bb6fecb5f9f2c6", Flags: []string{}, MemberOf: []string{}}, - "md592f413f3974bdf3799bb6fecb5f9f2c6"}} + "md592f413f3974bdf3799bb6fecb5f9f2c6", "md592f413f3974bdf3799bb6fecb5f9f2c6"}, + {spec.PgUser{ + Name: "test", + Password: "SCRAM-SHA-256$4096:S1ByZWhvYVV5VDlJNGZoVw==$ozLevu5k0pAQYRrSY+vZhetO6+/oB+qZvuutOdXR94U=:yADwhy0LGloXzh5RaVwLMFyUokwI17VkHVfKVuHu0Zs=", + Flags: []string{}, + MemberOf: []string{}}, + "SCRAM-SHA-256$4096:S1ByZWhvYVV5VDlJNGZoVw==$ozLevu5k0pAQYRrSY+vZhetO6+/oB+qZvuutOdXR94U=:yADwhy0LGloXzh5RaVwLMFyUokwI17VkHVfKVuHu0Zs=", "SCRAM-SHA-256$4096:S1ByZWhvYVV5VDlJNGZoVw==$ozLevu5k0pAQYRrSY+vZhetO6+/oB+qZvuutOdXR94U=:yADwhy0LGloXzh5RaVwLMFyUokwI17VkHVfKVuHu0Zs="}} var prettyDiffTest = []struct { inA interface{} @@ -107,9 +114,16 @@ func TestNameFromMeta(t *testing.T) { func TestPGUserPassword(t *testing.T) { for _, tt := range pgUsers { - pwd := PGUserPassword(tt.in) - if pwd != tt.out { - t.Errorf("PgUserPassword expected: %q, got: %q", tt.out, pwd) + e := NewEncryptor("md5") + pwd := e.PGUserPassword(tt.in) + if pwd != tt.outmd5 { + t.Errorf("PgUserPassword expected: %q, got: %q", tt.outmd5, pwd) + } + e = NewEncryptor("scram-sha-256") + e.random = func(n int) string { return "salt" } + pwd = e.PGUserPassword(tt.in) + if pwd != tt.outscramsha256 { + t.Errorf("PgUserPassword expected: %q, got: %q", tt.outscramsha256, pwd) } } } From 102a3536497cf9deae466dce7c02c3e6bb4569a3 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 29 Jul 2020 15:57:55 +0200 Subject: [PATCH 07/35] update dependencies (#1080) --- go.mod | 15 +++++++-------- go.sum | 42 +++++++++++++++++++++++------------------- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index 6e8cd8ef4..49ba3682b 100644 --- a/go.mod +++ b/go.mod @@ -9,13 +9,12 @@ require ( github.com/r3labs/diff v1.1.0 github.com/sirupsen/logrus v1.6.0 github.com/stretchr/testify v1.5.1 - golang.org/x/mod v0.3.0 // indirect - golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 - golang.org/x/tools v0.0.0-20200615222825-6aa8f57aacd9 // indirect + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 + golang.org/x/tools v0.0.0-20200729041821-df70183b1872 // indirect gopkg.in/yaml.v2 v2.2.8 - k8s.io/api v0.18.3 - k8s.io/apiextensions-apiserver v0.18.3 - k8s.io/apimachinery v0.18.3 - k8s.io/client-go v0.18.3 - k8s.io/code-generator v0.18.3 + k8s.io/api v0.18.6 + k8s.io/apiextensions-apiserver v0.18.6 + k8s.io/apimachinery v0.18.6 + k8s.io/client-go v0.18.6 + k8s.io/code-generator v0.18.6 ) diff --git a/go.sum b/go.sum index b3d154b98..389608b82 100644 --- a/go.sum +++ b/go.sum @@ -291,7 +291,7 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -312,13 +312,13 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 h1:/Tl7pH94bvbAAHBdZJT947M/+gp0+CqQXDtMRC0fseo= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -339,8 +339,8 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -351,6 +351,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -368,6 +369,8 @@ golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hqSUlvZU0rab6x5EXfGU= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -392,8 +395,8 @@ golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200615222825-6aa8f57aacd9 h1:cwgUY+1ja2qxWb2dyaCoixaA66WGWmrijSlxaM+JM/g= -golang.org/x/tools v0.0.0-20200615222825-6aa8f57aacd9/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729041821-df70183b1872 h1:/U95VAvB4ZsR91rpZX2MwiKpejhWr+UxJ+N2VlJuESk= +golang.org/x/tools v0.0.0-20200729041821-df70183b1872/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -435,19 +438,20 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.18.3 h1:2AJaUQdgUZLoDZHrun21PW2Nx9+ll6cUzvn3IKhSIn0= -k8s.io/api v0.18.3/go.mod h1:UOaMwERbqJMfeeeHc8XJKawj4P9TgDRnViIqqBeH2QA= -k8s.io/apiextensions-apiserver v0.18.3 h1:h6oZO+iAgg0HjxmuNnguNdKNB9+wv3O1EBDdDWJViQ0= -k8s.io/apiextensions-apiserver v0.18.3/go.mod h1:TMsNGs7DYpMXd+8MOCX8KzPOCx8fnZMoIGB24m03+JE= -k8s.io/apimachinery v0.18.3 h1:pOGcbVAhxADgUYnjS08EFXs9QMl8qaH5U4fr5LGUrSk= -k8s.io/apimachinery v0.18.3/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= -k8s.io/apiserver v0.18.3/go.mod h1:tHQRmthRPLUtwqsOnJJMoI8SW3lnoReZeE861lH8vUw= -k8s.io/client-go v0.18.3 h1:QaJzz92tsN67oorwzmoB0a9r9ZVHuD5ryjbCKP0U22k= -k8s.io/client-go v0.18.3/go.mod h1:4a/dpQEvzAhT1BbuWW09qvIaGw6Gbu1gZYiQZIi1DMw= -k8s.io/code-generator v0.18.3 h1:5H57pYEbkMMXCLKD16YQH3yDPAbVLweUsB1M3m70D1c= -k8s.io/code-generator v0.18.3/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= -k8s.io/component-base v0.18.3/go.mod h1:bp5GzGR0aGkYEfTj+eTY0AN/vXTgkJdQXjNTTVUaa3k= +k8s.io/api v0.18.6 h1:osqrAXbOQjkKIWDTjrqxWQ3w0GkKb1KA1XkUGHHYpeE= +k8s.io/api v0.18.6/go.mod h1:eeyxr+cwCjMdLAmr2W3RyDI0VvTawSg/3RFFBEnmZGI= +k8s.io/apiextensions-apiserver v0.18.6 h1:vDlk7cyFsDyfwn2rNAO2DbmUbvXy5yT5GE3rrqOzaMo= +k8s.io/apiextensions-apiserver v0.18.6/go.mod h1:lv89S7fUysXjLZO7ke783xOwVTm6lKizADfvUM/SS/M= +k8s.io/apimachinery v0.18.6 h1:RtFHnfGNfd1N0LeSrKCUznz5xtUP1elRGvHJbL3Ntag= +k8s.io/apimachinery v0.18.6/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= +k8s.io/apiserver v0.18.6/go.mod h1:Zt2XvTHuaZjBz6EFYzpp+X4hTmgWGy8AthNVnTdm3Wg= +k8s.io/client-go v0.18.6 h1:I+oWqJbibLSGsZj8Xs8F0aWVXJVIoUHWaaJV3kUN/Zw= +k8s.io/client-go v0.18.6/go.mod h1:/fwtGLjYMS1MaM5oi+eXhKwG+1UHidUEXRh6cNsdO0Q= +k8s.io/code-generator v0.18.6 h1:QdfvGfs4gUCS1dru+rLbCKIFxYEV0IRfF8MXwY/ozLk= +k8s.io/code-generator v0.18.6/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= +k8s.io/component-base v0.18.6/go.mod h1:knSVsibPR5K6EW2XOjEHik6sdU5nCvKMrzMt2D4In14= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200114144118-36b2048a9120 h1:RPscN6KhmG54S33L+lr3GS+oD1jmchIU0ll519K6FA4= k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= From ece341d5160d30cb8e70ce521a615b74c8d7c4d9 Mon Sep 17 00:00:00 2001 From: Christian Rohmann Date: Thu, 30 Jul 2020 10:48:16 +0200 Subject: [PATCH 08/35] Allow pod environment variables to also be sourced from a secret (#946) * Extend operator configuration to allow for a pod_environment_secret just like pod_environment_configmap * Add all keys from PodEnvironmentSecrets as ENV vars (using SecretKeyRef to protect the value) * Apply envVars from pod_environment_configmap and pod_environment_secrets before doing the global settings from the operator config. This allows them to be overriden by the user (via configmap / secret) * Add ability use a Secret for custom pod envVars (via pod_environment_secret) to admin documentation * Add pod_environment_secret to Helm chart values.yaml * Add unit tests for PodEnvironmentConfigMap and PodEnvironmentSecret - highly inspired by @kupson and his very similar PR #481 * Added new parameter pod_environment_secret to operatorconfig CRD and configmap examples * Add pod_environment_secret to the operationconfiguration CRD Co-authored-by: Christian Rohmann --- .../crds/operatorconfigurations.yaml | 2 + charts/postgres-operator/values-crd.yaml | 2 + charts/postgres-operator/values.yaml | 2 + docs/administrator.md | 64 +++++- manifests/configmap.yaml | 1 + manifests/operatorconfiguration.crd.yaml | 2 + ...gresql-operator-default-configuration.yaml | 1 + pkg/apis/acid.zalan.do/v1/crds.go | 3 + .../v1/operator_configuration_type.go | 1 + pkg/cluster/k8sres.go | 159 ++++++++----- pkg/cluster/k8sres_test.go | 208 ++++++++++++++++++ pkg/controller/operator_config.go | 1 + pkg/util/config/config.go | 1 + 13 files changed, 393 insertions(+), 54 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index ffcef7b4a..89f495367 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -149,6 +149,8 @@ spec: type: string pod_environment_configmap: type: string + pod_environment_secret: + type: string pod_management_policy: type: string enum: diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 2652d02e1..44a7f315b 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -104,6 +104,8 @@ configKubernetes: pod_antiaffinity_topology_key: "kubernetes.io/hostname" # namespaced name of the ConfigMap with environment variables to populate on every pod # pod_environment_configmap: "default/my-custom-config" + # name of the Secret (in cluster namespace) with environment variables to populate on every pod + # pod_environment_secret: "my-custom-secret" # specify the pod management policy of stateful sets of Postgres clusters pod_management_policy: "ordered_ready" diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 7e83a32fa..b64495bee 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -95,6 +95,8 @@ configKubernetes: pod_antiaffinity_topology_key: "kubernetes.io/hostname" # namespaced name of the ConfigMap with environment variables to populate on every pod # pod_environment_configmap: "default/my-custom-config" + # name of the Secret (in cluster namespace) with environment variables to populate on every pod + # pod_environment_secret: "my-custom-secret" # specify the pod management policy of stateful sets of Postgres clusters pod_management_policy: "ordered_ready" diff --git a/docs/administrator.md b/docs/administrator.md index e2c2e01eb..b3d4d9efa 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -319,11 +319,18 @@ spec: ## Custom Pod Environment Variables - -It is possible to configure a ConfigMap which is used by the Postgres pods as +It is possible to configure a ConfigMap as well as a Secret which are used by the Postgres pods as an additional provider for environment variables. One use case is to customize -the Spilo image and configure it with environment variables. The ConfigMap with -the additional settings is referenced in the operator's main configuration. +the Spilo image and configure it with environment variables. Another case could be to provide custom +cloud provider or backup settings. + +In general the Operator will give preference to the globally configured variables, to not have the custom +ones interfere with core functionality. Variables with the 'WAL_' and 'LOG_' prefix can be overwritten though, to allow +backup and logshipping to be specified differently. + + +### Via ConfigMap +The ConfigMap with the additional settings is referenced in the operator's main configuration. A namespace can be specified along with the name. If left out, the configured default namespace of your K8s client will be used and if the ConfigMap is not found there, the Postgres cluster's namespace is taken when different: @@ -365,7 +372,54 @@ data: MY_CUSTOM_VAR: value ``` -This ConfigMap is then added as a source of environment variables to the +The key-value pairs of the ConfigMap are then added as environment variables to the +Postgres StatefulSet/pods. + + +### Via Secret +The Secret with the additional variables is referenced in the operator's main configuration. +To protect the values of the secret from being exposed in the pod spec they are each referenced +as SecretKeyRef. +This does not allow for the secret to be in a different namespace as the pods though + +**postgres-operator ConfigMap** + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-operator +data: + # referencing secret with custom environment variables + pod_environment_secret: postgres-pod-secrets +``` + +**OperatorConfiguration** + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: OperatorConfiguration +metadata: + name: postgresql-operator-configuration +configuration: + kubernetes: + # referencing secret with custom environment variables + pod_environment_secret: postgres-pod-secrets +``` + +**referenced Secret `postgres-pod-secrets`** + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: postgres-pod-secrets + namespace: default +data: + MY_CUSTOM_VAR: dmFsdWU= +``` + +The key-value pairs of the Secret are all accessible as environment variables to the Postgres StatefulSet/pods. ## Limiting the number of min and max instances in clusters diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 2af4c8f8b..d1c1b3d17 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -74,6 +74,7 @@ data: # pod_antiaffinity_topology_key: "kubernetes.io/hostname" pod_deletion_wait_timeout: 10m # pod_environment_configmap: "default/my-custom-config" + # pod_environment_secret: "my-custom-secret" pod_label_wait_timeout: 10m pod_management_policy: "ordered_ready" pod_role_label: spilo-role diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 346eabb4a..2b6e8ae67 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -145,6 +145,8 @@ spec: type: string pod_environment_configmap: type: string + pod_environment_secret: + type: string pod_management_policy: type: string enum: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 2cd71fff3..f7eba1f6c 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -49,6 +49,7 @@ configuration: pdb_name_format: "postgres-{cluster}-pdb" pod_antiaffinity_topology_key: "kubernetes.io/hostname" # pod_environment_configmap: "default/my-custom-config" + # pod_environment_secret: "my-custom-secret" pod_management_policy: "ordered_ready" # pod_priority_class_name: "" pod_role_label: spilo-role diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index bc38d6dfd..6f907e266 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -942,6 +942,9 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "pod_environment_configmap": { Type: "string", }, + "pod_environment_secret": { + Type: "string", + }, "pod_management_policy": { Type: "string", Enum: []apiextv1beta1.JSON{ diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 5ac5a4677..e6e13cbd3 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -70,6 +70,7 @@ type KubernetesMetaConfiguration struct { // TODO: use a proper toleration structure? PodToleration map[string]string `json:"toleration,omitempty"` PodEnvironmentConfigMap spec.NamespacedName `json:"pod_environment_configmap,omitempty"` + PodEnvironmentSecret string `json:"pod_environment_secret,omitempty"` PodPriorityClassName string `json:"pod_priority_class_name,omitempty"` MasterPodMoveTimeout Duration `json:"master_pod_move_timeout,omitempty"` EnablePodAntiAffinity bool `json:"enable_pod_antiaffinity,omitempty"` diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index ef20da062..21875f953 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -7,6 +7,7 @@ import ( "path" "sort" "strconv" + "strings" "github.com/sirupsen/logrus" @@ -20,7 +21,6 @@ import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/spec" - pkgspec "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" @@ -715,28 +715,6 @@ func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration stri envVars = append(envVars, v1.EnvVar{Name: "SPILO_CONFIGURATION", Value: spiloConfiguration}) } - if c.OpConfig.WALES3Bucket != "" { - envVars = append(envVars, v1.EnvVar{Name: "WAL_S3_BUCKET", Value: c.OpConfig.WALES3Bucket}) - envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) - envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""}) - } - - if c.OpConfig.WALGSBucket != "" { - envVars = append(envVars, v1.EnvVar{Name: "WAL_GS_BUCKET", Value: c.OpConfig.WALGSBucket}) - envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) - envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""}) - } - - if c.OpConfig.GCPCredentials != "" { - envVars = append(envVars, v1.EnvVar{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: c.OpConfig.GCPCredentials}) - } - - if c.OpConfig.LogS3Bucket != "" { - envVars = append(envVars, v1.EnvVar{Name: "LOG_S3_BUCKET", Value: c.OpConfig.LogS3Bucket}) - envVars = append(envVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) - envVars = append(envVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_PREFIX", Value: ""}) - } - if c.patroniUsesKubernetes() { envVars = append(envVars, v1.EnvVar{Name: "DCS_ENABLE_KUBERNETES_API", Value: "true"}) } else { @@ -755,10 +733,34 @@ func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration stri envVars = append(envVars, c.generateStandbyEnvironment(standbyDescription)...) } + // add vars taken from pod_environment_configmap and pod_environment_secret first + // (to allow them to override the globals set in the operator config) if len(customPodEnvVarsList) > 0 { envVars = append(envVars, customPodEnvVarsList...) } + if c.OpConfig.WALES3Bucket != "" { + envVars = append(envVars, v1.EnvVar{Name: "WAL_S3_BUCKET", Value: c.OpConfig.WALES3Bucket}) + envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) + envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""}) + } + + if c.OpConfig.WALGSBucket != "" { + envVars = append(envVars, v1.EnvVar{Name: "WAL_GS_BUCKET", Value: c.OpConfig.WALGSBucket}) + envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) + envVars = append(envVars, v1.EnvVar{Name: "WAL_BUCKET_SCOPE_PREFIX", Value: ""}) + } + + if c.OpConfig.GCPCredentials != "" { + envVars = append(envVars, v1.EnvVar{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: c.OpConfig.GCPCredentials}) + } + + if c.OpConfig.LogS3Bucket != "" { + envVars = append(envVars, v1.EnvVar{Name: "LOG_S3_BUCKET", Value: c.OpConfig.LogS3Bucket}) + envVars = append(envVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_SUFFIX", Value: getBucketScopeSuffix(string(uid))}) + envVars = append(envVars, v1.EnvVar{Name: "LOG_BUCKET_SCOPE_PREFIX", Value: ""}) + } + return envVars } @@ -777,13 +779,81 @@ func deduplicateEnvVars(input []v1.EnvVar, containerName string, logger *logrus. result = append(result, input[i]) } else if names[va.Name] == 1 { names[va.Name]++ - logger.Warningf("variable %q is defined in %q more than once, the subsequent definitions are ignored", - va.Name, containerName) + + // Some variables (those to configure the WAL_ and LOG_ shipping) may be overriden, only log as info + if strings.HasPrefix(va.Name, "WAL_") || strings.HasPrefix(va.Name, "LOG_") { + logger.Infof("global variable %q has been overwritten by configmap/secret for container %q", + va.Name, containerName) + } else { + logger.Warningf("variable %q is defined in %q more than once, the subsequent definitions are ignored", + va.Name, containerName) + } } } return result } +// Return list of variables the pod recieved from the configured ConfigMap +func (c *Cluster) getPodEnvironmentConfigMapVariables() ([]v1.EnvVar, error) { + configMapPodEnvVarsList := make([]v1.EnvVar, 0) + + if c.OpConfig.PodEnvironmentConfigMap.Name == "" { + return configMapPodEnvVarsList, nil + } + + cm, err := c.KubeClient.ConfigMaps(c.OpConfig.PodEnvironmentConfigMap.Namespace).Get( + context.TODO(), + c.OpConfig.PodEnvironmentConfigMap.Name, + metav1.GetOptions{}) + if err != nil { + // if not found, try again using the cluster's namespace if it's different (old behavior) + if k8sutil.ResourceNotFound(err) && c.Namespace != c.OpConfig.PodEnvironmentConfigMap.Namespace { + cm, err = c.KubeClient.ConfigMaps(c.Namespace).Get( + context.TODO(), + c.OpConfig.PodEnvironmentConfigMap.Name, + metav1.GetOptions{}) + } + if err != nil { + return nil, fmt.Errorf("could not read PodEnvironmentConfigMap: %v", err) + } + } + for k, v := range cm.Data { + configMapPodEnvVarsList = append(configMapPodEnvVarsList, v1.EnvVar{Name: k, Value: v}) + } + return configMapPodEnvVarsList, nil +} + +// Return list of variables the pod recieved from the configured Secret +func (c *Cluster) getPodEnvironmentSecretVariables() ([]v1.EnvVar, error) { + secretPodEnvVarsList := make([]v1.EnvVar, 0) + + if c.OpConfig.PodEnvironmentSecret == "" { + return secretPodEnvVarsList, nil + } + + secret, err := c.KubeClient.Secrets(c.OpConfig.PodEnvironmentSecret).Get( + context.TODO(), + c.OpConfig.PodEnvironmentSecret, + metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("could not read Secret PodEnvironmentSecretName: %v", err) + } + + for k := range secret.Data { + secretPodEnvVarsList = append(secretPodEnvVarsList, + v1.EnvVar{Name: k, ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: c.OpConfig.PodEnvironmentSecret, + }, + Key: k, + }, + }}) + } + + return secretPodEnvVarsList, nil +} + func getSidecarContainer(sidecar acidv1.Sidecar, index int, resources *v1.ResourceRequirements) *v1.Container { name := sidecar.Name if name == "" { @@ -943,32 +1013,23 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef initContainers = spec.InitContainers } - customPodEnvVarsList := make([]v1.EnvVar, 0) + // fetch env vars from custom ConfigMap + configMapEnvVarsList, err := c.getPodEnvironmentConfigMapVariables() + if err != nil { + return nil, err + } - if c.OpConfig.PodEnvironmentConfigMap != (pkgspec.NamespacedName{}) { - var cm *v1.ConfigMap - cm, err = c.KubeClient.ConfigMaps(c.OpConfig.PodEnvironmentConfigMap.Namespace).Get( - context.TODO(), - c.OpConfig.PodEnvironmentConfigMap.Name, - metav1.GetOptions{}) - if err != nil { - // if not found, try again using the cluster's namespace if it's different (old behavior) - if k8sutil.ResourceNotFound(err) && c.Namespace != c.OpConfig.PodEnvironmentConfigMap.Namespace { - cm, err = c.KubeClient.ConfigMaps(c.Namespace).Get( - context.TODO(), - c.OpConfig.PodEnvironmentConfigMap.Name, - metav1.GetOptions{}) - } - if err != nil { - return nil, fmt.Errorf("could not read PodEnvironmentConfigMap: %v", err) - } - } - for k, v := range cm.Data { - customPodEnvVarsList = append(customPodEnvVarsList, v1.EnvVar{Name: k, Value: v}) - } - sort.Slice(customPodEnvVarsList, - func(i, j int) bool { return customPodEnvVarsList[i].Name < customPodEnvVarsList[j].Name }) + // fetch env vars from custom ConfigMap + secretEnvVarsList, err := c.getPodEnvironmentSecretVariables() + if err != nil { + return nil, err } + + // concat all custom pod env vars and sort them + customPodEnvVarsList := append(configMapEnvVarsList, secretEnvVarsList...) + sort.Slice(customPodEnvVarsList, + func(i, j int) bool { return customPodEnvVarsList[i].Name < customPodEnvVarsList[j].Name }) + if spec.StandbyCluster != nil && spec.StandbyCluster.S3WalPath == "" { return nil, fmt.Errorf("s3_wal_path is empty for standby cluster") } diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index ff830a1f5..f324a9bd3 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -1,6 +1,7 @@ package cluster import ( + "context" "errors" "fmt" "reflect" @@ -10,6 +11,7 @@ import ( "github.com/stretchr/testify/assert" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" @@ -22,6 +24,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + v1core "k8s.io/client-go/kubernetes/typed/core/v1" ) // For testing purposes @@ -713,6 +716,211 @@ func TestSecretVolume(t *testing.T) { } } +const ( + testPodEnvironmentConfigMapName = "pod_env_cm" + testPodEnvironmentSecretName = "pod_env_sc" +) + +type mockSecret struct { + v1core.SecretInterface +} + +type mockConfigMap struct { + v1core.ConfigMapInterface +} + +func (c *mockSecret) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.Secret, error) { + if name != testPodEnvironmentSecretName { + return nil, fmt.Errorf("Secret PodEnvironmentSecret not found") + } + secret := &v1.Secret{} + secret.Name = testPodEnvironmentSecretName + secret.Data = map[string][]byte{ + "minio_access_key": []byte("alpha"), + "minio_secret_key": []byte("beta"), + } + return secret, nil +} + +func (c *mockConfigMap) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.ConfigMap, error) { + if name != testPodEnvironmentConfigMapName { + return nil, fmt.Errorf("NotFound") + } + configmap := &v1.ConfigMap{} + configmap.Name = testPodEnvironmentConfigMapName + configmap.Data = map[string]string{ + "foo": "bar", + } + return configmap, nil +} + +type MockSecretGetter struct { +} + +type MockConfigMapsGetter struct { +} + +func (c *MockSecretGetter) Secrets(namespace string) v1core.SecretInterface { + return &mockSecret{} +} + +func (c *MockConfigMapsGetter) ConfigMaps(namespace string) v1core.ConfigMapInterface { + return &mockConfigMap{} +} + +func newMockKubernetesClient() k8sutil.KubernetesClient { + return k8sutil.KubernetesClient{ + SecretsGetter: &MockSecretGetter{}, + ConfigMapsGetter: &MockConfigMapsGetter{}, + } +} +func newMockCluster(opConfig config.Config) *Cluster { + cluster := &Cluster{ + Config: Config{OpConfig: opConfig}, + KubeClient: newMockKubernetesClient(), + } + return cluster +} + +func TestPodEnvironmentConfigMapVariables(t *testing.T) { + testName := "TestPodEnvironmentConfigMapVariables" + tests := []struct { + subTest string + opConfig config.Config + envVars []v1.EnvVar + err error + }{ + { + subTest: "no PodEnvironmentConfigMap", + envVars: []v1.EnvVar{}, + }, + { + subTest: "missing PodEnvironmentConfigMap", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentConfigMap: spec.NamespacedName{ + Name: "idonotexist", + }, + }, + }, + err: fmt.Errorf("could not read PodEnvironmentConfigMap: NotFound"), + }, + { + subTest: "simple PodEnvironmentConfigMap", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentConfigMap: spec.NamespacedName{ + Name: testPodEnvironmentConfigMapName, + }, + }, + }, + envVars: []v1.EnvVar{ + { + Name: "foo", + Value: "bar", + }, + }, + }, + } + for _, tt := range tests { + c := newMockCluster(tt.opConfig) + vars, err := c.getPodEnvironmentConfigMapVariables() + if !reflect.DeepEqual(vars, tt.envVars) { + t.Errorf("%s %s: expected `%v` but got `%v`", + testName, tt.subTest, tt.envVars, vars) + } + if tt.err != nil { + if err.Error() != tt.err.Error() { + t.Errorf("%s %s: expected error `%v` but got `%v`", + testName, tt.subTest, tt.err, err) + } + } else { + if err != nil { + t.Errorf("%s %s: expected no error but got error: `%v`", + testName, tt.subTest, err) + } + } + } +} + +// Test if the keys of an existing secret are properly referenced +func TestPodEnvironmentSecretVariables(t *testing.T) { + testName := "TestPodEnvironmentSecretVariables" + tests := []struct { + subTest string + opConfig config.Config + envVars []v1.EnvVar + err error + }{ + { + subTest: "No PodEnvironmentSecret configured", + envVars: []v1.EnvVar{}, + }, + { + subTest: "Secret referenced by PodEnvironmentSecret does not exist", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentSecret: "idonotexist", + }, + }, + err: fmt.Errorf("could not read Secret PodEnvironmentSecretName: Secret PodEnvironmentSecret not found"), + }, + { + subTest: "Pod environment vars reference all keys from secret configured by PodEnvironmentSecret", + opConfig: config.Config{ + Resources: config.Resources{ + PodEnvironmentSecret: testPodEnvironmentSecretName, + }, + }, + envVars: []v1.EnvVar{ + { + Name: "minio_access_key", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: testPodEnvironmentSecretName, + }, + Key: "minio_access_key", + }, + }, + }, + { + Name: "minio_secret_key", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: testPodEnvironmentSecretName, + }, + Key: "minio_secret_key", + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + c := newMockCluster(tt.opConfig) + vars, err := c.getPodEnvironmentSecretVariables() + if !reflect.DeepEqual(vars, tt.envVars) { + t.Errorf("%s %s: expected `%v` but got `%v`", + testName, tt.subTest, tt.envVars, vars) + } + if tt.err != nil { + if err.Error() != tt.err.Error() { + t.Errorf("%s %s: expected error `%v` but got `%v`", + testName, tt.subTest, tt.err, err) + } + } else { + if err != nil { + t.Errorf("%s %s: expected no error but got error: `%v`", + testName, tt.subTest, err) + } + } + } + +} + func testResources(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { cpuReq := podSpec.Spec.Containers[0].Resources.Requests["cpu"] if cpuReq.String() != cluster.OpConfig.ConnectionPooler.ConnectionPoolerDefaultCPURequest { diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index a5a91dba7..e2d8636a1 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -58,6 +58,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.PodServiceAccountDefinition = fromCRD.Kubernetes.PodServiceAccountDefinition result.PodServiceAccountRoleBindingDefinition = fromCRD.Kubernetes.PodServiceAccountRoleBindingDefinition result.PodEnvironmentConfigMap = fromCRD.Kubernetes.PodEnvironmentConfigMap + result.PodEnvironmentSecret = fromCRD.Kubernetes.PodEnvironmentSecret result.PodTerminateGracePeriod = util.CoalesceDuration(time.Duration(fromCRD.Kubernetes.PodTerminateGracePeriod), "5m") result.SpiloPrivileged = fromCRD.Kubernetes.SpiloPrivileged result.SpiloFSGroup = fromCRD.Kubernetes.SpiloFSGroup diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index bf1f5b70a..6cab8af45 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -45,6 +45,7 @@ type Resources struct { MinCPULimit string `name:"min_cpu_limit" default:"250m"` MinMemoryLimit string `name:"min_memory_limit" default:"250Mi"` PodEnvironmentConfigMap spec.NamespacedName `name:"pod_environment_configmap"` + PodEnvironmentSecret string `name:"pod_environment_secret"` NodeReadinessLabel map[string]string `name:"node_readiness_label" default:""` MaxInstances int32 `name:"max_instances" default:"-1"` MinInstances int32 `name:"min_instances" default:"-1"` From aab9b0aff9ac43d7559f51213ff504fc24835e66 Mon Sep 17 00:00:00 2001 From: Allison Richardet Date: Thu, 30 Jul 2020 04:08:33 -0500 Subject: [PATCH 09/35] chart ui: fix target namespace to allow '*' (#1082) --- charts/postgres-operator-ui/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/postgres-operator-ui/templates/deployment.yaml b/charts/postgres-operator-ui/templates/deployment.yaml index 00610c799..4c6d46689 100644 --- a/charts/postgres-operator-ui/templates/deployment.yaml +++ b/charts/postgres-operator-ui/templates/deployment.yaml @@ -46,7 +46,7 @@ spec: - name: "RESOURCES_VISIBLE" value: "{{ .Values.envs.resourcesVisible }}" - name: "TARGET_NAMESPACE" - value: {{ .Values.envs.targetNamespace }} + value: "{{ .Values.envs.targetNamespace }}" - name: "TEAMS" value: |- [ From 3bee590d439d6f697166441093e3ef2f1ba2ddcd Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 30 Jul 2020 13:35:37 +0200 Subject: [PATCH 10/35] fix index in TestGenerateSpiloPodEnvVarswq (#1084) Co-authored-by: Felix Kunde --- pkg/cluster/k8sres_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index f324a9bd3..7261d5902 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -119,17 +119,17 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) { expectedValuesGSBucket := []ExpectedValue{ ExpectedValue{ - envIndex: 14, + envIndex: 15, envVarConstant: "WAL_GS_BUCKET", envVarValue: "wale-gs-bucket", }, ExpectedValue{ - envIndex: 15, + envIndex: 16, envVarConstant: "WAL_BUCKET_SCOPE_SUFFIX", envVarValue: "/SomeUUID", }, ExpectedValue{ - envIndex: 16, + envIndex: 17, envVarConstant: "WAL_BUCKET_SCOPE_PREFIX", envVarValue: "", }, @@ -137,22 +137,22 @@ func TestGenerateSpiloPodEnvVars(t *testing.T) { expectedValuesGCPCreds := []ExpectedValue{ ExpectedValue{ - envIndex: 14, + envIndex: 15, envVarConstant: "WAL_GS_BUCKET", envVarValue: "wale-gs-bucket", }, ExpectedValue{ - envIndex: 15, + envIndex: 16, envVarConstant: "WAL_BUCKET_SCOPE_SUFFIX", envVarValue: "/SomeUUID", }, ExpectedValue{ - envIndex: 16, + envIndex: 17, envVarConstant: "WAL_BUCKET_SCOPE_PREFIX", envVarValue: "", }, ExpectedValue{ - envIndex: 17, + envIndex: 18, envVarConstant: "GOOGLE_APPLICATION_CREDENTIALS", envVarValue: "some_path_to_credentials", }, From 47b11f7f8993a3f4bd39d7aa63aacdd3f66774f3 Mon Sep 17 00:00:00 2001 From: hlihhovac Date: Thu, 30 Jul 2020 16:31:29 +0200 Subject: [PATCH 11/35] change Clone attribute of PostgresSpec to *CloneDescription (#1020) * change Clone attribute of PostgresSpec to *ConnectionPooler * update go.mod from master * fix TestConnectionPoolerSynchronization() * Update pkg/apis/acid.zalan.do/v1/postgresql_type.go Co-authored-by: Felix Kunde Co-authored-by: Pavlo Golub Co-authored-by: Felix Kunde --- .gitignore | 1 + pkg/apis/acid.zalan.do/v1/marshal.go | 5 +- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 2 +- pkg/apis/acid.zalan.do/v1/util.go | 2 +- pkg/apis/acid.zalan.do/v1/util_test.go | 12 ++-- .../acid.zalan.do/v1/zz_generated.deepcopy.go | 6 +- pkg/cluster/k8sres.go | 4 +- pkg/cluster/sync_test.go | 59 ++++++++++--------- 8 files changed, 50 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index 0fdb50756..559c92499 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ _testmain.go /docker/build/ /github.com/ .idea +.vscode scm-source.json diff --git a/pkg/apis/acid.zalan.do/v1/marshal.go b/pkg/apis/acid.zalan.do/v1/marshal.go index 336b0da41..9521082fc 100644 --- a/pkg/apis/acid.zalan.do/v1/marshal.go +++ b/pkg/apis/acid.zalan.do/v1/marshal.go @@ -112,8 +112,9 @@ func (p *Postgresql) UnmarshalJSON(data []byte) error { if clusterName, err := extractClusterName(tmp2.ObjectMeta.Name, tmp2.Spec.TeamID); err != nil { tmp2.Error = err.Error() - tmp2.Status.PostgresClusterStatus = ClusterStatusInvalid - } else if err := validateCloneClusterDescription(&tmp2.Spec.Clone); err != nil { + tmp2.Status = PostgresStatus{PostgresClusterStatus: ClusterStatusInvalid} + } else if err := validateCloneClusterDescription(tmp2.Spec.Clone); err != nil { + tmp2.Error = err.Error() tmp2.Status.PostgresClusterStatus = ClusterStatusInvalid } else { diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 5df82e947..24ef24d63 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -53,7 +53,7 @@ type PostgresSpec struct { NumberOfInstances int32 `json:"numberOfInstances"` Users map[string]UserFlags `json:"users"` MaintenanceWindows []MaintenanceWindow `json:"maintenanceWindows,omitempty"` - Clone CloneDescription `json:"clone"` + Clone *CloneDescription `json:"clone"` ClusterName string `json:"-"` Databases map[string]string `json:"databases,omitempty"` PreparedDatabases map[string]PreparedDatabase `json:"preparedDatabases,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/util.go b/pkg/apis/acid.zalan.do/v1/util.go index db6efcd71..a795ec685 100644 --- a/pkg/apis/acid.zalan.do/v1/util.go +++ b/pkg/apis/acid.zalan.do/v1/util.go @@ -72,7 +72,7 @@ func extractClusterName(clusterName string, teamName string) (string, error) { func validateCloneClusterDescription(clone *CloneDescription) error { // when cloning from the basebackup (no end timestamp) check that the cluster name is a valid service name - if clone.ClusterName != "" && clone.EndTimestamp == "" { + if clone != nil && clone.ClusterName != "" && clone.EndTimestamp == "" { if !serviceNameRegex.MatchString(clone.ClusterName) { return fmt.Errorf("clone cluster name must confirm to DNS-1035, regex used for validation is %q", serviceNameRegexString) diff --git a/pkg/apis/acid.zalan.do/v1/util_test.go b/pkg/apis/acid.zalan.do/v1/util_test.go index 28e9e8ca4..bf6875a82 100644 --- a/pkg/apis/acid.zalan.do/v1/util_test.go +++ b/pkg/apis/acid.zalan.do/v1/util_test.go @@ -163,7 +163,7 @@ var unmarshalCluster = []struct { "kind": "Postgresql","apiVersion": "acid.zalan.do/v1", "metadata": {"name": "acid-testcluster1"}, "spec": {"teamId": 100}}`), &tmp).Error(), }, - marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":{}},"status":"Invalid"}`), + marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":null},"status":"Invalid"}`), err: nil}, { about: "example with /status subresource", @@ -184,7 +184,7 @@ var unmarshalCluster = []struct { "kind": "Postgresql","apiVersion": "acid.zalan.do/v1", "metadata": {"name": "acid-testcluster1"}, "spec": {"teamId": 100}}`), &tmp).Error(), }, - marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":{}},"status":{"PostgresClusterStatus":"Invalid"}}`), + marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":null},"status":{"PostgresClusterStatus":"Invalid"}}`), err: nil}, { about: "example with detailed input manifest and deprecated pod_priority_class_name -> podPriorityClassName", @@ -327,7 +327,7 @@ var unmarshalCluster = []struct { EndTime: mustParseTime("05:15"), }, }, - Clone: CloneDescription{ + Clone: &CloneDescription{ ClusterName: "acid-batman", }, ClusterName: "testcluster1", @@ -351,7 +351,7 @@ var unmarshalCluster = []struct { Status: PostgresStatus{PostgresClusterStatus: ClusterStatusInvalid}, Error: errors.New("name must match {TEAM}-{NAME} format").Error(), }, - marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"teapot-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null} ,"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":{}},"status":{"PostgresClusterStatus":"Invalid"}}`), + marshal: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"teapot-testcluster1","creationTimestamp":null},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null} ,"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":null},"status":{"PostgresClusterStatus":"Invalid"}}`), err: nil}, { about: "example with clone", @@ -366,7 +366,7 @@ var unmarshalCluster = []struct { }, Spec: PostgresSpec{ TeamID: "acid", - Clone: CloneDescription{ + Clone: &CloneDescription{ ClusterName: "team-batman", }, ClusterName: "testcluster1", @@ -405,7 +405,7 @@ var unmarshalCluster = []struct { err: errors.New("unexpected end of JSON input")}, { about: "expect error on JSON with field's value malformatted", - in: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster","creationTimestamp":qaz},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":{}},"status":{"PostgresClusterStatus":"Invalid"}}`), + in: []byte(`{"kind":"Postgresql","apiVersion":"acid.zalan.do/v1","metadata":{"name":"acid-testcluster","creationTimestamp":qaz},"spec":{"postgresql":{"version":"","parameters":null},"volume":{"size":"","storageClass":""},"patroni":{"initdb":null,"pg_hba":null,"ttl":0,"loop_wait":0,"retry_timeout":0,"maximum_lag_on_failover":0,"slots":null},"resources":{"requests":{"cpu":"","memory":""},"limits":{"cpu":"","memory":""}},"teamId":"acid","allowedSourceRanges":null,"numberOfInstances":0,"users":null,"clone":null},"status":{"PostgresClusterStatus":"Invalid"}}`), out: Postgresql{}, marshal: []byte{}, err: errors.New("invalid character 'q' looking for beginning of value"), diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 5879c9b73..064ced184 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -567,7 +567,11 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - in.Clone.DeepCopyInto(&out.Clone) + if in.Clone != nil { + in, out := &in.Clone, &out.Clone + *out = new(CloneDescription) + (*in).DeepCopyInto(*out) + } if in.Databases != nil { in, out := &in.Databases, &out.Databases *out = make(map[string]string, len(*in)) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 21875f953..0f9a1a5bc 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -725,7 +725,7 @@ func (c *Cluster) generateSpiloPodEnvVars(uid types.UID, spiloConfiguration stri envVars = append(envVars, v1.EnvVar{Name: "KUBERNETES_USE_CONFIGMAPS", Value: "true"}) } - if cloneDescription.ClusterName != "" { + if cloneDescription != nil && cloneDescription.ClusterName != "" { envVars = append(envVars, c.generateCloneEnvironment(cloneDescription)...) } @@ -1065,7 +1065,7 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef spiloEnvVars := c.generateSpiloPodEnvVars( c.Postgresql.GetUID(), spiloConfiguration, - &spec.Clone, + spec.Clone, spec.StandbyCluster, customPodEnvVarsList, ) diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go index 3a7317938..d9248ae33 100644 --- a/pkg/cluster/sync_test.go +++ b/pkg/cluster/sync_test.go @@ -63,23 +63,26 @@ func noEmptySync(cluster *Cluster, err error, reason SyncReason) error { func TestConnectionPoolerSynchronization(t *testing.T) { testName := "Test connection pooler synchronization" - var cluster = New( - Config{ - OpConfig: config.Config{ - ProtectedRoles: []string{"admin"}, - Auth: config.Auth{ - SuperUsername: superUserName, - ReplicationUsername: replicationUserName, - }, - ConnectionPooler: config.ConnectionPooler{ - ConnectionPoolerDefaultCPURequest: "100m", - ConnectionPoolerDefaultCPULimit: "100m", - ConnectionPoolerDefaultMemoryRequest: "100Mi", - ConnectionPoolerDefaultMemoryLimit: "100Mi", - NumberOfInstances: int32ToPointer(1), + newCluster := func() *Cluster { + return New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", + NumberOfInstances: int32ToPointer(1), + }, }, - }, - }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) + } + cluster := newCluster() cluster.Statefulset = &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ @@ -87,20 +90,20 @@ func TestConnectionPoolerSynchronization(t *testing.T) { }, } - clusterMissingObjects := *cluster + clusterMissingObjects := newCluster() clusterMissingObjects.KubeClient = k8sutil.ClientMissingObjects() - clusterMock := *cluster + clusterMock := newCluster() clusterMock.KubeClient = k8sutil.NewMockKubernetesClient() - clusterDirtyMock := *cluster + clusterDirtyMock := newCluster() clusterDirtyMock.KubeClient = k8sutil.NewMockKubernetesClient() clusterDirtyMock.ConnectionPooler = &ConnectionPoolerObjects{ Deployment: &appsv1.Deployment{}, Service: &v1.Service{}, } - clusterNewDefaultsMock := *cluster + clusterNewDefaultsMock := newCluster() clusterNewDefaultsMock.KubeClient = k8sutil.NewMockKubernetesClient() tests := []struct { @@ -124,7 +127,7 @@ func TestConnectionPoolerSynchronization(t *testing.T) { ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, - cluster: &clusterMissingObjects, + cluster: clusterMissingObjects, defaultImage: "pooler:1.0", defaultInstances: 1, check: objectsAreSaved, @@ -139,7 +142,7 @@ func TestConnectionPoolerSynchronization(t *testing.T) { EnableConnectionPooler: boolToPointer(true), }, }, - cluster: &clusterMissingObjects, + cluster: clusterMissingObjects, defaultImage: "pooler:1.0", defaultInstances: 1, check: objectsAreSaved, @@ -154,7 +157,7 @@ func TestConnectionPoolerSynchronization(t *testing.T) { ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, - cluster: &clusterMissingObjects, + cluster: clusterMissingObjects, defaultImage: "pooler:1.0", defaultInstances: 1, check: objectsAreSaved, @@ -169,7 +172,7 @@ func TestConnectionPoolerSynchronization(t *testing.T) { newSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{}, }, - cluster: &clusterMock, + cluster: clusterMock, defaultImage: "pooler:1.0", defaultInstances: 1, check: objectsAreDeleted, @@ -182,7 +185,7 @@ func TestConnectionPoolerSynchronization(t *testing.T) { newSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{}, }, - cluster: &clusterDirtyMock, + cluster: clusterDirtyMock, defaultImage: "pooler:1.0", defaultInstances: 1, check: objectsAreDeleted, @@ -203,7 +206,7 @@ func TestConnectionPoolerSynchronization(t *testing.T) { }, }, }, - cluster: &clusterMock, + cluster: clusterMock, defaultImage: "pooler:1.0", defaultInstances: 1, check: deploymentUpdated, @@ -220,7 +223,7 @@ func TestConnectionPoolerSynchronization(t *testing.T) { ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, - cluster: &clusterNewDefaultsMock, + cluster: clusterNewDefaultsMock, defaultImage: "pooler:2.0", defaultInstances: 2, check: deploymentUpdated, @@ -239,7 +242,7 @@ func TestConnectionPoolerSynchronization(t *testing.T) { ConnectionPooler: &acidv1.ConnectionPooler{}, }, }, - cluster: &clusterMock, + cluster: clusterMock, defaultImage: "pooler:1.0", defaultInstances: 1, check: noEmptySync, From f3ddce81d50d5d816c1976d8b0f635b045e2de0f Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 30 Jul 2020 17:48:15 +0200 Subject: [PATCH 12/35] fix random order for pod environment tests (#1085) --- pkg/cluster/k8sres.go | 2 +- pkg/cluster/k8sres_test.go | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 0f9a1a5bc..d7878942c 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -780,7 +780,7 @@ func deduplicateEnvVars(input []v1.EnvVar, containerName string, logger *logrus. } else if names[va.Name] == 1 { names[va.Name]++ - // Some variables (those to configure the WAL_ and LOG_ shipping) may be overriden, only log as info + // Some variables (those to configure the WAL_ and LOG_ shipping) may be overwritten, only log as info if strings.HasPrefix(va.Name, "WAL_") || strings.HasPrefix(va.Name, "LOG_") { logger.Infof("global variable %q has been overwritten by configmap/secret for container %q", va.Name, containerName) diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 7261d5902..1e474fbf5 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "reflect" + "sort" "testing" @@ -749,7 +750,8 @@ func (c *mockConfigMap) Get(ctx context.Context, name string, options metav1.Get configmap := &v1.ConfigMap{} configmap.Name = testPodEnvironmentConfigMapName configmap.Data = map[string]string{ - "foo": "bar", + "foo1": "bar1", + "foo2": "bar2", } return configmap, nil } @@ -816,8 +818,12 @@ func TestPodEnvironmentConfigMapVariables(t *testing.T) { }, envVars: []v1.EnvVar{ { - Name: "foo", - Value: "bar", + Name: "foo1", + Value: "bar1", + }, + { + Name: "foo2", + Value: "bar2", }, }, }, @@ -825,6 +831,7 @@ func TestPodEnvironmentConfigMapVariables(t *testing.T) { for _, tt := range tests { c := newMockCluster(tt.opConfig) vars, err := c.getPodEnvironmentConfigMapVariables() + sort.Slice(vars, func(i, j int) bool { return vars[i].Name < vars[j].Name }) if !reflect.DeepEqual(vars, tt.envVars) { t.Errorf("%s %s: expected `%v` but got `%v`", testName, tt.subTest, tt.envVars, vars) @@ -902,6 +909,7 @@ func TestPodEnvironmentSecretVariables(t *testing.T) { for _, tt := range tests { c := newMockCluster(tt.opConfig) vars, err := c.getPodEnvironmentSecretVariables() + sort.Slice(vars, func(i, j int) bool { return vars[i].Name < vars[j].Name }) if !reflect.DeepEqual(vars, tt.envVars) { t.Errorf("%s %s: expected `%v` but got `%v`", testName, tt.subTest, tt.envVars, vars) From 7cf2fae6df5dfde03bd9ac87e9ab4493bab4aeef Mon Sep 17 00:00:00 2001 From: Dmitry Dolgov <9erthalion6@gmail.com> Date: Wed, 5 Aug 2020 14:18:56 +0200 Subject: [PATCH 13/35] [WIP] Extend infrastructure roles handling (#1064) Extend infrastructure roles handling Postgres Operator uses infrastructure roles to provide access to a database for external users e.g. for monitoring purposes. Such infrastructure roles are expected to be present in the form of k8s secrets with the following content: inrole1: some_encrypted_role password1: some_encrypted_password user1: some_entrypted_name inrole2: some_encrypted_role password2: some_encrypted_password user2: some_entrypted_name The format of this content is implied implicitly and not flexible enough. In case if we do not have possibility to change the format of a secret we want to use in the Operator, we need to recreate it in this format. To address this lets make the format of secret content explicitly. The idea is to introduce a new configuration option for the Operator. infrastructure_roles_secrets: - secretname: k8s_secret_name userkey: some_encrypted_name passwordkey: some_encrypted_password rolekey: some_encrypted_role - secretname: k8s_secret_name userkey: some_encrypted_name passwordkey: some_encrypted_password rolekey: some_encrypted_role This would allow Operator to use any avalable secrets to prepare infrastructure roles. To make it backward compatible simulate the old behaviour if the new option is not present. The new configuration option is intended be used mainly from CRD, but it's also available via Operator ConfigMap in a limited fashion. For ConfigMap one can put there only a string with one secret definition in the following format (as a string): infrastructure_roles_secrets: | secretname: k8s_secret_name, userkey: some_encrypted_name, passwordkey: some_encrypted_password, rolekey: some_encrypted_role Note than only one secret could be specified this way, no multiple secrets are allowed. Eventually the resulting list of infrastructure roles would be a total sum of all supported ways to describe it, namely legacy via infrastructure_roles_secret_name and infrastructure_roles_secrets from both ConfigMap and CRD. --- .../crds/operatorconfigurations.yaml | 22 + docs/reference/operator_parameters.md | 10 +- docs/user.md | 61 +- e2e/Dockerfile | 3 + e2e/exec.sh | 2 + e2e/tests/test_e2e.py | 1028 +++++++++-------- manifests/configmap.yaml | 3 +- manifests/infrastructure-roles-new.yaml | 14 + manifests/operatorconfiguration.crd.yaml | 22 + ...gresql-operator-default-configuration.yaml | 8 + pkg/apis/acid.zalan.do/v1/crds.go | 31 +- .../v1/operator_configuration_type.go | 45 +- .../acid.zalan.do/v1/zz_generated.deepcopy.go | 12 + pkg/controller/controller.go | 3 +- pkg/controller/operator_config.go | 15 + pkg/controller/util.go | 286 ++++- pkg/controller/util_test.go | 297 ++++- pkg/spec/types.go | 4 + pkg/util/config/config.go | 42 +- pkg/util/k8sutil/k8sutil.go | 70 +- 20 files changed, 1366 insertions(+), 612 deletions(-) create mode 100755 e2e/exec.sh create mode 100644 manifests/infrastructure-roles-new.yaml diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 89f495367..3218decd7 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -131,6 +131,28 @@ spec: type: boolean infrastructure_roles_secret_name: type: string + infrastructure_roles_secrets: + type: array + nullable: true + items: + type: object + required: + - secretname + - userkey + - passwordkey + properties: + secretname: + type: string + userkey: + type: string + passwordkey: + type: string + rolekey: + type: string + details: + type: string + template: + type: boolean inherited_labels: type: array items: diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 7e5196d56..20771078f 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -252,8 +252,14 @@ configuration they are grouped under the `kubernetes` key. teams API. The default is `postgresql-operator`. * **infrastructure_roles_secret_name** - namespaced name of the secret containing infrastructure roles names and - passwords. + *deprecated*: namespaced name of the secret containing infrastructure roles + with user names, passwords and role membership. + +* **infrastructure_roles_secrets** + array of infrastructure role definitions which reference existing secrets + and specify the key names from which user name, password and role membership + are extracted. For the ConfigMap this has to be a string which allows + referencing only one infrastructure roles secret. The default is empty. * **pod_role_label** name of the label assigned to the Postgres pods (and services/endpoints) by diff --git a/docs/user.md b/docs/user.md index 3683fdf61..a4b1424b8 100644 --- a/docs/user.md +++ b/docs/user.md @@ -150,23 +150,62 @@ user. There are two ways to define them: #### Infrastructure roles secret -The infrastructure roles secret is specified by the `infrastructure_roles_secret_name` -parameter. The role definition looks like this (values are base64 encoded): +Infrastructure roles can be specified by the `infrastructure_roles_secrets` +parameter where you can reference multiple existing secrets. Prior to `v1.6.0` +the operator could only reference one secret with the +`infrastructure_roles_secret_name` option. However, this secret could contain +multiple roles using the same set of keys plus incrementing index. ```yaml -user1: ZGJ1c2Vy -password1: c2VjcmV0 -inrole1: b3BlcmF0b3I= +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-infrastructure-roles +data: + user1: ZGJ1c2Vy + password1: c2VjcmV0 + inrole1: b3BlcmF0b3I= + user2: ... ``` The block above describes the infrastructure role 'dbuser' with password -'secret' that is a member of the 'operator' role. For the following definitions -one must increase the index, i.e. the next role will be defined as 'user2' and -so on. The resulting role will automatically be a login role. +'secret' that is a member of the 'operator' role. The resulting role will +automatically be a login role. + +With the new option users can configure the names of secret keys that contain +the user name, password etc. The secret itself is referenced by the +`secretname` key. If the secret uses a template for multiple roles as described +above list them separately. -Note that with definitions that solely use the infrastructure roles secret -there is no way to specify role options (like superuser or nologin) or role -memberships. This is where the ConfigMap comes into play. +```yaml +apiVersion: v1 +kind: OperatorConfiguration +metadata: + name: postgresql-operator-configuration +configuration: + kubernetes: + infrastructure_roles_secrets: + - secretname: "postgresql-infrastructure-roles" + userkey: "user1" + passwordkey: "password1" + rolekey: "inrole1" + - secretname: "postgresql-infrastructure-roles" + userkey: "user2" + ... +``` + +Note, only the CRD-based configuration allows for referencing multiple secrets. +As of now, the ConfigMap is restricted to either one or the existing template +option with `infrastructure_roles_secret_name`. Please, refer to the example +manifests to understand how `infrastructure_roles_secrets` has to be configured +for the [configmap](../manifests/configmap.yaml) or [CRD configuration](../manifests/postgresql-operator-default-configuration.yaml). + +If both `infrastructure_roles_secret_name` and `infrastructure_roles_secrets` +are defined the operator will create roles for both of them. So make sure, +they do not collide. Note also, that with definitions that solely use the +infrastructure roles secret there is no way to specify role options (like +superuser or nologin) or role memberships. This is where the additional +ConfigMap comes into play. #### Secret plus ConfigMap diff --git a/e2e/Dockerfile b/e2e/Dockerfile index 236942d04..a250ea9cb 100644 --- a/e2e/Dockerfile +++ b/e2e/Dockerfile @@ -1,7 +1,10 @@ +# An image to perform the actual test. Do not forget to copy all necessary test +# files here. FROM ubuntu:18.04 LABEL maintainer="Team ACID @ Zalando " COPY manifests ./manifests +COPY exec.sh ./exec.sh COPY requirements.txt tests ./ RUN apt-get update \ diff --git a/e2e/exec.sh b/e2e/exec.sh new file mode 100755 index 000000000..56276bc3c --- /dev/null +++ b/e2e/exec.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +kubectl exec -it $1 -- sh -c "$2" diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 18b9852c4..4cd1c6a30 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -1,3 +1,4 @@ +import json import unittest import time import timeout_decorator @@ -50,7 +51,8 @@ def setUpClass(cls): for filename in ["operator-service-account-rbac.yaml", "configmap.yaml", - "postgres-operator.yaml"]: + "postgres-operator.yaml", + "infrastructure-roles-new.yaml"]: result = k8s.create_with_kubectl("manifests/" + filename) print("stdout: {}, stderr: {}".format(result.stdout, result.stderr)) @@ -69,506 +71,547 @@ def setUpClass(cls): print('Operator log: {}'.format(k8s.get_operator_log())) raise - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_enable_disable_connection_pooler(self): - ''' - For a database without connection pooler, then turns it on, scale up, - turn off and on again. Test with different ways of doing this (via - enableConnectionPooler or connectionPooler configuration section). At - the end turn connection pooler off to not interfere with other tests. - ''' - k8s = self.k8s - service_labels = { - 'cluster-name': 'acid-minimal-cluster', - } - pod_labels = dict({ - 'connection-pooler': 'acid-minimal-cluster-pooler', - }) - - pod_selector = to_selector(pod_labels) - service_selector = to_selector(service_labels) - - try: - # enable connection pooler - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'enableConnectionPooler': True, - } - }) - k8s.wait_for_pod_start(pod_selector) - - pods = k8s.api.core_v1.list_namespaced_pod( - 'default', label_selector=pod_selector - ).items - - self.assertTrue(pods, 'No connection pooler pods') - - k8s.wait_for_service(service_selector) - services = k8s.api.core_v1.list_namespaced_service( - 'default', label_selector=service_selector - ).items - services = [ - s for s in services - if s.metadata.name.endswith('pooler') - ] - - self.assertTrue(services, 'No connection pooler service') - - # scale up connection pooler deployment - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'connectionPooler': { - 'numberOfInstances': 2, - }, - } - }) - - k8s.wait_for_running_pods(pod_selector, 2) - - # turn it off, keeping configuration section - k8s.api.custom_objects_api.patch_namespaced_custom_object( - 'acid.zalan.do', 'v1', 'default', - 'postgresqls', 'acid-minimal-cluster', - { - 'spec': { - 'enableConnectionPooler': False, - } - }) - k8s.wait_for_pods_to_stop(pod_selector) - - except timeout_decorator.TimeoutError: - print('Operator log: {}'.format(k8s.get_operator_log())) - raise - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_enable_load_balancer(self): - ''' - Test if services are updated when enabling/disabling load balancers - ''' - - k8s = self.k8s - cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - - # enable load balancer services - pg_patch_enable_lbs = { - "spec": { - "enableMasterLoadBalancer": True, - "enableReplicaLoadBalancer": True - } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_lbs) - # wait for service recreation - time.sleep(60) - - master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') - self.assertEqual(master_svc_type, 'LoadBalancer', - "Expected LoadBalancer service type for master, found {}".format(master_svc_type)) - - repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') - self.assertEqual(repl_svc_type, 'LoadBalancer', - "Expected LoadBalancer service type for replica, found {}".format(repl_svc_type)) - - # disable load balancer services again - pg_patch_disable_lbs = { - "spec": { - "enableMasterLoadBalancer": False, - "enableReplicaLoadBalancer": False - } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_lbs) - # wait for service recreation - time.sleep(60) - - master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') - self.assertEqual(master_svc_type, 'ClusterIP', - "Expected ClusterIP service type for master, found {}".format(master_svc_type)) - - repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') - self.assertEqual(repl_svc_type, 'ClusterIP', - "Expected ClusterIP service type for replica, found {}".format(repl_svc_type)) - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_lazy_spilo_upgrade(self): - ''' - Test lazy upgrade for the Spilo image: operator changes a stateful set but lets pods run with the old image - until they are recreated for reasons other than operator's activity. That works because the operator configures - stateful sets to use "onDelete" pod update policy. - - The test covers: - 1) enabling lazy upgrade in existing operator deployment - 2) forcing the normal rolling upgrade by changing the operator configmap and restarting its pod - ''' - - k8s = self.k8s - - # update docker image in config and enable the lazy upgrade - conf_image = "registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p114" - patch_lazy_spilo_upgrade = { - "data": { - "docker_image": conf_image, - "enable_lazy_spilo_upgrade": "true" - } - } - k8s.update_config(patch_lazy_spilo_upgrade) - - pod0 = 'acid-minimal-cluster-0' - pod1 = 'acid-minimal-cluster-1' - - # restart the pod to get a container with the new image - k8s.api.core_v1.delete_namespaced_pod(pod0, 'default') - time.sleep(60) - - # lazy update works if the restarted pod and older pods run different Spilo versions - new_image = k8s.get_effective_pod_image(pod0) - old_image = k8s.get_effective_pod_image(pod1) - self.assertNotEqual(new_image, old_image, "Lazy updated failed: pods have the same image {}".format(new_image)) - - # sanity check - assert_msg = "Image {} of a new pod differs from {} in operator conf".format(new_image, conf_image) - self.assertEqual(new_image, conf_image, assert_msg) - - # clean up - unpatch_lazy_spilo_upgrade = { - "data": { - "enable_lazy_spilo_upgrade": "false", - } - } - k8s.update_config(unpatch_lazy_spilo_upgrade) - - # at this point operator will complete the normal rolling upgrade - # so we additonally test if disabling the lazy upgrade - forcing the normal rolling upgrade - works - - # XXX there is no easy way to wait until the end of Sync() - time.sleep(60) - - image0 = k8s.get_effective_pod_image(pod0) - image1 = k8s.get_effective_pod_image(pod1) - - assert_msg = "Disabling lazy upgrade failed: pods still have different images {} and {}".format(image0, image1) - self.assertEqual(image0, image1, assert_msg) - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_logical_backup_cron_job(self): - ''' - Ensure we can (a) create the cron job at user request for a specific PG cluster - (b) update the cluster-wide image for the logical backup pod - (c) delete the job at user request - - Limitations: - (a) Does not run the actual batch job because there is no S3 mock to upload backups to - (b) Assumes 'acid-minimal-cluster' exists as defined in setUp - ''' - - k8s = self.k8s - - # create the cron job - schedule = "7 7 7 7 *" - pg_patch_enable_backup = { - "spec": { - "enableLogicalBackup": True, - "logicalBackupSchedule": schedule - } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_backup) - k8s.wait_for_logical_backup_job_creation() - - jobs = k8s.get_logical_backup_job().items - self.assertEqual(1, len(jobs), "Expected 1 logical backup job, found {}".format(len(jobs))) - - job = jobs[0] - self.assertEqual(job.metadata.name, "logical-backup-acid-minimal-cluster", - "Expected job name {}, found {}" - .format("logical-backup-acid-minimal-cluster", job.metadata.name)) - self.assertEqual(job.spec.schedule, schedule, - "Expected {} schedule, found {}" - .format(schedule, job.spec.schedule)) - - # update the cluster-wide image of the logical backup pod - image = "test-image-name" - patch_logical_backup_image = { - "data": { - "logical_backup_docker_image": image, - } - } - k8s.update_config(patch_logical_backup_image) - - jobs = k8s.get_logical_backup_job().items - actual_image = jobs[0].spec.job_template.spec.template.spec.containers[0].image - self.assertEqual(actual_image, image, - "Expected job image {}, found {}".format(image, actual_image)) - - # delete the logical backup cron job - pg_patch_disable_backup = { - "spec": { - "enableLogicalBackup": False, - } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_backup) - k8s.wait_for_logical_backup_job_deletion() - jobs = k8s.get_logical_backup_job().items - self.assertEqual(0, len(jobs), - "Expected 0 logical backup jobs, found {}".format(len(jobs))) + # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + # def test_enable_disable_connection_pooler(self): + # ''' + # For a database without connection pooler, then turns it on, scale up, + # turn off and on again. Test with different ways of doing this (via + # enableConnectionPooler or connectionPooler configuration section). At + # the end turn connection pooler off to not interfere with other tests. + # ''' + # k8s = self.k8s + # service_labels = { + # 'cluster-name': 'acid-minimal-cluster', + # } + # pod_labels = dict({ + # 'connection-pooler': 'acid-minimal-cluster-pooler', + # }) + + # pod_selector = to_selector(pod_labels) + # service_selector = to_selector(service_labels) + + # try: + # # enable connection pooler + # k8s.api.custom_objects_api.patch_namespaced_custom_object( + # 'acid.zalan.do', 'v1', 'default', + # 'postgresqls', 'acid-minimal-cluster', + # { + # 'spec': { + # 'enableConnectionPooler': True, + # } + # }) + # k8s.wait_for_pod_start(pod_selector) + + # pods = k8s.api.core_v1.list_namespaced_pod( + # 'default', label_selector=pod_selector + # ).items + + # self.assertTrue(pods, 'No connection pooler pods') + + # k8s.wait_for_service(service_selector) + # services = k8s.api.core_v1.list_namespaced_service( + # 'default', label_selector=service_selector + # ).items + # services = [ + # s for s in services + # if s.metadata.name.endswith('pooler') + # ] + + # self.assertTrue(services, 'No connection pooler service') + + # # scale up connection pooler deployment + # k8s.api.custom_objects_api.patch_namespaced_custom_object( + # 'acid.zalan.do', 'v1', 'default', + # 'postgresqls', 'acid-minimal-cluster', + # { + # 'spec': { + # 'connectionPooler': { + # 'numberOfInstances': 2, + # }, + # } + # }) + + # k8s.wait_for_running_pods(pod_selector, 2) + + # # turn it off, keeping configuration section + # k8s.api.custom_objects_api.patch_namespaced_custom_object( + # 'acid.zalan.do', 'v1', 'default', + # 'postgresqls', 'acid-minimal-cluster', + # { + # 'spec': { + # 'enableConnectionPooler': False, + # } + # }) + # k8s.wait_for_pods_to_stop(pod_selector) + + # except timeout_decorator.TimeoutError: + # print('Operator log: {}'.format(k8s.get_operator_log())) + # raise + + # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + # def test_enable_load_balancer(self): + # ''' + # Test if services are updated when enabling/disabling load balancers + # ''' + + # k8s = self.k8s + # cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + + # # enable load balancer services + # pg_patch_enable_lbs = { + # "spec": { + # "enableMasterLoadBalancer": True, + # "enableReplicaLoadBalancer": True + # } + # } + # k8s.api.custom_objects_api.patch_namespaced_custom_object( + # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_lbs) + # # wait for service recreation + # time.sleep(60) + + # master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') + # self.assertEqual(master_svc_type, 'LoadBalancer', + # "Expected LoadBalancer service type for master, found {}".format(master_svc_type)) + + # repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') + # self.assertEqual(repl_svc_type, 'LoadBalancer', + # "Expected LoadBalancer service type for replica, found {}".format(repl_svc_type)) + + # # disable load balancer services again + # pg_patch_disable_lbs = { + # "spec": { + # "enableMasterLoadBalancer": False, + # "enableReplicaLoadBalancer": False + # } + # } + # k8s.api.custom_objects_api.patch_namespaced_custom_object( + # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_lbs) + # # wait for service recreation + # time.sleep(60) + + # master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') + # self.assertEqual(master_svc_type, 'ClusterIP', + # "Expected ClusterIP service type for master, found {}".format(master_svc_type)) + + # repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') + # self.assertEqual(repl_svc_type, 'ClusterIP', + # "Expected ClusterIP service type for replica, found {}".format(repl_svc_type)) + + # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + # def test_lazy_spilo_upgrade(self): + # ''' + # Test lazy upgrade for the Spilo image: operator changes a stateful set but lets pods run with the old image + # until they are recreated for reasons other than operator's activity. That works because the operator configures + # stateful sets to use "onDelete" pod update policy. + + # The test covers: + # 1) enabling lazy upgrade in existing operator deployment + # 2) forcing the normal rolling upgrade by changing the operator configmap and restarting its pod + # ''' + + # k8s = self.k8s + + # # update docker image in config and enable the lazy upgrade + # conf_image = "registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p114" + # patch_lazy_spilo_upgrade = { + # "data": { + # "docker_image": conf_image, + # "enable_lazy_spilo_upgrade": "true" + # } + # } + # k8s.update_config(patch_lazy_spilo_upgrade) + + # pod0 = 'acid-minimal-cluster-0' + # pod1 = 'acid-minimal-cluster-1' + + # # restart the pod to get a container with the new image + # k8s.api.core_v1.delete_namespaced_pod(pod0, 'default') + # time.sleep(60) + + # # lazy update works if the restarted pod and older pods run different Spilo versions + # new_image = k8s.get_effective_pod_image(pod0) + # old_image = k8s.get_effective_pod_image(pod1) + # self.assertNotEqual(new_image, old_image, "Lazy updated failed: pods have the same image {}".format(new_image)) + + # # sanity check + # assert_msg = "Image {} of a new pod differs from {} in operator conf".format(new_image, conf_image) + # self.assertEqual(new_image, conf_image, assert_msg) + + # # clean up + # unpatch_lazy_spilo_upgrade = { + # "data": { + # "enable_lazy_spilo_upgrade": "false", + # } + # } + # k8s.update_config(unpatch_lazy_spilo_upgrade) + + # # at this point operator will complete the normal rolling upgrade + # # so we additonally test if disabling the lazy upgrade - forcing the normal rolling upgrade - works + + # # XXX there is no easy way to wait until the end of Sync() + # time.sleep(60) + + # image0 = k8s.get_effective_pod_image(pod0) + # image1 = k8s.get_effective_pod_image(pod1) + + # assert_msg = "Disabling lazy upgrade failed: pods still have different images {} and {}".format(image0, image1) + # self.assertEqual(image0, image1, assert_msg) + + # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + # def test_logical_backup_cron_job(self): + # ''' + # Ensure we can (a) create the cron job at user request for a specific PG cluster + # (b) update the cluster-wide image for the logical backup pod + # (c) delete the job at user request + + # Limitations: + # (a) Does not run the actual batch job because there is no S3 mock to upload backups to + # (b) Assumes 'acid-minimal-cluster' exists as defined in setUp + # ''' + + # k8s = self.k8s + + # # create the cron job + # schedule = "7 7 7 7 *" + # pg_patch_enable_backup = { + # "spec": { + # "enableLogicalBackup": True, + # "logicalBackupSchedule": schedule + # } + # } + # k8s.api.custom_objects_api.patch_namespaced_custom_object( + # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_backup) + # k8s.wait_for_logical_backup_job_creation() + + # jobs = k8s.get_logical_backup_job().items + # self.assertEqual(1, len(jobs), "Expected 1 logical backup job, found {}".format(len(jobs))) + + # job = jobs[0] + # self.assertEqual(job.metadata.name, "logical-backup-acid-minimal-cluster", + # "Expected job name {}, found {}" + # .format("logical-backup-acid-minimal-cluster", job.metadata.name)) + # self.assertEqual(job.spec.schedule, schedule, + # "Expected {} schedule, found {}" + # .format(schedule, job.spec.schedule)) + + # # update the cluster-wide image of the logical backup pod + # image = "test-image-name" + # patch_logical_backup_image = { + # "data": { + # "logical_backup_docker_image": image, + # } + # } + # k8s.update_config(patch_logical_backup_image) + + # jobs = k8s.get_logical_backup_job().items + # actual_image = jobs[0].spec.job_template.spec.template.spec.containers[0].image + # self.assertEqual(actual_image, image, + # "Expected job image {}, found {}".format(image, actual_image)) + + # # delete the logical backup cron job + # pg_patch_disable_backup = { + # "spec": { + # "enableLogicalBackup": False, + # } + # } + # k8s.api.custom_objects_api.patch_namespaced_custom_object( + # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_backup) + # k8s.wait_for_logical_backup_job_deletion() + # jobs = k8s.get_logical_backup_job().items + # self.assertEqual(0, len(jobs), + # "Expected 0 logical backup jobs, found {}".format(len(jobs))) + + # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + # def test_min_resource_limits(self): + # ''' + # Lower resource limits below configured minimum and let operator fix it + # ''' + # k8s = self.k8s + # cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + # labels = 'spilo-role=master,' + cluster_label + # _, failover_targets = k8s.get_pg_nodes(cluster_label) + + # # configure minimum boundaries for CPU and memory limits + # minCPULimit = '500m' + # minMemoryLimit = '500Mi' + # patch_min_resource_limits = { + # "data": { + # "min_cpu_limit": minCPULimit, + # "min_memory_limit": minMemoryLimit + # } + # } + # k8s.update_config(patch_min_resource_limits) + + # # lower resource limits below minimum + # pg_patch_resources = { + # "spec": { + # "resources": { + # "requests": { + # "cpu": "10m", + # "memory": "50Mi" + # }, + # "limits": { + # "cpu": "200m", + # "memory": "200Mi" + # } + # } + # } + # } + # k8s.api.custom_objects_api.patch_namespaced_custom_object( + # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources) + # k8s.wait_for_pod_failover(failover_targets, labels) + # k8s.wait_for_pod_start('spilo-role=replica') + + # pods = k8s.api.core_v1.list_namespaced_pod( + # 'default', label_selector=labels).items + # self.assert_master_is_unique() + # masterPod = pods[0] + + # self.assertEqual(masterPod.spec.containers[0].resources.limits['cpu'], minCPULimit, + # "Expected CPU limit {}, found {}" + # .format(minCPULimit, masterPod.spec.containers[0].resources.limits['cpu'])) + # self.assertEqual(masterPod.spec.containers[0].resources.limits['memory'], minMemoryLimit, + # "Expected memory limit {}, found {}" + # .format(minMemoryLimit, masterPod.spec.containers[0].resources.limits['memory'])) + + # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + # def test_multi_namespace_support(self): + # ''' + # Create a customized Postgres cluster in a non-default namespace. + # ''' + # k8s = self.k8s + + # with open("manifests/complete-postgres-manifest.yaml", 'r+') as f: + # pg_manifest = yaml.safe_load(f) + # pg_manifest["metadata"]["namespace"] = self.namespace + # yaml.dump(pg_manifest, f, Dumper=yaml.Dumper) + + # k8s.create_with_kubectl("manifests/complete-postgres-manifest.yaml") + # k8s.wait_for_pod_start("spilo-role=master", self.namespace) + # self.assert_master_is_unique(self.namespace, "acid-test-cluster") + + # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + # def test_node_readiness_label(self): + # ''' + # Remove node readiness label from master node. This must cause a failover. + # ''' + # k8s = self.k8s + # cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + # readiness_label = 'lifecycle-status' + # readiness_value = 'ready' + + # # get nodes of master and replica(s) (expected target of new master) + # current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) + # num_replicas = len(current_replica_nodes) + # failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) + + # # add node_readiness_label to potential failover nodes + # patch_readiness_label = { + # "metadata": { + # "labels": { + # readiness_label: readiness_value + # } + # } + # } + # for failover_target in failover_targets: + # k8s.api.core_v1.patch_node(failover_target, patch_readiness_label) + + # # define node_readiness_label in config map which should trigger a failover of the master + # patch_readiness_label_config = { + # "data": { + # "node_readiness_label": readiness_label + ':' + readiness_value, + # } + # } + # k8s.update_config(patch_readiness_label_config) + # new_master_node, new_replica_nodes = self.assert_failover( + # current_master_node, num_replicas, failover_targets, cluster_label) + + # # patch also node where master ran before + # k8s.api.core_v1.patch_node(current_master_node, patch_readiness_label) + + # # wait a little before proceeding with the pod distribution test + # time.sleep(30) + + # # toggle pod anti affinity to move replica away from master node + # self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) + + # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + # def test_scaling(self): + # ''' + # Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime. + # ''' + # k8s = self.k8s + # labels = "application=spilo,cluster-name=acid-minimal-cluster" + + # k8s.wait_for_pg_to_scale(3) + # self.assertEqual(3, k8s.count_pods_with_label(labels)) + # self.assert_master_is_unique() + + # k8s.wait_for_pg_to_scale(2) + # self.assertEqual(2, k8s.count_pods_with_label(labels)) + # self.assert_master_is_unique() + + # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + # def test_service_annotations(self): + # ''' + # Create a Postgres cluster with service annotations and check them. + # ''' + # k8s = self.k8s + # patch_custom_service_annotations = { + # "data": { + # "custom_service_annotations": "foo:bar", + # } + # } + # k8s.update_config(patch_custom_service_annotations) + + # pg_patch_custom_annotations = { + # "spec": { + # "serviceAnnotations": { + # "annotation.key": "value", + # "foo": "bar", + # } + # } + # } + # k8s.api.custom_objects_api.patch_namespaced_custom_object( + # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_custom_annotations) + + # # wait a little before proceeding + # time.sleep(30) + # annotations = { + # "annotation.key": "value", + # "foo": "bar", + # } + # self.assertTrue(k8s.check_service_annotations( + # "cluster-name=acid-minimal-cluster,spilo-role=master", annotations)) + # self.assertTrue(k8s.check_service_annotations( + # "cluster-name=acid-minimal-cluster,spilo-role=replica", annotations)) + + # # clean up + # unpatch_custom_service_annotations = { + # "data": { + # "custom_service_annotations": "", + # } + # } + # k8s.update_config(unpatch_custom_service_annotations) + + # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + # def test_statefulset_annotation_propagation(self): + # ''' + # Inject annotation to Postgresql CRD and check it's propagation to stateful set + # ''' + # k8s = self.k8s + # cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + + # patch_sset_propagate_annotations = { + # "data": { + # "downscaler_annotations": "deployment-time,downscaler/*", + # } + # } + # k8s.update_config(patch_sset_propagate_annotations) + + # pg_crd_annotations = { + # "metadata": { + # "annotations": { + # "deployment-time": "2020-04-30 12:00:00", + # "downscaler/downtime_replicas": "0", + # }, + # } + # } + # k8s.api.custom_objects_api.patch_namespaced_custom_object( + # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_crd_annotations) + + # # wait a little before proceeding + # time.sleep(60) + # annotations = { + # "deployment-time": "2020-04-30 12:00:00", + # "downscaler/downtime_replicas": "0", + # } + # self.assertTrue(k8s.check_statefulset_annotations(cluster_label, annotations)) + + # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + # def test_taint_based_eviction(self): + # ''' + # Add taint "postgres=:NoExecute" to node with master. This must cause a failover. + # ''' + # k8s = self.k8s + # cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + + # # get nodes of master and replica(s) (expected target of new master) + # current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) + # num_replicas = len(current_replica_nodes) + # failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) + + # # taint node with postgres=:NoExecute to force failover + # body = { + # "spec": { + # "taints": [ + # { + # "effect": "NoExecute", + # "key": "postgres" + # } + # ] + # } + # } + + # # patch node and test if master is failing over to one of the expected nodes + # k8s.api.core_v1.patch_node(current_master_node, body) + # new_master_node, new_replica_nodes = self.assert_failover( + # current_master_node, num_replicas, failover_targets, cluster_label) + + # # add toleration to pods + # patch_toleration_config = { + # "data": { + # "toleration": "key:postgres,operator:Exists,effect:NoExecute" + # } + # } + # k8s.update_config(patch_toleration_config) + + # # wait a little before proceeding with the pod distribution test + # time.sleep(30) + + # # toggle pod anti affinity to move replica away from master node + # self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_min_resource_limits(self): + def test_infrastructure_roles(self): ''' - Lower resource limits below configured minimum and let operator fix it + Test using external secrets for infrastructure roles ''' k8s = self.k8s - cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - labels = 'spilo-role=master,' + cluster_label - _, failover_targets = k8s.get_pg_nodes(cluster_label) - - # configure minimum boundaries for CPU and memory limits - minCPULimit = '500m' - minMemoryLimit = '500Mi' - patch_min_resource_limits = { + # update infrastructure roles description + secret_name = "postgresql-infrastructure-roles-old" + roles = "secretname: postgresql-infrastructure-roles-new, userkey: user, rolekey: role, passwordkey: password" + patch_infrastructure_roles = { "data": { - "min_cpu_limit": minCPULimit, - "min_memory_limit": minMemoryLimit - } + "infrastructure_roles_secret_name": secret_name, + "infrastructure_roles_secrets": roles, + }, } - k8s.update_config(patch_min_resource_limits) - - # lower resource limits below minimum - pg_patch_resources = { - "spec": { - "resources": { - "requests": { - "cpu": "10m", - "memory": "50Mi" - }, - "limits": { - "cpu": "200m", - "memory": "200Mi" - } - } - } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources) - k8s.wait_for_pod_failover(failover_targets, labels) - k8s.wait_for_pod_start('spilo-role=replica') - - pods = k8s.api.core_v1.list_namespaced_pod( - 'default', label_selector=labels).items - self.assert_master_is_unique() - masterPod = pods[0] - - self.assertEqual(masterPod.spec.containers[0].resources.limits['cpu'], minCPULimit, - "Expected CPU limit {}, found {}" - .format(minCPULimit, masterPod.spec.containers[0].resources.limits['cpu'])) - self.assertEqual(masterPod.spec.containers[0].resources.limits['memory'], minMemoryLimit, - "Expected memory limit {}, found {}" - .format(minMemoryLimit, masterPod.spec.containers[0].resources.limits['memory'])) - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_multi_namespace_support(self): - ''' - Create a customized Postgres cluster in a non-default namespace. - ''' - k8s = self.k8s - - with open("manifests/complete-postgres-manifest.yaml", 'r+') as f: - pg_manifest = yaml.safe_load(f) - pg_manifest["metadata"]["namespace"] = self.namespace - yaml.dump(pg_manifest, f, Dumper=yaml.Dumper) - - k8s.create_with_kubectl("manifests/complete-postgres-manifest.yaml") - k8s.wait_for_pod_start("spilo-role=master", self.namespace) - self.assert_master_is_unique(self.namespace, "acid-test-cluster") - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_node_readiness_label(self): - ''' - Remove node readiness label from master node. This must cause a failover. - ''' - k8s = self.k8s - cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - readiness_label = 'lifecycle-status' - readiness_value = 'ready' - - # get nodes of master and replica(s) (expected target of new master) - current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) - num_replicas = len(current_replica_nodes) - failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) - - # add node_readiness_label to potential failover nodes - patch_readiness_label = { - "metadata": { - "labels": { - readiness_label: readiness_value - } - } - } - for failover_target in failover_targets: - k8s.api.core_v1.patch_node(failover_target, patch_readiness_label) - - # define node_readiness_label in config map which should trigger a failover of the master - patch_readiness_label_config = { - "data": { - "node_readiness_label": readiness_label + ':' + readiness_value, - } - } - k8s.update_config(patch_readiness_label_config) - new_master_node, new_replica_nodes = self.assert_failover( - current_master_node, num_replicas, failover_targets, cluster_label) - - # patch also node where master ran before - k8s.api.core_v1.patch_node(current_master_node, patch_readiness_label) - - # wait a little before proceeding with the pod distribution test - time.sleep(30) - - # toggle pod anti affinity to move replica away from master node - self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_scaling(self): - ''' - Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime. - ''' - k8s = self.k8s - labels = "application=spilo,cluster-name=acid-minimal-cluster" - - k8s.wait_for_pg_to_scale(3) - self.assertEqual(3, k8s.count_pods_with_label(labels)) - self.assert_master_is_unique() - - k8s.wait_for_pg_to_scale(2) - self.assertEqual(2, k8s.count_pods_with_label(labels)) - self.assert_master_is_unique() - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_service_annotations(self): - ''' - Create a Postgres cluster with service annotations and check them. - ''' - k8s = self.k8s - patch_custom_service_annotations = { - "data": { - "custom_service_annotations": "foo:bar", - } - } - k8s.update_config(patch_custom_service_annotations) - - pg_patch_custom_annotations = { - "spec": { - "serviceAnnotations": { - "annotation.key": "value", - "foo": "bar", - } - } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_custom_annotations) + k8s.update_config(patch_infrastructure_roles) # wait a little before proceeding time.sleep(30) - annotations = { - "annotation.key": "value", - "foo": "bar", - } - self.assertTrue(k8s.check_service_annotations( - "cluster-name=acid-minimal-cluster,spilo-role=master", annotations)) - self.assertTrue(k8s.check_service_annotations( - "cluster-name=acid-minimal-cluster,spilo-role=replica", annotations)) - - # clean up - unpatch_custom_service_annotations = { - "data": { - "custom_service_annotations": "", - } - } - k8s.update_config(unpatch_custom_service_annotations) - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_statefulset_annotation_propagation(self): - ''' - Inject annotation to Postgresql CRD and check it's propagation to stateful set - ''' - k8s = self.k8s - cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - patch_sset_propagate_annotations = { - "data": { - "downscaler_annotations": "deployment-time,downscaler/*", - } - } - k8s.update_config(patch_sset_propagate_annotations) - - pg_crd_annotations = { - "metadata": { - "annotations": { - "deployment-time": "2020-04-30 12:00:00", - "downscaler/downtime_replicas": "0", - }, - } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_crd_annotations) - - # wait a little before proceeding - time.sleep(60) - annotations = { - "deployment-time": "2020-04-30 12:00:00", - "downscaler/downtime_replicas": "0", - } - self.assertTrue(k8s.check_statefulset_annotations(cluster_label, annotations)) - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_taint_based_eviction(self): - ''' - Add taint "postgres=:NoExecute" to node with master. This must cause a failover. - ''' - k8s = self.k8s - cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - - # get nodes of master and replica(s) (expected target of new master) - current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) - num_replicas = len(current_replica_nodes) - failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) - - # taint node with postgres=:NoExecute to force failover - body = { - "spec": { - "taints": [ - { - "effect": "NoExecute", - "key": "postgres" - } - ] - } - } - - # patch node and test if master is failing over to one of the expected nodes - k8s.api.core_v1.patch_node(current_master_node, body) - new_master_node, new_replica_nodes = self.assert_failover( - current_master_node, num_replicas, failover_targets, cluster_label) - - # add toleration to pods - patch_toleration_config = { - "data": { - "toleration": "key:postgres,operator:Exists,effect:NoExecute" - } - } - k8s.update_config(patch_toleration_config) - - # wait a little before proceeding with the pod distribution test - time.sleep(30) - - # toggle pod anti affinity to move replica away from master node - self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) + # check that new roles are represented in the config by requesting the + # operator configuration via API + operator_pod = k8s.get_operator_pod() + get_config_cmd = "wget --quiet -O - localhost:8080/config" + result = k8s.exec_with_kubectl(operator_pod.metadata.name, get_config_cmd) + roles_dict = (json.loads(result.stdout) + .get("controller", {}) + .get("InfrastructureRoles")) + + self.assertTrue("robot_zmon_acid_monitoring_new" in roles_dict) + role = roles_dict["robot_zmon_acid_monitoring_new"] + role.pop("Password", None) + self.assertDictEqual(role, { + "Name": "robot_zmon_acid_monitoring_new", + "Flags": None, + "MemberOf": ["robot_zmon_new"], + "Parameters": None, + "AdminRole": "", + "Origin": 2, + }) def get_failover_targets(self, master_node, replica_nodes): ''' @@ -820,6 +863,11 @@ def create_with_kubectl(self, path): stdout=subprocess.PIPE, stderr=subprocess.PIPE) + def exec_with_kubectl(self, pod, cmd): + return subprocess.run(["./exec.sh", pod, cmd], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + def get_effective_pod_image(self, pod_name, namespace='default'): ''' Get the Spilo image pod currently uses. In case of lazy rolling updates diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index d1c1b3d17..1210d5015 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -47,7 +47,8 @@ data: # etcd_host: "" # gcp_credentials: "" # kubernetes_use_configmaps: "false" - # infrastructure_roles_secret_name: postgresql-infrastructure-roles + # infrastructure_roles_secret_name: "postgresql-infrastructure-roles" + # infrastructure_roles_secrets: "secretname:monitoring-roles,userkey:user,passwordkey:password,rolekey:inrole" # inherited_labels: application,environment # kube_iam_role: "" # log_s3_bucket: "" diff --git a/manifests/infrastructure-roles-new.yaml b/manifests/infrastructure-roles-new.yaml new file mode 100644 index 000000000..e4f378396 --- /dev/null +++ b/manifests/infrastructure-roles-new.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +data: + # infrastructure role definition in the new format + # robot_zmon_acid_monitoring_new + user: cm9ib3Rfem1vbl9hY2lkX21vbml0b3JpbmdfbmV3 + # robot_zmon_new + role: cm9ib3Rfem1vbl9uZXc= + # foobar_new + password: Zm9vYmFyX25ldw== +kind: Secret +metadata: + name: postgresql-infrastructure-roles-new + namespace: default +type: Opaque diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 2b6e8ae67..55b7653ef 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -127,6 +127,28 @@ spec: type: boolean infrastructure_roles_secret_name: type: string + infrastructure_roles_secrets: + type: array + nullable: true + items: + type: object + required: + - secretname + - userkey + - passwordkey + properties: + secretname: + type: string + userkey: + type: string + passwordkey: + type: string + rolekey: + type: string + details: + type: string + template: + type: boolean inherited_labels: type: array items: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index f7eba1f6c..c0dce42ee 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -39,6 +39,14 @@ configuration: enable_pod_disruption_budget: true enable_sidecars: true # infrastructure_roles_secret_name: "postgresql-infrastructure-roles" + # infrastructure_roles_secrets: + # - secretname: "monitoring-roles" + # userkey: "user" + # passwordkey: "password" + # rolekey: "inrole" + # - secretname: "other-infrastructure-role" + # userkey: "other-user-key" + # passwordkey: "other-password-key" # inherited_labels: # - application # - environment diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 6f907e266..c22ed25c0 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -911,6 +911,35 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "infrastructure_roles_secret_name": { Type: "string", }, + "infrastructure_roles_secrets": { + Type: "array", + Items: &apiextv1beta1.JSONSchemaPropsOrArray{ + Schema: &apiextv1beta1.JSONSchemaProps{ + Type: "object", + Required: []string{"secretname", "userkey", "passwordkey"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "secretname": { + Type: "string", + }, + "userkey": { + Type: "string", + }, + "passwordkey": { + Type: "string", + }, + "rolekey": { + Type: "string", + }, + "details": { + Type: "string", + }, + "template": { + Type: "boolean", + }, + }, + }, + }, + }, "inherited_labels": { Type: "array", Items: &apiextv1beta1.JSONSchemaPropsOrArray{ @@ -983,7 +1012,7 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "spilo_privileged": { Type: "boolean", }, - "storage_resize_mode": { + "storage_resize_mode": { Type: "string", Enum: []apiextv1beta1.JSON{ { diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index e6e13cbd3..ea08f2ff3 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -45,28 +45,29 @@ type PostgresUsersConfiguration struct { type KubernetesMetaConfiguration struct { PodServiceAccountName string `json:"pod_service_account_name,omitempty"` // TODO: change it to the proper json - PodServiceAccountDefinition string `json:"pod_service_account_definition,omitempty"` - PodServiceAccountRoleBindingDefinition string `json:"pod_service_account_role_binding_definition,omitempty"` - PodTerminateGracePeriod Duration `json:"pod_terminate_grace_period,omitempty"` - SpiloPrivileged bool `json:"spilo_privileged,omitempty"` - SpiloFSGroup *int64 `json:"spilo_fsgroup,omitempty"` - WatchedNamespace string `json:"watched_namespace,omitempty"` - PDBNameFormat config.StringTemplate `json:"pdb_name_format,omitempty"` - EnablePodDisruptionBudget *bool `json:"enable_pod_disruption_budget,omitempty"` - StorageResizeMode string `json:"storage_resize_mode,omitempty"` - EnableInitContainers *bool `json:"enable_init_containers,omitempty"` - EnableSidecars *bool `json:"enable_sidecars,omitempty"` - SecretNameTemplate config.StringTemplate `json:"secret_name_template,omitempty"` - ClusterDomain string `json:"cluster_domain,omitempty"` - OAuthTokenSecretName spec.NamespacedName `json:"oauth_token_secret_name,omitempty"` - InfrastructureRolesSecretName spec.NamespacedName `json:"infrastructure_roles_secret_name,omitempty"` - PodRoleLabel string `json:"pod_role_label,omitempty"` - ClusterLabels map[string]string `json:"cluster_labels,omitempty"` - InheritedLabels []string `json:"inherited_labels,omitempty"` - DownscalerAnnotations []string `json:"downscaler_annotations,omitempty"` - ClusterNameLabel string `json:"cluster_name_label,omitempty"` - NodeReadinessLabel map[string]string `json:"node_readiness_label,omitempty"` - CustomPodAnnotations map[string]string `json:"custom_pod_annotations,omitempty"` + PodServiceAccountDefinition string `json:"pod_service_account_definition,omitempty"` + PodServiceAccountRoleBindingDefinition string `json:"pod_service_account_role_binding_definition,omitempty"` + PodTerminateGracePeriod Duration `json:"pod_terminate_grace_period,omitempty"` + SpiloPrivileged bool `json:"spilo_privileged,omitempty"` + SpiloFSGroup *int64 `json:"spilo_fsgroup,omitempty"` + WatchedNamespace string `json:"watched_namespace,omitempty"` + PDBNameFormat config.StringTemplate `json:"pdb_name_format,omitempty"` + EnablePodDisruptionBudget *bool `json:"enable_pod_disruption_budget,omitempty"` + StorageResizeMode string `json:"storage_resize_mode,omitempty"` + EnableInitContainers *bool `json:"enable_init_containers,omitempty"` + EnableSidecars *bool `json:"enable_sidecars,omitempty"` + SecretNameTemplate config.StringTemplate `json:"secret_name_template,omitempty"` + ClusterDomain string `json:"cluster_domain,omitempty"` + OAuthTokenSecretName spec.NamespacedName `json:"oauth_token_secret_name,omitempty"` + InfrastructureRolesSecretName spec.NamespacedName `json:"infrastructure_roles_secret_name,omitempty"` + InfrastructureRolesDefs []*config.InfrastructureRole `json:"infrastructure_roles_secrets,omitempty"` + PodRoleLabel string `json:"pod_role_label,omitempty"` + ClusterLabels map[string]string `json:"cluster_labels,omitempty"` + InheritedLabels []string `json:"inherited_labels,omitempty"` + DownscalerAnnotations []string `json:"downscaler_annotations,omitempty"` + ClusterNameLabel string `json:"cluster_name_label,omitempty"` + NodeReadinessLabel map[string]string `json:"node_readiness_label,omitempty"` + CustomPodAnnotations map[string]string `json:"custom_pod_annotations,omitempty"` // TODO: use a proper toleration structure? PodToleration map[string]string `json:"toleration,omitempty"` PodEnvironmentConfigMap spec.NamespacedName `json:"pod_environment_configmap,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index 064ced184..efc31d6b6 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -27,6 +27,7 @@ SOFTWARE. package v1 import ( + config "github.com/zalando/postgres-operator/pkg/util/config" corev1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -168,6 +169,17 @@ func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfigura } out.OAuthTokenSecretName = in.OAuthTokenSecretName out.InfrastructureRolesSecretName = in.InfrastructureRolesSecretName + if in.InfrastructureRolesDefs != nil { + in, out := &in.InfrastructureRolesDefs, &out.InfrastructureRolesDefs + *out = make([]*config.InfrastructureRole, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(config.InfrastructureRole) + **out = **in + } + } + } if in.ClusterLabels != nil { in, out := &in.ClusterLabels, &out.ClusterLabels *out = make(map[string]string, len(*in)) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 6011d3863..10c817016 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -300,7 +300,8 @@ func (c *Controller) initController() { c.logger.Infof("config: %s", c.opConfig.MustMarshal()) - if infraRoles, err := c.getInfrastructureRoles(&c.opConfig.InfrastructureRolesSecretName); err != nil { + roleDefs := c.getInfrastructureRoleDefinitions() + if infraRoles, err := c.getInfrastructureRoles(roleDefs); err != nil { c.logger.Warningf("could not get infrastructure roles: %v", err) } else { c.config.InfrastructureRoles = infraRoles diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index e2d8636a1..d115aa118 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -71,7 +71,22 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.EnableSidecars = util.CoalesceBool(fromCRD.Kubernetes.EnableSidecars, util.True()) result.SecretNameTemplate = fromCRD.Kubernetes.SecretNameTemplate result.OAuthTokenSecretName = fromCRD.Kubernetes.OAuthTokenSecretName + result.InfrastructureRolesSecretName = fromCRD.Kubernetes.InfrastructureRolesSecretName + if fromCRD.Kubernetes.InfrastructureRolesDefs != nil { + result.InfrastructureRoles = []*config.InfrastructureRole{} + for _, secret := range fromCRD.Kubernetes.InfrastructureRolesDefs { + result.InfrastructureRoles = append( + result.InfrastructureRoles, + &config.InfrastructureRole{ + SecretName: secret.SecretName, + UserKey: secret.UserKey, + RoleKey: secret.RoleKey, + PasswordKey: secret.PasswordKey, + }) + } + } + result.PodRoleLabel = util.Coalesce(fromCRD.Kubernetes.PodRoleLabel, "spilo-role") result.ClusterLabels = util.CoalesceStrMap(fromCRD.Kubernetes.ClusterLabels, map[string]string{"application": "spilo"}) result.InheritedLabels = fromCRD.Kubernetes.InheritedLabels diff --git a/pkg/controller/util.go b/pkg/controller/util.go index 511f02823..6035903dd 100644 --- a/pkg/controller/util.go +++ b/pkg/controller/util.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "strings" v1 "k8s.io/api/core/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" @@ -109,8 +110,161 @@ func readDecodedRole(s string) (*spec.PgUser, error) { return &result, nil } -func (c *Controller) getInfrastructureRoles(rolesSecret *spec.NamespacedName) (map[string]spec.PgUser, error) { - if *rolesSecret == (spec.NamespacedName{}) { +var emptyName = (spec.NamespacedName{}) + +// Return information about what secrets we need to use to create +// infrastructure roles and in which format are they. This is done in +// compatible way, so that the previous logic is not changed, and handles both +// configuration in ConfigMap & CRD. +func (c *Controller) getInfrastructureRoleDefinitions() []*config.InfrastructureRole { + var roleDef config.InfrastructureRole + rolesDefs := c.opConfig.InfrastructureRoles + + if c.opConfig.InfrastructureRolesSecretName == emptyName { + // All the other possibilities require secret name to be present, so if + // it is not, then nothing else to be done here. + return rolesDefs + } + + // check if we can extract something from the configmap config option + if c.opConfig.InfrastructureRolesDefs != "" { + // The configmap option could contain either a role description (in the + // form key1: value1, key2: value2), which has to be used together with + // an old secret name. + + var secretName spec.NamespacedName + var err error + propertySep := "," + valueSep := ":" + + // The field contains the format in which secret is written, let's + // convert it to a proper definition + properties := strings.Split(c.opConfig.InfrastructureRolesDefs, propertySep) + roleDef = config.InfrastructureRole{Template: false} + + for _, property := range properties { + values := strings.Split(property, valueSep) + if len(values) < 2 { + continue + } + name := strings.TrimSpace(values[0]) + value := strings.TrimSpace(values[1]) + + switch name { + case "secretname": + if err = secretName.DecodeWorker(value, "default"); err != nil { + c.logger.Warningf("Could not marshal secret name %s: %v", value, err) + } else { + roleDef.SecretName = secretName + } + case "userkey": + roleDef.UserKey = value + case "passwordkey": + roleDef.PasswordKey = value + case "rolekey": + roleDef.RoleKey = value + default: + c.logger.Warningf("Role description is not known: %s", properties) + } + } + } else { + // At this point we deal with the old format, let's replicate it + // via existing definition structure and remember that it's just a + // template, the real values are in user1,password1,inrole1 etc. + roleDef = config.InfrastructureRole{ + SecretName: c.opConfig.InfrastructureRolesSecretName, + UserKey: "user", + PasswordKey: "password", + RoleKey: "inrole", + Template: true, + } + } + + if roleDef.UserKey != "" && + roleDef.PasswordKey != "" && + roleDef.RoleKey != "" { + rolesDefs = append(rolesDefs, &roleDef) + } + + return rolesDefs +} + +func (c *Controller) getInfrastructureRoles( + rolesSecrets []*config.InfrastructureRole) ( + map[string]spec.PgUser, []error) { + + var errors []error + var noRolesProvided = true + + roles := []spec.PgUser{} + uniqRoles := map[string]spec.PgUser{} + + // To be compatible with the legacy implementation we need to return nil if + // the provided secret name is empty. The equivalent situation in the + // current implementation is an empty rolesSecrets slice or all its items + // are empty. + for _, role := range rolesSecrets { + if role.SecretName != emptyName { + noRolesProvided = false + } + } + + if noRolesProvided { + return nil, nil + } + + for _, secret := range rolesSecrets { + infraRoles, err := c.getInfrastructureRole(secret) + + if err != nil || infraRoles == nil { + c.logger.Debugf("Cannot get infrastructure role: %+v", *secret) + + if err != nil { + errors = append(errors, err) + } + + continue + } + + for _, r := range infraRoles { + roles = append(roles, r) + } + } + + for _, r := range roles { + if _, exists := uniqRoles[r.Name]; exists { + msg := "Conflicting infrastructure roles: roles[%s] = (%q, %q)" + c.logger.Debugf(msg, r.Name, uniqRoles[r.Name], r) + } + + uniqRoles[r.Name] = r + } + + return uniqRoles, errors +} + +// Generate list of users representing one infrastructure role based on its +// description in various K8S objects. An infrastructure role could be +// described by a secret and optionally a config map. The former should contain +// the secret information, i.e. username, password, role. The latter could +// contain an extensive description of the role and even override an +// information obtained from the secret (except a password). +// +// This function returns a list of users to be compatible with the previous +// behaviour, since we don't know how many users are actually encoded in the +// secret if it's a "template" role. If the provided role is not a template +// one, the result would be a list with just one user in it. +// +// FIXME: This dependency on two different objects is rather unnecessary +// complicated, so let's get rid of it via deprecation process. +func (c *Controller) getInfrastructureRole( + infraRole *config.InfrastructureRole) ( + []spec.PgUser, error) { + + rolesSecret := infraRole.SecretName + roles := []spec.PgUser{} + + if rolesSecret == emptyName { // we don't have infrastructure roles defined, bail out return nil, nil } @@ -119,52 +273,98 @@ func (c *Controller) getInfrastructureRoles(rolesSecret *spec.NamespacedName) (m Secrets(rolesSecret.Namespace). Get(context.TODO(), rolesSecret.Name, metav1.GetOptions{}) if err != nil { - c.logger.Debugf("infrastructure roles secret name: %q", *rolesSecret) - return nil, fmt.Errorf("could not get infrastructure roles secret: %v", err) + msg := "could not get infrastructure roles secret %s/%s: %v" + return nil, fmt.Errorf(msg, rolesSecret.Namespace, rolesSecret.Name, err) } secretData := infraRolesSecret.Data - result := make(map[string]spec.PgUser) -Users: - // in worst case we would have one line per user - for i := 1; i <= len(secretData); i++ { - properties := []string{"user", "password", "inrole"} - t := spec.PgUser{Origin: spec.RoleOriginInfrastructure} - for _, p := range properties { - key := fmt.Sprintf("%s%d", p, i) - if val, present := secretData[key]; !present { - if p == "user" { - // exit when the user name with the next sequence id is absent - break Users + + if infraRole.Template { + Users: + for i := 1; i <= len(secretData); i++ { + properties := []string{ + infraRole.UserKey, + infraRole.PasswordKey, + infraRole.RoleKey, + } + t := spec.PgUser{Origin: spec.RoleOriginInfrastructure} + for _, p := range properties { + key := fmt.Sprintf("%s%d", p, i) + if val, present := secretData[key]; !present { + if p == "user" { + // exit when the user name with the next sequence id is + // absent + break Users + } + } else { + s := string(val) + switch p { + case "user": + t.Name = s + case "password": + t.Password = s + case "inrole": + t.MemberOf = append(t.MemberOf, s) + default: + c.logger.Warningf("unknown key %q", p) + } } + // XXX: This is a part of the original implementation, which is + // rather obscure. Why do we delete this key? Wouldn't it be + // used later in comparison for configmap? + delete(secretData, key) + } + + if t.Valid() { + roles = append(roles, t) } else { - s := string(val) - switch p { - case "user": - t.Name = s - case "password": - t.Password = s - case "inrole": - t.MemberOf = append(t.MemberOf, s) - default: - c.logger.Warningf("unknown key %q", p) - } + msg := "infrastructure role %q is not complete and ignored" + c.logger.Warningf(msg, t) } - delete(secretData, key) } + } else { + roleDescr := &spec.PgUser{Origin: spec.RoleOriginInfrastructure} - if t.Name != "" { - if t.Password == "" { - c.logger.Warningf("infrastructure role %q has no password defined and is ignored", t.Name) - continue + if details, exists := secretData[infraRole.Details]; exists { + if err := yaml.Unmarshal(details, &roleDescr); err != nil { + return nil, fmt.Errorf("could not decode yaml role: %v", err) } - result[t.Name] = t + } else { + roleDescr.Name = string(secretData[infraRole.UserKey]) + roleDescr.Password = string(secretData[infraRole.PasswordKey]) + roleDescr.MemberOf = append(roleDescr.MemberOf, string(secretData[infraRole.RoleKey])) + } + + if roleDescr.Valid() { + roles = append(roles, *roleDescr) + } else { + msg := "infrastructure role %q is not complete and ignored" + c.logger.Warningf(msg, roleDescr) + + return nil, nil } + + if roleDescr.Name == "" { + msg := "infrastructure role %q has no name defined and is ignored" + c.logger.Warningf(msg, roleDescr.Name) + return nil, nil + } + + if roleDescr.Password == "" { + msg := "infrastructure role %q has no password defined and is ignored" + c.logger.Warningf(msg, roleDescr.Name) + return nil, nil + } + + roles = append(roles, *roleDescr) } - // perhaps we have some map entries with usernames, passwords, let's check if we have those users in the configmap - if infraRolesMap, err := c.KubeClient.ConfigMaps(rolesSecret.Namespace).Get( - context.TODO(), rolesSecret.Name, metav1.GetOptions{}); err == nil { + // Now plot twist. We need to check if there is a configmap with the same + // name and extract a role description if it exists. + infraRolesMap, err := c.KubeClient. + ConfigMaps(rolesSecret.Namespace). + Get(context.TODO(), rolesSecret.Name, metav1.GetOptions{}) + if err == nil { // we have a configmap with username - json description, let's read and decode it for role, s := range infraRolesMap.Data { roleDescr, err := readDecodedRole(s) @@ -182,20 +382,12 @@ Users: } roleDescr.Name = role roleDescr.Origin = spec.RoleOriginInfrastructure - result[role] = *roleDescr + roles = append(roles, *roleDescr) } } - if len(secretData) > 0 { - c.logger.Warningf("%d unprocessed entries in the infrastructure roles secret,"+ - " checking configmap %v", len(secretData), rolesSecret.Name) - c.logger.Info(`infrastructure role entries should be in the {key}{id} format,` + - ` where {key} can be either of "user", "password", "inrole" and the {id}` + - ` a monotonically increasing integer starting with 1`) - c.logger.Debugf("unprocessed entries: %#v", secretData) - } - - return result, nil + // TODO: check for role collisions + return roles, nil } func (c *Controller) podClusterName(pod *v1.Pod) spec.NamespacedName { diff --git a/pkg/controller/util_test.go b/pkg/controller/util_test.go index ef182248e..fd756a0c7 100644 --- a/pkg/controller/util_test.go +++ b/pkg/controller/util_test.go @@ -8,20 +8,25 @@ import ( b64 "encoding/base64" "github.com/zalando/postgres-operator/pkg/spec" + "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/k8sutil" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( - testInfrastructureRolesSecretName = "infrastructureroles-test" + testInfrastructureRolesOldSecretName = "infrastructureroles-old-test" + testInfrastructureRolesNewSecretName = "infrastructureroles-new-test" ) func newUtilTestController() *Controller { controller := NewController(&spec.ControllerConfig{}, "util-test") controller.opConfig.ClusterNameLabel = "cluster-name" controller.opConfig.InfrastructureRolesSecretName = - spec.NamespacedName{Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesSecretName} + spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesOldSecretName, + } controller.opConfig.Workers = 4 controller.KubeClient = k8sutil.NewMockKubernetesClient() return controller @@ -80,24 +85,32 @@ func TestClusterWorkerID(t *testing.T) { } } -func TestGetInfrastructureRoles(t *testing.T) { +// Test functionality of getting infrastructure roles from their description in +// corresponding secrets. Here we test only common stuff (e.g. when a secret do +// not exist, or empty) and the old format. +func TestOldInfrastructureRoleFormat(t *testing.T) { var testTable = []struct { - secretName spec.NamespacedName - expectedRoles map[string]spec.PgUser - expectedError error + secretName spec.NamespacedName + expectedRoles map[string]spec.PgUser + expectedErrors []error }{ { + // empty secret name spec.NamespacedName{}, nil, nil, }, { + // secret does not exist spec.NamespacedName{Namespace: v1.NamespaceDefault, Name: "null"}, - nil, - fmt.Errorf(`could not get infrastructure roles secret: NotFound`), + map[string]spec.PgUser{}, + []error{fmt.Errorf(`could not get infrastructure roles secret default/null: NotFound`)}, }, { - spec.NamespacedName{Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesSecretName}, + spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesOldSecretName, + }, map[string]spec.PgUser{ "testrole": { Name: "testrole", @@ -116,15 +129,269 @@ func TestGetInfrastructureRoles(t *testing.T) { }, } for _, test := range testTable { - roles, err := utilTestController.getInfrastructureRoles(&test.secretName) - if err != test.expectedError { - if err != nil && test.expectedError != nil && err.Error() == test.expectedError.Error() { - continue + roles, errors := utilTestController.getInfrastructureRoles( + []*config.InfrastructureRole{ + &config.InfrastructureRole{ + SecretName: test.secretName, + UserKey: "user", + PasswordKey: "password", + RoleKey: "inrole", + Template: true, + }, + }) + + if len(errors) != len(test.expectedErrors) { + t.Errorf("expected error '%v' does not match the actual error '%v'", + test.expectedErrors, errors) + } + + for idx := range errors { + err := errors[idx] + expectedErr := test.expectedErrors[idx] + + if err != expectedErr { + if err != nil && expectedErr != nil && err.Error() == expectedErr.Error() { + continue + } + t.Errorf("expected error '%v' does not match the actual error '%v'", + expectedErr, err) } - t.Errorf("expected error '%v' does not match the actual error '%v'", test.expectedError, err) } + if !reflect.DeepEqual(roles, test.expectedRoles) { - t.Errorf("expected roles output %v does not match the actual %v", test.expectedRoles, roles) + t.Errorf("expected roles output %#v does not match the actual %#v", + test.expectedRoles, roles) + } + } +} + +// Test functionality of getting infrastructure roles from their description in +// corresponding secrets. Here we test the new format. +func TestNewInfrastructureRoleFormat(t *testing.T) { + var testTable = []struct { + secrets []spec.NamespacedName + expectedRoles map[string]spec.PgUser + expectedErrors []error + }{ + // one secret with one configmap + { + []spec.NamespacedName{ + spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesNewSecretName, + }, + }, + map[string]spec.PgUser{ + "new-test-role": { + Name: "new-test-role", + Origin: spec.RoleOriginInfrastructure, + Password: "new-test-password", + MemberOf: []string{"new-test-inrole"}, + }, + "new-foobar": { + Name: "new-foobar", + Origin: spec.RoleOriginInfrastructure, + Password: b64.StdEncoding.EncodeToString([]byte("password")), + MemberOf: nil, + Flags: []string{"createdb"}, + }, + }, + nil, + }, + // multiple standalone secrets + { + []spec.NamespacedName{ + spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: "infrastructureroles-new-test1", + }, + spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: "infrastructureroles-new-test2", + }, + }, + map[string]spec.PgUser{ + "new-test-role1": { + Name: "new-test-role1", + Origin: spec.RoleOriginInfrastructure, + Password: "new-test-password1", + MemberOf: []string{"new-test-inrole1"}, + }, + "new-test-role2": { + Name: "new-test-role2", + Origin: spec.RoleOriginInfrastructure, + Password: "new-test-password2", + MemberOf: []string{"new-test-inrole2"}, + }, + }, + nil, + }, + } + for _, test := range testTable { + definitions := []*config.InfrastructureRole{} + for _, secret := range test.secrets { + definitions = append(definitions, &config.InfrastructureRole{ + SecretName: secret, + UserKey: "user", + PasswordKey: "password", + RoleKey: "inrole", + Template: false, + }) + } + + roles, errors := utilTestController.getInfrastructureRoles(definitions) + if len(errors) != len(test.expectedErrors) { + t.Errorf("expected error does not match the actual error:\n%+v\n%+v", + test.expectedErrors, errors) + + // Stop and do not do any further checks + return + } + + for idx := range errors { + err := errors[idx] + expectedErr := test.expectedErrors[idx] + + if err != expectedErr { + if err != nil && expectedErr != nil && err.Error() == expectedErr.Error() { + continue + } + t.Errorf("expected error '%v' does not match the actual error '%v'", + expectedErr, err) + } + } + + if !reflect.DeepEqual(roles, test.expectedRoles) { + t.Errorf("expected roles output/the actual:\n%#v\n%#v", + test.expectedRoles, roles) + } + } +} + +// Tests for getting correct infrastructure roles definitions from present +// configuration. E.g. in which secrets for which roles too look. The biggest +// point here is compatibility of old and new formats of defining +// infrastructure roles. +func TestInfrastructureRoleDefinitions(t *testing.T) { + var testTable = []struct { + rolesDefs []*config.InfrastructureRole + roleSecretName spec.NamespacedName + roleSecrets string + expectedDefs []*config.InfrastructureRole + }{ + // only new format + { + []*config.InfrastructureRole{ + &config.InfrastructureRole{ + SecretName: spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesNewSecretName, + }, + UserKey: "user", + PasswordKey: "password", + RoleKey: "inrole", + Template: false, + }, + }, + spec.NamespacedName{}, + "", + []*config.InfrastructureRole{ + &config.InfrastructureRole{ + SecretName: spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesNewSecretName, + }, + UserKey: "user", + PasswordKey: "password", + RoleKey: "inrole", + Template: false, + }, + }, + }, + // only old format + { + []*config.InfrastructureRole{}, + spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesOldSecretName, + }, + "", + []*config.InfrastructureRole{ + &config.InfrastructureRole{ + SecretName: spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesOldSecretName, + }, + UserKey: "user", + PasswordKey: "password", + RoleKey: "inrole", + Template: true, + }, + }, + }, + // only configmap format + { + []*config.InfrastructureRole{}, + spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesOldSecretName, + }, + "secretname: infrastructureroles-old-test, userkey: test-user, passwordkey: test-password, rolekey: test-role, template: false", + []*config.InfrastructureRole{ + &config.InfrastructureRole{ + SecretName: spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesOldSecretName, + }, + UserKey: "test-user", + PasswordKey: "test-password", + RoleKey: "test-role", + Template: false, + }, + }, + }, + // incorrect configmap format + { + []*config.InfrastructureRole{}, + spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesOldSecretName, + }, + "wrong-format", + []*config.InfrastructureRole{}, + }, + // configmap without a secret + { + []*config.InfrastructureRole{}, + spec.NamespacedName{}, + "userkey: test-user, passwordkey: test-password, rolekey: test-role, template: false", + []*config.InfrastructureRole{}, + }, + } + + for _, test := range testTable { + t.Logf("Test: %+v", test) + utilTestController.opConfig.InfrastructureRoles = test.rolesDefs + utilTestController.opConfig.InfrastructureRolesSecretName = test.roleSecretName + utilTestController.opConfig.InfrastructureRolesDefs = test.roleSecrets + + defs := utilTestController.getInfrastructureRoleDefinitions() + if len(defs) != len(test.expectedDefs) { + t.Errorf("expected definitions does not match the actual:\n%#v\n%#v", + test.expectedDefs, defs) + + // Stop and do not do any further checks + return + } + + for idx := range defs { + def := defs[idx] + expectedDef := test.expectedDefs[idx] + + if !reflect.DeepEqual(def, expectedDef) { + t.Errorf("expected definition/the actual:\n%#v\n%#v", + expectedDef, def) + } } } } diff --git a/pkg/spec/types.go b/pkg/spec/types.go index 08008267b..7a2c0ddac 100644 --- a/pkg/spec/types.go +++ b/pkg/spec/types.go @@ -55,6 +55,10 @@ type PgUser struct { AdminRole string `yaml:"admin_role"` } +func (user *PgUser) Valid() bool { + return user.Name != "" && user.Password != "" +} + // PgUserMap maps user names to the definitions. type PgUserMap map[string]PgUser diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 6cab8af45..5f262107f 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -52,16 +52,42 @@ type Resources struct { ShmVolume *bool `name:"enable_shm_volume" default:"true"` } +type InfrastructureRole struct { + // Name of a secret which describes the role, and optionally name of a + // configmap with an extra information + SecretName spec.NamespacedName + + UserKey string + PasswordKey string + RoleKey string + + // This field point out the detailed yaml definition of the role, if exists + Details string + + // Specify if a secret contains multiple fields in the following format: + // + // %(userkey)idx: ... + // %(passwordkey)idx: ... + // %(rolekey)idx: ... + // + // If it does, Name/Password/Role are interpreted not as unique field + // names, but as a template. + + Template bool +} + // Auth describes authentication specific configuration parameters type Auth struct { - SecretNameTemplate StringTemplate `name:"secret_name_template" default:"{username}.{cluster}.credentials.{tprkind}.{tprgroup}"` - PamRoleName string `name:"pam_role_name" default:"zalandos"` - PamConfiguration string `name:"pam_configuration" default:"https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees"` - TeamsAPIUrl string `name:"teams_api_url" default:"https://teams.example.com/api/"` - OAuthTokenSecretName spec.NamespacedName `name:"oauth_token_secret_name" default:"postgresql-operator"` - InfrastructureRolesSecretName spec.NamespacedName `name:"infrastructure_roles_secret_name"` - SuperUsername string `name:"super_username" default:"postgres"` - ReplicationUsername string `name:"replication_username" default:"standby"` + SecretNameTemplate StringTemplate `name:"secret_name_template" default:"{username}.{cluster}.credentials.{tprkind}.{tprgroup}"` + PamRoleName string `name:"pam_role_name" default:"zalandos"` + PamConfiguration string `name:"pam_configuration" default:"https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees"` + TeamsAPIUrl string `name:"teams_api_url" default:"https://teams.example.com/api/"` + OAuthTokenSecretName spec.NamespacedName `name:"oauth_token_secret_name" default:"postgresql-operator"` + InfrastructureRolesSecretName spec.NamespacedName `name:"infrastructure_roles_secret_name"` + InfrastructureRoles []*InfrastructureRole `name:"-"` + InfrastructureRolesDefs string `name:"infrastructure_roles_secrets"` + SuperUsername string `name:"super_username" default:"postgres"` + ReplicationUsername string `name:"replication_username" default:"standby"` } // Scalyr holds the configuration for the Scalyr Agent sidecar for log shipping: diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index 5cde1c3e8..1234ef74a 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -271,31 +271,73 @@ func SameLogicalBackupJob(cur, new *batchv1beta1.CronJob) (match bool, reason st } func (c *mockSecret) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.Secret, error) { - if name != "infrastructureroles-test" { - return nil, fmt.Errorf("NotFound") - } - secret := &v1.Secret{} - secret.Name = "testcluster" - secret.Data = map[string][]byte{ + oldFormatSecret := &v1.Secret{} + oldFormatSecret.Name = "testcluster" + oldFormatSecret.Data = map[string][]byte{ "user1": []byte("testrole"), "password1": []byte("testpassword"), "inrole1": []byte("testinrole"), "foobar": []byte(b64.StdEncoding.EncodeToString([]byte("password"))), } - return secret, nil + + newFormatSecret := &v1.Secret{} + newFormatSecret.Name = "test-secret-new-format" + newFormatSecret.Data = map[string][]byte{ + "user": []byte("new-test-role"), + "password": []byte("new-test-password"), + "inrole": []byte("new-test-inrole"), + "new-foobar": []byte(b64.StdEncoding.EncodeToString([]byte("password"))), + } + + secrets := map[string]*v1.Secret{ + "infrastructureroles-old-test": oldFormatSecret, + "infrastructureroles-new-test": newFormatSecret, + } + + for idx := 1; idx <= 2; idx++ { + newFormatStandaloneSecret := &v1.Secret{} + newFormatStandaloneSecret.Name = fmt.Sprintf("test-secret-new-format%d", idx) + newFormatStandaloneSecret.Data = map[string][]byte{ + "user": []byte(fmt.Sprintf("new-test-role%d", idx)), + "password": []byte(fmt.Sprintf("new-test-password%d", idx)), + "inrole": []byte(fmt.Sprintf("new-test-inrole%d", idx)), + } + + secrets[fmt.Sprintf("infrastructureroles-new-test%d", idx)] = + newFormatStandaloneSecret + } + + if secret, exists := secrets[name]; exists { + return secret, nil + } + + return nil, fmt.Errorf("NotFound") } func (c *mockConfigMap) Get(ctx context.Context, name string, options metav1.GetOptions) (*v1.ConfigMap, error) { - if name != "infrastructureroles-test" { - return nil, fmt.Errorf("NotFound") - } - configmap := &v1.ConfigMap{} - configmap.Name = "testcluster" - configmap.Data = map[string]string{ + oldFormatConfigmap := &v1.ConfigMap{} + oldFormatConfigmap.Name = "testcluster" + oldFormatConfigmap.Data = map[string]string{ "foobar": "{}", } - return configmap, nil + + newFormatConfigmap := &v1.ConfigMap{} + newFormatConfigmap.Name = "testcluster" + newFormatConfigmap.Data = map[string]string{ + "new-foobar": "{\"user_flags\": [\"createdb\"]}", + } + + configmaps := map[string]*v1.ConfigMap{ + "infrastructureroles-old-test": oldFormatConfigmap, + "infrastructureroles-new-test": newFormatConfigmap, + } + + if configmap, exists := configmaps[name]; exists { + return configmap, nil + } + + return nil, fmt.Errorf("NotFound") } // Secrets to be mocked From 43163cf83b463e5350073a8acb4bdb622a95b9a4 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 10 Aug 2020 15:08:03 +0200 Subject: [PATCH 14/35] allow using both infrastructure_roles_options (#1090) * allow using both infrastructure_roles_options * new default values for user and role definition * use robot_zmon as parent role * add operator log to debug * right name for old secret * only extract if rolesDefs is empty * set password1 in old infrastructure role * fix new infra rile secret * choose different role key for new secret * set memberof everywhere * reenable all tests * reflect feedback * remove condition for rolesDefs --- .../crds/operatorconfigurations.yaml | 4 + e2e/tests/test_e2e.py | 1007 +++++++++-------- manifests/infrastructure-roles-new.yaml | 2 - manifests/infrastructure-roles.yaml | 6 +- manifests/operatorconfiguration.crd.yaml | 4 + pkg/apis/acid.zalan.do/v1/crds.go | 6 + pkg/cluster/resources.go | 2 +- pkg/controller/util.go | 38 +- pkg/controller/util_test.go | 113 +- pkg/util/config/config.go | 3 + 10 files changed, 646 insertions(+), 539 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 3218decd7..4dde1fc23 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -149,6 +149,10 @@ spec: type: string rolekey: type: string + defaultuservalue: + type: string + defaultrolevalue: + type: string details: type: string template: diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 4cd1c6a30..9f0b946c9 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -52,6 +52,7 @@ def setUpClass(cls): for filename in ["operator-service-account-rbac.yaml", "configmap.yaml", "postgres-operator.yaml", + "infrastructure-roles.yaml", "infrastructure-roles-new.yaml"]: result = k8s.create_with_kubectl("manifests/" + filename) print("stdout: {}, stderr: {}".format(result.stdout, result.stderr)) @@ -71,506 +72,506 @@ def setUpClass(cls): print('Operator log: {}'.format(k8s.get_operator_log())) raise - # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - # def test_enable_disable_connection_pooler(self): - # ''' - # For a database without connection pooler, then turns it on, scale up, - # turn off and on again. Test with different ways of doing this (via - # enableConnectionPooler or connectionPooler configuration section). At - # the end turn connection pooler off to not interfere with other tests. - # ''' - # k8s = self.k8s - # service_labels = { - # 'cluster-name': 'acid-minimal-cluster', - # } - # pod_labels = dict({ - # 'connection-pooler': 'acid-minimal-cluster-pooler', - # }) - - # pod_selector = to_selector(pod_labels) - # service_selector = to_selector(service_labels) - - # try: - # # enable connection pooler - # k8s.api.custom_objects_api.patch_namespaced_custom_object( - # 'acid.zalan.do', 'v1', 'default', - # 'postgresqls', 'acid-minimal-cluster', - # { - # 'spec': { - # 'enableConnectionPooler': True, - # } - # }) - # k8s.wait_for_pod_start(pod_selector) - - # pods = k8s.api.core_v1.list_namespaced_pod( - # 'default', label_selector=pod_selector - # ).items - - # self.assertTrue(pods, 'No connection pooler pods') - - # k8s.wait_for_service(service_selector) - # services = k8s.api.core_v1.list_namespaced_service( - # 'default', label_selector=service_selector - # ).items - # services = [ - # s for s in services - # if s.metadata.name.endswith('pooler') - # ] - - # self.assertTrue(services, 'No connection pooler service') - - # # scale up connection pooler deployment - # k8s.api.custom_objects_api.patch_namespaced_custom_object( - # 'acid.zalan.do', 'v1', 'default', - # 'postgresqls', 'acid-minimal-cluster', - # { - # 'spec': { - # 'connectionPooler': { - # 'numberOfInstances': 2, - # }, - # } - # }) - - # k8s.wait_for_running_pods(pod_selector, 2) - - # # turn it off, keeping configuration section - # k8s.api.custom_objects_api.patch_namespaced_custom_object( - # 'acid.zalan.do', 'v1', 'default', - # 'postgresqls', 'acid-minimal-cluster', - # { - # 'spec': { - # 'enableConnectionPooler': False, - # } - # }) - # k8s.wait_for_pods_to_stop(pod_selector) - - # except timeout_decorator.TimeoutError: - # print('Operator log: {}'.format(k8s.get_operator_log())) - # raise - - # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - # def test_enable_load_balancer(self): - # ''' - # Test if services are updated when enabling/disabling load balancers - # ''' - - # k8s = self.k8s - # cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - - # # enable load balancer services - # pg_patch_enable_lbs = { - # "spec": { - # "enableMasterLoadBalancer": True, - # "enableReplicaLoadBalancer": True - # } - # } - # k8s.api.custom_objects_api.patch_namespaced_custom_object( - # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_lbs) - # # wait for service recreation - # time.sleep(60) - - # master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') - # self.assertEqual(master_svc_type, 'LoadBalancer', - # "Expected LoadBalancer service type for master, found {}".format(master_svc_type)) - - # repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') - # self.assertEqual(repl_svc_type, 'LoadBalancer', - # "Expected LoadBalancer service type for replica, found {}".format(repl_svc_type)) - - # # disable load balancer services again - # pg_patch_disable_lbs = { - # "spec": { - # "enableMasterLoadBalancer": False, - # "enableReplicaLoadBalancer": False - # } - # } - # k8s.api.custom_objects_api.patch_namespaced_custom_object( - # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_lbs) - # # wait for service recreation - # time.sleep(60) - - # master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') - # self.assertEqual(master_svc_type, 'ClusterIP', - # "Expected ClusterIP service type for master, found {}".format(master_svc_type)) - - # repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') - # self.assertEqual(repl_svc_type, 'ClusterIP', - # "Expected ClusterIP service type for replica, found {}".format(repl_svc_type)) - - # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - # def test_lazy_spilo_upgrade(self): - # ''' - # Test lazy upgrade for the Spilo image: operator changes a stateful set but lets pods run with the old image - # until they are recreated for reasons other than operator's activity. That works because the operator configures - # stateful sets to use "onDelete" pod update policy. - - # The test covers: - # 1) enabling lazy upgrade in existing operator deployment - # 2) forcing the normal rolling upgrade by changing the operator configmap and restarting its pod - # ''' - - # k8s = self.k8s - - # # update docker image in config and enable the lazy upgrade - # conf_image = "registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p114" - # patch_lazy_spilo_upgrade = { - # "data": { - # "docker_image": conf_image, - # "enable_lazy_spilo_upgrade": "true" - # } - # } - # k8s.update_config(patch_lazy_spilo_upgrade) - - # pod0 = 'acid-minimal-cluster-0' - # pod1 = 'acid-minimal-cluster-1' - - # # restart the pod to get a container with the new image - # k8s.api.core_v1.delete_namespaced_pod(pod0, 'default') - # time.sleep(60) - - # # lazy update works if the restarted pod and older pods run different Spilo versions - # new_image = k8s.get_effective_pod_image(pod0) - # old_image = k8s.get_effective_pod_image(pod1) - # self.assertNotEqual(new_image, old_image, "Lazy updated failed: pods have the same image {}".format(new_image)) - - # # sanity check - # assert_msg = "Image {} of a new pod differs from {} in operator conf".format(new_image, conf_image) - # self.assertEqual(new_image, conf_image, assert_msg) - - # # clean up - # unpatch_lazy_spilo_upgrade = { - # "data": { - # "enable_lazy_spilo_upgrade": "false", - # } - # } - # k8s.update_config(unpatch_lazy_spilo_upgrade) - - # # at this point operator will complete the normal rolling upgrade - # # so we additonally test if disabling the lazy upgrade - forcing the normal rolling upgrade - works - - # # XXX there is no easy way to wait until the end of Sync() - # time.sleep(60) - - # image0 = k8s.get_effective_pod_image(pod0) - # image1 = k8s.get_effective_pod_image(pod1) - - # assert_msg = "Disabling lazy upgrade failed: pods still have different images {} and {}".format(image0, image1) - # self.assertEqual(image0, image1, assert_msg) - - # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - # def test_logical_backup_cron_job(self): - # ''' - # Ensure we can (a) create the cron job at user request for a specific PG cluster - # (b) update the cluster-wide image for the logical backup pod - # (c) delete the job at user request - - # Limitations: - # (a) Does not run the actual batch job because there is no S3 mock to upload backups to - # (b) Assumes 'acid-minimal-cluster' exists as defined in setUp - # ''' - - # k8s = self.k8s - - # # create the cron job - # schedule = "7 7 7 7 *" - # pg_patch_enable_backup = { - # "spec": { - # "enableLogicalBackup": True, - # "logicalBackupSchedule": schedule - # } - # } - # k8s.api.custom_objects_api.patch_namespaced_custom_object( - # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_backup) - # k8s.wait_for_logical_backup_job_creation() - - # jobs = k8s.get_logical_backup_job().items - # self.assertEqual(1, len(jobs), "Expected 1 logical backup job, found {}".format(len(jobs))) - - # job = jobs[0] - # self.assertEqual(job.metadata.name, "logical-backup-acid-minimal-cluster", - # "Expected job name {}, found {}" - # .format("logical-backup-acid-minimal-cluster", job.metadata.name)) - # self.assertEqual(job.spec.schedule, schedule, - # "Expected {} schedule, found {}" - # .format(schedule, job.spec.schedule)) - - # # update the cluster-wide image of the logical backup pod - # image = "test-image-name" - # patch_logical_backup_image = { - # "data": { - # "logical_backup_docker_image": image, - # } - # } - # k8s.update_config(patch_logical_backup_image) - - # jobs = k8s.get_logical_backup_job().items - # actual_image = jobs[0].spec.job_template.spec.template.spec.containers[0].image - # self.assertEqual(actual_image, image, - # "Expected job image {}, found {}".format(image, actual_image)) - - # # delete the logical backup cron job - # pg_patch_disable_backup = { - # "spec": { - # "enableLogicalBackup": False, - # } - # } - # k8s.api.custom_objects_api.patch_namespaced_custom_object( - # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_backup) - # k8s.wait_for_logical_backup_job_deletion() - # jobs = k8s.get_logical_backup_job().items - # self.assertEqual(0, len(jobs), - # "Expected 0 logical backup jobs, found {}".format(len(jobs))) - - # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - # def test_min_resource_limits(self): - # ''' - # Lower resource limits below configured minimum and let operator fix it - # ''' - # k8s = self.k8s - # cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - # labels = 'spilo-role=master,' + cluster_label - # _, failover_targets = k8s.get_pg_nodes(cluster_label) - - # # configure minimum boundaries for CPU and memory limits - # minCPULimit = '500m' - # minMemoryLimit = '500Mi' - # patch_min_resource_limits = { - # "data": { - # "min_cpu_limit": minCPULimit, - # "min_memory_limit": minMemoryLimit - # } - # } - # k8s.update_config(patch_min_resource_limits) - - # # lower resource limits below minimum - # pg_patch_resources = { - # "spec": { - # "resources": { - # "requests": { - # "cpu": "10m", - # "memory": "50Mi" - # }, - # "limits": { - # "cpu": "200m", - # "memory": "200Mi" - # } - # } - # } - # } - # k8s.api.custom_objects_api.patch_namespaced_custom_object( - # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources) - # k8s.wait_for_pod_failover(failover_targets, labels) - # k8s.wait_for_pod_start('spilo-role=replica') - - # pods = k8s.api.core_v1.list_namespaced_pod( - # 'default', label_selector=labels).items - # self.assert_master_is_unique() - # masterPod = pods[0] - - # self.assertEqual(masterPod.spec.containers[0].resources.limits['cpu'], minCPULimit, - # "Expected CPU limit {}, found {}" - # .format(minCPULimit, masterPod.spec.containers[0].resources.limits['cpu'])) - # self.assertEqual(masterPod.spec.containers[0].resources.limits['memory'], minMemoryLimit, - # "Expected memory limit {}, found {}" - # .format(minMemoryLimit, masterPod.spec.containers[0].resources.limits['memory'])) - - # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - # def test_multi_namespace_support(self): - # ''' - # Create a customized Postgres cluster in a non-default namespace. - # ''' - # k8s = self.k8s - - # with open("manifests/complete-postgres-manifest.yaml", 'r+') as f: - # pg_manifest = yaml.safe_load(f) - # pg_manifest["metadata"]["namespace"] = self.namespace - # yaml.dump(pg_manifest, f, Dumper=yaml.Dumper) - - # k8s.create_with_kubectl("manifests/complete-postgres-manifest.yaml") - # k8s.wait_for_pod_start("spilo-role=master", self.namespace) - # self.assert_master_is_unique(self.namespace, "acid-test-cluster") - - # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - # def test_node_readiness_label(self): - # ''' - # Remove node readiness label from master node. This must cause a failover. - # ''' - # k8s = self.k8s - # cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - # readiness_label = 'lifecycle-status' - # readiness_value = 'ready' - - # # get nodes of master and replica(s) (expected target of new master) - # current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) - # num_replicas = len(current_replica_nodes) - # failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) - - # # add node_readiness_label to potential failover nodes - # patch_readiness_label = { - # "metadata": { - # "labels": { - # readiness_label: readiness_value - # } - # } - # } - # for failover_target in failover_targets: - # k8s.api.core_v1.patch_node(failover_target, patch_readiness_label) - - # # define node_readiness_label in config map which should trigger a failover of the master - # patch_readiness_label_config = { - # "data": { - # "node_readiness_label": readiness_label + ':' + readiness_value, - # } - # } - # k8s.update_config(patch_readiness_label_config) - # new_master_node, new_replica_nodes = self.assert_failover( - # current_master_node, num_replicas, failover_targets, cluster_label) - - # # patch also node where master ran before - # k8s.api.core_v1.patch_node(current_master_node, patch_readiness_label) - - # # wait a little before proceeding with the pod distribution test - # time.sleep(30) - - # # toggle pod anti affinity to move replica away from master node - # self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) - - # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - # def test_scaling(self): - # ''' - # Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime. - # ''' - # k8s = self.k8s - # labels = "application=spilo,cluster-name=acid-minimal-cluster" - - # k8s.wait_for_pg_to_scale(3) - # self.assertEqual(3, k8s.count_pods_with_label(labels)) - # self.assert_master_is_unique() - - # k8s.wait_for_pg_to_scale(2) - # self.assertEqual(2, k8s.count_pods_with_label(labels)) - # self.assert_master_is_unique() - - # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - # def test_service_annotations(self): - # ''' - # Create a Postgres cluster with service annotations and check them. - # ''' - # k8s = self.k8s - # patch_custom_service_annotations = { - # "data": { - # "custom_service_annotations": "foo:bar", - # } - # } - # k8s.update_config(patch_custom_service_annotations) - - # pg_patch_custom_annotations = { - # "spec": { - # "serviceAnnotations": { - # "annotation.key": "value", - # "foo": "bar", - # } - # } - # } - # k8s.api.custom_objects_api.patch_namespaced_custom_object( - # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_custom_annotations) - - # # wait a little before proceeding - # time.sleep(30) - # annotations = { - # "annotation.key": "value", - # "foo": "bar", - # } - # self.assertTrue(k8s.check_service_annotations( - # "cluster-name=acid-minimal-cluster,spilo-role=master", annotations)) - # self.assertTrue(k8s.check_service_annotations( - # "cluster-name=acid-minimal-cluster,spilo-role=replica", annotations)) - - # # clean up - # unpatch_custom_service_annotations = { - # "data": { - # "custom_service_annotations": "", - # } - # } - # k8s.update_config(unpatch_custom_service_annotations) - - # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - # def test_statefulset_annotation_propagation(self): - # ''' - # Inject annotation to Postgresql CRD and check it's propagation to stateful set - # ''' - # k8s = self.k8s - # cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - - # patch_sset_propagate_annotations = { - # "data": { - # "downscaler_annotations": "deployment-time,downscaler/*", - # } - # } - # k8s.update_config(patch_sset_propagate_annotations) - - # pg_crd_annotations = { - # "metadata": { - # "annotations": { - # "deployment-time": "2020-04-30 12:00:00", - # "downscaler/downtime_replicas": "0", - # }, - # } - # } - # k8s.api.custom_objects_api.patch_namespaced_custom_object( - # "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_crd_annotations) - - # # wait a little before proceeding - # time.sleep(60) - # annotations = { - # "deployment-time": "2020-04-30 12:00:00", - # "downscaler/downtime_replicas": "0", - # } - # self.assertTrue(k8s.check_statefulset_annotations(cluster_label, annotations)) - - # @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - # def test_taint_based_eviction(self): - # ''' - # Add taint "postgres=:NoExecute" to node with master. This must cause a failover. - # ''' - # k8s = self.k8s - # cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - - # # get nodes of master and replica(s) (expected target of new master) - # current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) - # num_replicas = len(current_replica_nodes) - # failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) - - # # taint node with postgres=:NoExecute to force failover - # body = { - # "spec": { - # "taints": [ - # { - # "effect": "NoExecute", - # "key": "postgres" - # } - # ] - # } - # } - - # # patch node and test if master is failing over to one of the expected nodes - # k8s.api.core_v1.patch_node(current_master_node, body) - # new_master_node, new_replica_nodes = self.assert_failover( - # current_master_node, num_replicas, failover_targets, cluster_label) - - # # add toleration to pods - # patch_toleration_config = { - # "data": { - # "toleration": "key:postgres,operator:Exists,effect:NoExecute" - # } - # } - # k8s.update_config(patch_toleration_config) - - # # wait a little before proceeding with the pod distribution test - # time.sleep(30) - - # # toggle pod anti affinity to move replica away from master node - # self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_enable_disable_connection_pooler(self): + ''' + For a database without connection pooler, then turns it on, scale up, + turn off and on again. Test with different ways of doing this (via + enableConnectionPooler or connectionPooler configuration section). At + the end turn connection pooler off to not interfere with other tests. + ''' + k8s = self.k8s + service_labels = { + 'cluster-name': 'acid-minimal-cluster', + } + pod_labels = dict({ + 'connection-pooler': 'acid-minimal-cluster-pooler', + }) + + pod_selector = to_selector(pod_labels) + service_selector = to_selector(service_labels) + + try: + # enable connection pooler + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPooler': True, + } + }) + k8s.wait_for_pod_start(pod_selector) + + pods = k8s.api.core_v1.list_namespaced_pod( + 'default', label_selector=pod_selector + ).items + + self.assertTrue(pods, 'No connection pooler pods') + + k8s.wait_for_service(service_selector) + services = k8s.api.core_v1.list_namespaced_service( + 'default', label_selector=service_selector + ).items + services = [ + s for s in services + if s.metadata.name.endswith('pooler') + ] + + self.assertTrue(services, 'No connection pooler service') + + # scale up connection pooler deployment + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'connectionPooler': { + 'numberOfInstances': 2, + }, + } + }) + + k8s.wait_for_running_pods(pod_selector, 2) + + # turn it off, keeping configuration section + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPooler': False, + } + }) + k8s.wait_for_pods_to_stop(pod_selector) + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_enable_load_balancer(self): + ''' + Test if services are updated when enabling/disabling load balancers + ''' + + k8s = self.k8s + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + + # enable load balancer services + pg_patch_enable_lbs = { + "spec": { + "enableMasterLoadBalancer": True, + "enableReplicaLoadBalancer": True + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_lbs) + # wait for service recreation + time.sleep(60) + + master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') + self.assertEqual(master_svc_type, 'LoadBalancer', + "Expected LoadBalancer service type for master, found {}".format(master_svc_type)) + + repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') + self.assertEqual(repl_svc_type, 'LoadBalancer', + "Expected LoadBalancer service type for replica, found {}".format(repl_svc_type)) + + # disable load balancer services again + pg_patch_disable_lbs = { + "spec": { + "enableMasterLoadBalancer": False, + "enableReplicaLoadBalancer": False + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_lbs) + # wait for service recreation + time.sleep(60) + + master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') + self.assertEqual(master_svc_type, 'ClusterIP', + "Expected ClusterIP service type for master, found {}".format(master_svc_type)) + + repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') + self.assertEqual(repl_svc_type, 'ClusterIP', + "Expected ClusterIP service type for replica, found {}".format(repl_svc_type)) + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_lazy_spilo_upgrade(self): + ''' + Test lazy upgrade for the Spilo image: operator changes a stateful set but lets pods run with the old image + until they are recreated for reasons other than operator's activity. That works because the operator configures + stateful sets to use "onDelete" pod update policy. + + The test covers: + 1) enabling lazy upgrade in existing operator deployment + 2) forcing the normal rolling upgrade by changing the operator configmap and restarting its pod + ''' + + k8s = self.k8s + + # update docker image in config and enable the lazy upgrade + conf_image = "registry.opensource.zalan.do/acid/spilo-cdp-12:1.6-p114" + patch_lazy_spilo_upgrade = { + "data": { + "docker_image": conf_image, + "enable_lazy_spilo_upgrade": "true" + } + } + k8s.update_config(patch_lazy_spilo_upgrade) + + pod0 = 'acid-minimal-cluster-0' + pod1 = 'acid-minimal-cluster-1' + + # restart the pod to get a container with the new image + k8s.api.core_v1.delete_namespaced_pod(pod0, 'default') + time.sleep(60) + + # lazy update works if the restarted pod and older pods run different Spilo versions + new_image = k8s.get_effective_pod_image(pod0) + old_image = k8s.get_effective_pod_image(pod1) + self.assertNotEqual(new_image, old_image, "Lazy updated failed: pods have the same image {}".format(new_image)) + + # sanity check + assert_msg = "Image {} of a new pod differs from {} in operator conf".format(new_image, conf_image) + self.assertEqual(new_image, conf_image, assert_msg) + + # clean up + unpatch_lazy_spilo_upgrade = { + "data": { + "enable_lazy_spilo_upgrade": "false", + } + } + k8s.update_config(unpatch_lazy_spilo_upgrade) + + # at this point operator will complete the normal rolling upgrade + # so we additonally test if disabling the lazy upgrade - forcing the normal rolling upgrade - works + + # XXX there is no easy way to wait until the end of Sync() + time.sleep(60) + + image0 = k8s.get_effective_pod_image(pod0) + image1 = k8s.get_effective_pod_image(pod1) + + assert_msg = "Disabling lazy upgrade failed: pods still have different images {} and {}".format(image0, image1) + self.assertEqual(image0, image1, assert_msg) + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_logical_backup_cron_job(self): + ''' + Ensure we can (a) create the cron job at user request for a specific PG cluster + (b) update the cluster-wide image for the logical backup pod + (c) delete the job at user request + + Limitations: + (a) Does not run the actual batch job because there is no S3 mock to upload backups to + (b) Assumes 'acid-minimal-cluster' exists as defined in setUp + ''' + + k8s = self.k8s + + # create the cron job + schedule = "7 7 7 7 *" + pg_patch_enable_backup = { + "spec": { + "enableLogicalBackup": True, + "logicalBackupSchedule": schedule + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_backup) + k8s.wait_for_logical_backup_job_creation() + + jobs = k8s.get_logical_backup_job().items + self.assertEqual(1, len(jobs), "Expected 1 logical backup job, found {}".format(len(jobs))) + + job = jobs[0] + self.assertEqual(job.metadata.name, "logical-backup-acid-minimal-cluster", + "Expected job name {}, found {}" + .format("logical-backup-acid-minimal-cluster", job.metadata.name)) + self.assertEqual(job.spec.schedule, schedule, + "Expected {} schedule, found {}" + .format(schedule, job.spec.schedule)) + + # update the cluster-wide image of the logical backup pod + image = "test-image-name" + patch_logical_backup_image = { + "data": { + "logical_backup_docker_image": image, + } + } + k8s.update_config(patch_logical_backup_image) + + jobs = k8s.get_logical_backup_job().items + actual_image = jobs[0].spec.job_template.spec.template.spec.containers[0].image + self.assertEqual(actual_image, image, + "Expected job image {}, found {}".format(image, actual_image)) + + # delete the logical backup cron job + pg_patch_disable_backup = { + "spec": { + "enableLogicalBackup": False, + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_backup) + k8s.wait_for_logical_backup_job_deletion() + jobs = k8s.get_logical_backup_job().items + self.assertEqual(0, len(jobs), + "Expected 0 logical backup jobs, found {}".format(len(jobs))) + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_min_resource_limits(self): + ''' + Lower resource limits below configured minimum and let operator fix it + ''' + k8s = self.k8s + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + labels = 'spilo-role=master,' + cluster_label + _, failover_targets = k8s.get_pg_nodes(cluster_label) + + # configure minimum boundaries for CPU and memory limits + minCPULimit = '500m' + minMemoryLimit = '500Mi' + patch_min_resource_limits = { + "data": { + "min_cpu_limit": minCPULimit, + "min_memory_limit": minMemoryLimit + } + } + k8s.update_config(patch_min_resource_limits) + + # lower resource limits below minimum + pg_patch_resources = { + "spec": { + "resources": { + "requests": { + "cpu": "10m", + "memory": "50Mi" + }, + "limits": { + "cpu": "200m", + "memory": "200Mi" + } + } + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources) + k8s.wait_for_pod_failover(failover_targets, labels) + k8s.wait_for_pod_start('spilo-role=replica') + + pods = k8s.api.core_v1.list_namespaced_pod( + 'default', label_selector=labels).items + self.assert_master_is_unique() + masterPod = pods[0] + + self.assertEqual(masterPod.spec.containers[0].resources.limits['cpu'], minCPULimit, + "Expected CPU limit {}, found {}" + .format(minCPULimit, masterPod.spec.containers[0].resources.limits['cpu'])) + self.assertEqual(masterPod.spec.containers[0].resources.limits['memory'], minMemoryLimit, + "Expected memory limit {}, found {}" + .format(minMemoryLimit, masterPod.spec.containers[0].resources.limits['memory'])) + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_multi_namespace_support(self): + ''' + Create a customized Postgres cluster in a non-default namespace. + ''' + k8s = self.k8s + + with open("manifests/complete-postgres-manifest.yaml", 'r+') as f: + pg_manifest = yaml.safe_load(f) + pg_manifest["metadata"]["namespace"] = self.namespace + yaml.dump(pg_manifest, f, Dumper=yaml.Dumper) + + k8s.create_with_kubectl("manifests/complete-postgres-manifest.yaml") + k8s.wait_for_pod_start("spilo-role=master", self.namespace) + self.assert_master_is_unique(self.namespace, "acid-test-cluster") + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_node_readiness_label(self): + ''' + Remove node readiness label from master node. This must cause a failover. + ''' + k8s = self.k8s + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + readiness_label = 'lifecycle-status' + readiness_value = 'ready' + + # get nodes of master and replica(s) (expected target of new master) + current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) + num_replicas = len(current_replica_nodes) + failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) + + # add node_readiness_label to potential failover nodes + patch_readiness_label = { + "metadata": { + "labels": { + readiness_label: readiness_value + } + } + } + for failover_target in failover_targets: + k8s.api.core_v1.patch_node(failover_target, patch_readiness_label) + + # define node_readiness_label in config map which should trigger a failover of the master + patch_readiness_label_config = { + "data": { + "node_readiness_label": readiness_label + ':' + readiness_value, + } + } + k8s.update_config(patch_readiness_label_config) + new_master_node, new_replica_nodes = self.assert_failover( + current_master_node, num_replicas, failover_targets, cluster_label) + + # patch also node where master ran before + k8s.api.core_v1.patch_node(current_master_node, patch_readiness_label) + + # wait a little before proceeding with the pod distribution test + time.sleep(30) + + # toggle pod anti affinity to move replica away from master node + self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_scaling(self): + ''' + Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime. + ''' + k8s = self.k8s + labels = "application=spilo,cluster-name=acid-minimal-cluster" + + k8s.wait_for_pg_to_scale(3) + self.assertEqual(3, k8s.count_pods_with_label(labels)) + self.assert_master_is_unique() + + k8s.wait_for_pg_to_scale(2) + self.assertEqual(2, k8s.count_pods_with_label(labels)) + self.assert_master_is_unique() + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_service_annotations(self): + ''' + Create a Postgres cluster with service annotations and check them. + ''' + k8s = self.k8s + patch_custom_service_annotations = { + "data": { + "custom_service_annotations": "foo:bar", + } + } + k8s.update_config(patch_custom_service_annotations) + + pg_patch_custom_annotations = { + "spec": { + "serviceAnnotations": { + "annotation.key": "value", + "foo": "bar", + } + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_custom_annotations) + + # wait a little before proceeding + time.sleep(30) + annotations = { + "annotation.key": "value", + "foo": "bar", + } + self.assertTrue(k8s.check_service_annotations( + "cluster-name=acid-minimal-cluster,spilo-role=master", annotations)) + self.assertTrue(k8s.check_service_annotations( + "cluster-name=acid-minimal-cluster,spilo-role=replica", annotations)) + + # clean up + unpatch_custom_service_annotations = { + "data": { + "custom_service_annotations": "", + } + } + k8s.update_config(unpatch_custom_service_annotations) + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_statefulset_annotation_propagation(self): + ''' + Inject annotation to Postgresql CRD and check it's propagation to stateful set + ''' + k8s = self.k8s + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + + patch_sset_propagate_annotations = { + "data": { + "downscaler_annotations": "deployment-time,downscaler/*", + } + } + k8s.update_config(patch_sset_propagate_annotations) + + pg_crd_annotations = { + "metadata": { + "annotations": { + "deployment-time": "2020-04-30 12:00:00", + "downscaler/downtime_replicas": "0", + }, + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_crd_annotations) + + # wait a little before proceeding + time.sleep(60) + annotations = { + "deployment-time": "2020-04-30 12:00:00", + "downscaler/downtime_replicas": "0", + } + self.assertTrue(k8s.check_statefulset_annotations(cluster_label, annotations)) + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_taint_based_eviction(self): + ''' + Add taint "postgres=:NoExecute" to node with master. This must cause a failover. + ''' + k8s = self.k8s + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + + # get nodes of master and replica(s) (expected target of new master) + current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) + num_replicas = len(current_replica_nodes) + failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) + + # taint node with postgres=:NoExecute to force failover + body = { + "spec": { + "taints": [ + { + "effect": "NoExecute", + "key": "postgres" + } + ] + } + } + + # patch node and test if master is failing over to one of the expected nodes + k8s.api.core_v1.patch_node(current_master_node, body) + new_master_node, new_replica_nodes = self.assert_failover( + current_master_node, num_replicas, failover_targets, cluster_label) + + # add toleration to pods + patch_toleration_config = { + "data": { + "toleration": "key:postgres,operator:Exists,effect:NoExecute" + } + } + k8s.update_config(patch_toleration_config) + + # wait a little before proceeding with the pod distribution test + time.sleep(30) + + # toggle pod anti affinity to move replica away from master node + self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_infrastructure_roles(self): @@ -579,8 +580,8 @@ def test_infrastructure_roles(self): ''' k8s = self.k8s # update infrastructure roles description - secret_name = "postgresql-infrastructure-roles-old" - roles = "secretname: postgresql-infrastructure-roles-new, userkey: user, rolekey: role, passwordkey: password" + secret_name = "postgresql-infrastructure-roles" + roles = "secretname: postgresql-infrastructure-roles-new, userkey: user, rolekey: memberof, passwordkey: password, defaultrolevalue: robot_zmon" patch_infrastructure_roles = { "data": { "infrastructure_roles_secret_name": secret_name, @@ -607,7 +608,7 @@ def test_infrastructure_roles(self): self.assertDictEqual(role, { "Name": "robot_zmon_acid_monitoring_new", "Flags": None, - "MemberOf": ["robot_zmon_new"], + "MemberOf": ["robot_zmon"], "Parameters": None, "AdminRole": "", "Origin": 2, diff --git a/manifests/infrastructure-roles-new.yaml b/manifests/infrastructure-roles-new.yaml index e4f378396..64b854c6a 100644 --- a/manifests/infrastructure-roles-new.yaml +++ b/manifests/infrastructure-roles-new.yaml @@ -3,8 +3,6 @@ data: # infrastructure role definition in the new format # robot_zmon_acid_monitoring_new user: cm9ib3Rfem1vbl9hY2lkX21vbml0b3JpbmdfbmV3 - # robot_zmon_new - role: cm9ib3Rfem1vbl9uZXc= # foobar_new password: Zm9vYmFyX25ldw== kind: Secret diff --git a/manifests/infrastructure-roles.yaml b/manifests/infrastructure-roles.yaml index 3c2d86850..c66d79139 100644 --- a/manifests/infrastructure-roles.yaml +++ b/manifests/infrastructure-roles.yaml @@ -7,12 +7,14 @@ data: # provide other options in the configmap. # robot_zmon_acid_monitoring user1: cm9ib3Rfem1vbl9hY2lkX21vbml0b3Jpbmc= + # foobar + password1: Zm9vYmFy # robot_zmon inrole1: cm9ib3Rfem1vbg== # testuser user2: dGVzdHVzZXI= - # foobar - password2: Zm9vYmFy + # testpassword + password2: dGVzdHBhc3N3b3Jk # user batman with the password justice # look for other fields in the infrastructure roles configmap batman: anVzdGljZQ== diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 55b7653ef..95c4678a8 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -145,6 +145,10 @@ spec: type: string rolekey: type: string + defaultuservalue: + type: string + defaultrolevalue: + type: string details: type: string template: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index c22ed25c0..b5695bb4e 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -930,6 +930,12 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "rolekey": { Type: "string", }, + "defaultuservalue": { + Type: "string", + }, + "defaultrolevalue": { + Type: "string", + }, "details": { Type: "string", }, diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index c75457a5a..3066b78c6 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -207,7 +207,7 @@ func (c *Cluster) deleteConnectionPooler() (err error) { serviceName = service.Name } - // set delete propagation policy to foreground, so that all the dependant + // set delete propagation policy to foreground, so that all the dependent // will be deleted. err = c.KubeClient. Services(c.Namespace). diff --git a/pkg/controller/util.go b/pkg/controller/util.go index 6035903dd..e460db2a5 100644 --- a/pkg/controller/util.go +++ b/pkg/controller/util.go @@ -15,6 +15,7 @@ import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/cluster" "github.com/zalando/postgres-operator/pkg/spec" + "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/k8sutil" "gopkg.in/yaml.v2" @@ -118,13 +119,9 @@ var emptyName = (spec.NamespacedName{}) // configuration in ConfigMap & CRD. func (c *Controller) getInfrastructureRoleDefinitions() []*config.InfrastructureRole { var roleDef config.InfrastructureRole - rolesDefs := c.opConfig.InfrastructureRoles - if c.opConfig.InfrastructureRolesSecretName == emptyName { - // All the other possibilities require secret name to be present, so if - // it is not, then nothing else to be done here. - return rolesDefs - } + // take from CRD configuration + rolesDefs := c.opConfig.InfrastructureRoles // check if we can extract something from the configmap config option if c.opConfig.InfrastructureRolesDefs != "" { @@ -163,27 +160,33 @@ func (c *Controller) getInfrastructureRoleDefinitions() []*config.Infrastructure roleDef.PasswordKey = value case "rolekey": roleDef.RoleKey = value + case "defaultuservalue": + roleDef.DefaultUserValue = value + case "defaultrolevalue": + roleDef.DefaultRoleValue = value default: c.logger.Warningf("Role description is not known: %s", properties) } } - } else { + + if roleDef.SecretName != emptyName && + (roleDef.UserKey != "" || roleDef.DefaultUserValue != "") && + roleDef.PasswordKey != "" { + rolesDefs = append(rolesDefs, &roleDef) + } + } + + if c.opConfig.InfrastructureRolesSecretName != emptyName { // At this point we deal with the old format, let's replicate it // via existing definition structure and remember that it's just a // template, the real values are in user1,password1,inrole1 etc. - roleDef = config.InfrastructureRole{ + rolesDefs = append(rolesDefs, &config.InfrastructureRole{ SecretName: c.opConfig.InfrastructureRolesSecretName, UserKey: "user", PasswordKey: "password", RoleKey: "inrole", Template: true, - } - } - - if roleDef.UserKey != "" && - roleDef.PasswordKey != "" && - roleDef.RoleKey != "" { - rolesDefs = append(rolesDefs, &roleDef) + }) } return rolesDefs @@ -330,9 +333,10 @@ func (c *Controller) getInfrastructureRole( return nil, fmt.Errorf("could not decode yaml role: %v", err) } } else { - roleDescr.Name = string(secretData[infraRole.UserKey]) + roleDescr.Name = util.Coalesce(string(secretData[infraRole.UserKey]), infraRole.DefaultUserValue) roleDescr.Password = string(secretData[infraRole.PasswordKey]) - roleDescr.MemberOf = append(roleDescr.MemberOf, string(secretData[infraRole.RoleKey])) + roleDescr.MemberOf = append(roleDescr.MemberOf, + util.Coalesce(string(secretData[infraRole.RoleKey]), infraRole.DefaultRoleValue)) } if roleDescr.Valid() { diff --git a/pkg/controller/util_test.go b/pkg/controller/util_test.go index fd756a0c7..edc05d67e 100644 --- a/pkg/controller/util_test.go +++ b/pkg/controller/util_test.go @@ -279,7 +279,7 @@ func TestInfrastructureRoleDefinitions(t *testing.T) { roleSecrets string expectedDefs []*config.InfrastructureRole }{ - // only new format + // only new CRD format { []*config.InfrastructureRole{ &config.InfrastructureRole{ @@ -287,9 +287,9 @@ func TestInfrastructureRoleDefinitions(t *testing.T) { Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesNewSecretName, }, - UserKey: "user", - PasswordKey: "password", - RoleKey: "inrole", + UserKey: "test-user", + PasswordKey: "test-password", + RoleKey: "test-role", Template: false, }, }, @@ -301,14 +301,50 @@ func TestInfrastructureRoleDefinitions(t *testing.T) { Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesNewSecretName, }, - UserKey: "user", - PasswordKey: "password", - RoleKey: "inrole", + UserKey: "test-user", + PasswordKey: "test-password", + RoleKey: "test-role", Template: false, }, }, }, - // only old format + // only new configmap format + { + []*config.InfrastructureRole{}, + spec.NamespacedName{}, + "secretname: infrastructureroles-new-test, userkey: test-user, passwordkey: test-password, rolekey: test-role", + []*config.InfrastructureRole{ + &config.InfrastructureRole{ + SecretName: spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesNewSecretName, + }, + UserKey: "test-user", + PasswordKey: "test-password", + RoleKey: "test-role", + Template: false, + }, + }, + }, + // new configmap format with defaultRoleValue + { + []*config.InfrastructureRole{}, + spec.NamespacedName{}, + "secretname: infrastructureroles-new-test, userkey: test-user, passwordkey: test-password, defaultrolevalue: test-role", + []*config.InfrastructureRole{ + &config.InfrastructureRole{ + SecretName: spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesNewSecretName, + }, + UserKey: "test-user", + PasswordKey: "test-password", + DefaultRoleValue: "test-role", + Template: false, + }, + }, + }, + // only old CRD and configmap format { []*config.InfrastructureRole{}, spec.NamespacedName{ @@ -329,34 +365,83 @@ func TestInfrastructureRoleDefinitions(t *testing.T) { }, }, }, - // only configmap format + // both formats for CRD { - []*config.InfrastructureRole{}, + []*config.InfrastructureRole{ + &config.InfrastructureRole{ + SecretName: spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesNewSecretName, + }, + UserKey: "test-user", + PasswordKey: "test-password", + RoleKey: "test-role", + Template: false, + }, + }, spec.NamespacedName{ Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesOldSecretName, }, - "secretname: infrastructureroles-old-test, userkey: test-user, passwordkey: test-password, rolekey: test-role, template: false", + "", []*config.InfrastructureRole{ &config.InfrastructureRole{ SecretName: spec.NamespacedName{ Namespace: v1.NamespaceDefault, - Name: testInfrastructureRolesOldSecretName, + Name: testInfrastructureRolesNewSecretName, }, UserKey: "test-user", PasswordKey: "test-password", RoleKey: "test-role", Template: false, }, + &config.InfrastructureRole{ + SecretName: spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesOldSecretName, + }, + UserKey: "user", + PasswordKey: "password", + RoleKey: "inrole", + Template: true, + }, }, }, - // incorrect configmap format + // both formats for configmap { []*config.InfrastructureRole{}, spec.NamespacedName{ Namespace: v1.NamespaceDefault, Name: testInfrastructureRolesOldSecretName, }, + "secretname: infrastructureroles-new-test, userkey: test-user, passwordkey: test-password, rolekey: test-role", + []*config.InfrastructureRole{ + &config.InfrastructureRole{ + SecretName: spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesNewSecretName, + }, + UserKey: "test-user", + PasswordKey: "test-password", + RoleKey: "test-role", + Template: false, + }, + &config.InfrastructureRole{ + SecretName: spec.NamespacedName{ + Namespace: v1.NamespaceDefault, + Name: testInfrastructureRolesOldSecretName, + }, + UserKey: "user", + PasswordKey: "password", + RoleKey: "inrole", + Template: true, + }, + }, + }, + // incorrect configmap format + { + []*config.InfrastructureRole{}, + spec.NamespacedName{}, "wrong-format", []*config.InfrastructureRole{}, }, @@ -364,7 +449,7 @@ func TestInfrastructureRoleDefinitions(t *testing.T) { { []*config.InfrastructureRole{}, spec.NamespacedName{}, - "userkey: test-user, passwordkey: test-password, rolekey: test-role, template: false", + "userkey: test-user, passwordkey: test-password, rolekey: test-role", []*config.InfrastructureRole{}, }, } diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 5f262107f..4fe66910a 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -61,6 +61,9 @@ type InfrastructureRole struct { PasswordKey string RoleKey string + DefaultUserValue string + DefaultRoleValue string + // This field point out the detailed yaml definition of the role, if exists Details string From 0508266219c9aa8235862ecaa089e082aa97672c Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Mon, 10 Aug 2020 18:26:26 +0200 Subject: [PATCH 15/35] Remove all secrets on delete incl. pooler (#1091) * fix syncSecrets and remove pooler secret * update log for deleteSecret * use c.credentialSecretName(username) * minor fix --- go.mod | 3 ++- go.sum | 6 ++++-- pkg/cluster/cluster.go | 2 +- pkg/cluster/resources.go | 38 +++++++++++++++++++++++++++++++------- pkg/cluster/sync.go | 1 + 5 files changed, 39 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 49ba3682b..f6cc39f32 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,8 @@ require ( github.com/sirupsen/logrus v1.6.0 github.com/stretchr/testify v1.5.1 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/tools v0.0.0-20200729041821-df70183b1872 // indirect + golang.org/x/tools v0.0.0-20200809012840-6f4f008689da // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/yaml.v2 v2.2.8 k8s.io/api v0.18.6 k8s.io/apiextensions-apiserver v0.18.6 diff --git a/go.sum b/go.sum index 389608b82..651fb34fc 100644 --- a/go.sum +++ b/go.sum @@ -395,12 +395,14 @@ golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200729041821-df70183b1872 h1:/U95VAvB4ZsR91rpZX2MwiKpejhWr+UxJ+N2VlJuESk= -golang.org/x/tools v0.0.0-20200729041821-df70183b1872/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200809012840-6f4f008689da h1:ml5G98G4/tdKT1XNq+ky5iSRdKKux0TANlLAzmXT/hg= +golang.org/x/tools v0.0.0-20200809012840-6f4f008689da/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index a88cde53e..19712cccd 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -124,7 +124,7 @@ func New(cfg Config, kubeClient k8sutil.KubernetesClient, pgSpec acidv1.Postgres return fmt.Sprintf("%s-%s", e.PodName, e.ResourceVersion), nil }) - password_encryption, ok := pgSpec.Spec.PostgresqlParam.Parameters["password_encryption"] + password_encryption, ok := pgSpec.Spec.PostgresqlParam.Parameters["password_encryption"] if !ok { password_encryption = "md5" } diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index 3066b78c6..a9d13c124 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -207,8 +207,6 @@ func (c *Cluster) deleteConnectionPooler() (err error) { serviceName = service.Name } - // set delete propagation policy to foreground, so that all the dependent - // will be deleted. err = c.KubeClient. Services(c.Namespace). Delete(context.TODO(), serviceName, options) @@ -221,6 +219,21 @@ func (c *Cluster) deleteConnectionPooler() (err error) { c.logger.Infof("Connection pooler service %q has been deleted", serviceName) + // Repeat the same for the secret object + secretName := c.credentialSecretName(c.OpConfig.ConnectionPooler.User) + + secret, err := c.KubeClient. + Secrets(c.Namespace). + Get(context.TODO(), secretName, metav1.GetOptions{}) + + if err != nil { + c.logger.Debugf("could not get connection pooler secret %q: %v", secretName, err) + } else { + if err = c.deleteSecret(secret.UID, *secret); err != nil { + return fmt.Errorf("could not delete pooler secret: %v", err) + } + } + c.ConnectionPooler = nil return nil } @@ -730,14 +743,11 @@ func (c *Cluster) deleteSecrets() error { var errors []string errorCount := 0 for uid, secret := range c.Secrets { - c.logger.Debugf("deleting secret %q", util.NameFromMeta(secret.ObjectMeta)) - err := c.KubeClient.Secrets(secret.Namespace).Delete(context.TODO(), secret.Name, c.deleteOptions) + err := c.deleteSecret(uid, *secret) if err != nil { - errors = append(errors, fmt.Sprintf("could not delete secret %q: %v", util.NameFromMeta(secret.ObjectMeta), err)) + errors = append(errors, fmt.Sprintf("%v", err)) errorCount++ } - c.logger.Infof("secret %q has been deleted", util.NameFromMeta(secret.ObjectMeta)) - c.Secrets[uid] = nil } if errorCount > 0 { @@ -747,6 +757,20 @@ func (c *Cluster) deleteSecrets() error { return nil } +func (c *Cluster) deleteSecret(uid types.UID, secret v1.Secret) error { + c.setProcessName("deleting secret") + secretName := util.NameFromMeta(secret.ObjectMeta) + c.logger.Debugf("deleting secret %q", secretName) + err := c.KubeClient.Secrets(secret.Namespace).Delete(context.TODO(), secret.Name, c.deleteOptions) + if err != nil { + return fmt.Errorf("could not delete secret %q: %v", secretName, err) + } + c.logger.Infof("secret %q has been deleted", secretName) + c.Secrets[uid] = nil + + return nil +} + func (c *Cluster) createRoles() (err error) { // TODO: figure out what to do with duplicate names (humans and robots) among pgUsers return c.syncRoles() diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index b03b5d494..056e43043 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -500,6 +500,7 @@ func (c *Cluster) syncSecrets() error { c.logger.Warningf("secret %q does not contain the role %q", secretSpec.Name, secretUsername) continue } + c.Secrets[secret.UID] = secret c.logger.Debugf("secret %q already exists, fetching its password", util.NameFromMeta(secret.ObjectMeta)) if secretUsername == c.systemUsers[constants.SuperuserKeyName].Name { secretUsername = constants.SuperuserKeyName From dfd0dd90ed07365dba7da8f080d266201685fa31 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Tue, 11 Aug 2020 10:42:31 +0200 Subject: [PATCH 16/35] set search_path for default roles (#1065) * set search_path for default roles * deployment back to 1.5.0 Co-authored-by: Felix Kunde --- pkg/cluster/cluster.go | 41 +++++++++++++++++++++++-------------- pkg/util/constants/roles.go | 31 ++++++++++++++-------------- pkg/util/users/users.go | 16 ++++++++++----- 3 files changed, 53 insertions(+), 35 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 19712cccd..51c5d3809 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -959,32 +959,42 @@ func (c *Cluster) initPreparedDatabaseRoles() error { } for preparedDbName, preparedDB := range c.Spec.PreparedDatabases { + // get list of prepared schemas to set in search_path + preparedSchemas := preparedDB.PreparedSchemas + if len(preparedDB.PreparedSchemas) == 0 { + preparedSchemas = map[string]acidv1.PreparedSchema{"data": {DefaultRoles: util.True()}} + } + + var searchPath strings.Builder + searchPath.WriteString(constants.DefaultSearchPath) + for preparedSchemaName := range preparedSchemas { + searchPath.WriteString(", " + preparedSchemaName) + } + // default roles per database - if err := c.initDefaultRoles(defaultRoles, "admin", preparedDbName); err != nil { + if err := c.initDefaultRoles(defaultRoles, "admin", preparedDbName, searchPath.String()); err != nil { return fmt.Errorf("could not initialize default roles for database %s: %v", preparedDbName, err) } if preparedDB.DefaultUsers { - if err := c.initDefaultRoles(defaultUsers, "admin", preparedDbName); err != nil { + if err := c.initDefaultRoles(defaultUsers, "admin", preparedDbName, searchPath.String()); err != nil { return fmt.Errorf("could not initialize default roles for database %s: %v", preparedDbName, err) } } // default roles per database schema - preparedSchemas := preparedDB.PreparedSchemas - if len(preparedDB.PreparedSchemas) == 0 { - preparedSchemas = map[string]acidv1.PreparedSchema{"data": {DefaultRoles: util.True()}} - } for preparedSchemaName, preparedSchema := range preparedSchemas { if preparedSchema.DefaultRoles == nil || *preparedSchema.DefaultRoles { if err := c.initDefaultRoles(defaultRoles, preparedDbName+constants.OwnerRoleNameSuffix, - preparedDbName+"_"+preparedSchemaName); err != nil { + preparedDbName+"_"+preparedSchemaName, + constants.DefaultSearchPath+", "+preparedSchemaName); err != nil { return fmt.Errorf("could not initialize default roles for database schema %s: %v", preparedSchemaName, err) } if preparedSchema.DefaultUsers { if err := c.initDefaultRoles(defaultUsers, preparedDbName+constants.OwnerRoleNameSuffix, - preparedDbName+"_"+preparedSchemaName); err != nil { + preparedDbName+"_"+preparedSchemaName, + constants.DefaultSearchPath+", "+preparedSchemaName); err != nil { return fmt.Errorf("could not initialize default users for database schema %s: %v", preparedSchemaName, err) } } @@ -994,7 +1004,7 @@ func (c *Cluster) initPreparedDatabaseRoles() error { return nil } -func (c *Cluster) initDefaultRoles(defaultRoles map[string]string, admin, prefix string) error { +func (c *Cluster) initDefaultRoles(defaultRoles map[string]string, admin, prefix string, searchPath string) error { for defaultRole, inherits := range defaultRoles { @@ -1018,12 +1028,13 @@ func (c *Cluster) initDefaultRoles(defaultRoles map[string]string, admin, prefix } newRole := spec.PgUser{ - Origin: spec.RoleOriginBootstrap, - Name: roleName, - Password: util.RandomPassword(constants.PasswordLength), - Flags: flags, - MemberOf: memberOf, - AdminRole: adminRole, + Origin: spec.RoleOriginBootstrap, + Name: roleName, + Password: util.RandomPassword(constants.PasswordLength), + Flags: flags, + MemberOf: memberOf, + Parameters: map[string]string{"search_path": searchPath}, + AdminRole: adminRole, } if currentRole, present := c.pgUsers[roleName]; present { c.pgUsers[roleName] = c.resolveNameConflict(¤tRole, &newRole) diff --git a/pkg/util/constants/roles.go b/pkg/util/constants/roles.go index 87c9c51ce..dd906fe80 100644 --- a/pkg/util/constants/roles.go +++ b/pkg/util/constants/roles.go @@ -2,20 +2,21 @@ package constants // Roles specific constants const ( - PasswordLength = 64 - SuperuserKeyName = "superuser" + PasswordLength = 64 + SuperuserKeyName = "superuser" ConnectionPoolerUserKeyName = "pooler" - ReplicationUserKeyName = "replication" - RoleFlagSuperuser = "SUPERUSER" - RoleFlagInherit = "INHERIT" - RoleFlagLogin = "LOGIN" - RoleFlagNoLogin = "NOLOGIN" - RoleFlagCreateRole = "CREATEROLE" - RoleFlagCreateDB = "CREATEDB" - RoleFlagReplication = "REPLICATION" - RoleFlagByPassRLS = "BYPASSRLS" - OwnerRoleNameSuffix = "_owner" - ReaderRoleNameSuffix = "_reader" - WriterRoleNameSuffix = "_writer" - UserRoleNameSuffix = "_user" + ReplicationUserKeyName = "replication" + RoleFlagSuperuser = "SUPERUSER" + RoleFlagInherit = "INHERIT" + RoleFlagLogin = "LOGIN" + RoleFlagNoLogin = "NOLOGIN" + RoleFlagCreateRole = "CREATEROLE" + RoleFlagCreateDB = "CREATEDB" + RoleFlagReplication = "REPLICATION" + RoleFlagByPassRLS = "BYPASSRLS" + OwnerRoleNameSuffix = "_owner" + ReaderRoleNameSuffix = "_reader" + WriterRoleNameSuffix = "_writer" + UserRoleNameSuffix = "_user" + DefaultSearchPath = "\"$user\"" ) diff --git a/pkg/util/users/users.go b/pkg/util/users/users.go index 166e90264..5d97336e6 100644 --- a/pkg/util/users/users.go +++ b/pkg/util/users/users.go @@ -114,14 +114,14 @@ func (strategy DefaultUserSyncStrategy) ExecuteSyncRequests(requests []spec.PgSy return nil } -func (strategy DefaultUserSyncStrategy) alterPgUserSet(user spec.PgUser, db *sql.DB) (err error) { + +func (strategy DefaultUserSyncStrategy) alterPgUserSet(user spec.PgUser, db *sql.DB) error { queries := produceAlterRoleSetStmts(user) query := fmt.Sprintf(doBlockStmt, strings.Join(queries, ";")) - if _, err = db.Exec(query); err != nil { - err = fmt.Errorf("dB error: %v, query: %s", err, query) - return + if _, err := db.Exec(query); err != nil { + return fmt.Errorf("dB error: %v, query: %s", err, query) } - return + return nil } func (strategy DefaultUserSyncStrategy) createPgUser(user spec.PgUser, db *sql.DB) error { @@ -149,6 +149,12 @@ func (strategy DefaultUserSyncStrategy) createPgUser(user spec.PgUser, db *sql.D return fmt.Errorf("dB error: %v, query: %s", err, query) } + if len(user.Parameters) > 0 { + if err := strategy.alterPgUserSet(user, db); err != nil { + return fmt.Errorf("incomplete setup for user %s: %v", user.Name, err) + } + } + return nil } From fc9ee76832612d15a569a3ccaee5f0cd834693c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sonay=20=20=C5=9Eevik?= Date: Tue, 11 Aug 2020 14:14:39 +0100 Subject: [PATCH 17/35] UI Service port forwarding internal port is updated to 80 from 8081. (#1096) Fix #1093 --- docs/quickstart.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/quickstart.md b/docs/quickstart.md index 034d32e39..16b587d84 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -160,7 +160,7 @@ You can now access the web interface by port forwarding the UI pod (mind the label selector) and enter `localhost:8081` in your browser: ```bash -kubectl port-forward svc/postgres-operator-ui 8081:8081 +kubectl port-forward svc/postgres-operator-ui 8081:80 ``` Available option are explained in detail in the [UI docs](operator-ui.md). From 808030ad1774e616405f6b6d1747519f0948c4ff Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 12 Aug 2020 15:37:40 +0200 Subject: [PATCH 18/35] update go modules (#1097) --- go.mod | 6 +++--- go.sum | 17 ++++++----------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index f6cc39f32..5d3a760df 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,14 @@ module github.com/zalando/postgres-operator go 1.14 require ( - github.com/aws/aws-sdk-go v1.32.2 - github.com/lib/pq v1.7.0 + github.com/aws/aws-sdk-go v1.34.1 + github.com/lib/pq v1.8.0 github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d github.com/r3labs/diff v1.1.0 github.com/sirupsen/logrus v1.6.0 github.com/stretchr/testify v1.5.1 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/tools v0.0.0-20200809012840-6f4f008689da // indirect + golang.org/x/tools v0.0.0-20200811032001-fd80f4dbb3ea // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/yaml.v2 v2.2.8 k8s.io/api v0.18.6 diff --git a/go.sum b/go.sum index 651fb34fc..a12e65ce6 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,8 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.32.2 h1:X5/tQ4cuqCCUZgeOh41WFh9Eq5xe32JzWe4PSE2i1ME= -github.com/aws/aws-sdk-go v1.32.2/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.34.1 h1:jM0mJ9JSJyhujwxBNYKrNB8Iwp8N7J2WsQxTR4yPSck= +github.com/aws/aws-sdk-go v1.34.1/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= @@ -198,8 +198,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY= -github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= +github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -308,9 +308,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 h1:/Tl7pH94bvbAAHBdZJT947M/+gp0+CqQXDtMRC0fseo= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -364,10 +362,8 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f h1:25KHgbfyiSm6vwQLbM3zZIe1v9p/3ea4Rz+nnM5K/i4= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hqSUlvZU0rab6x5EXfGU= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -395,8 +391,8 @@ golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200809012840-6f4f008689da h1:ml5G98G4/tdKT1XNq+ky5iSRdKKux0TANlLAzmXT/hg= -golang.org/x/tools v0.0.0-20200809012840-6f4f008689da/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200811032001-fd80f4dbb3ea h1:9ym67RBRK/wN50W0T3g8g1n8viM1D2ofgWufDlMfWe0= +golang.org/x/tools v0.0.0-20200811032001-fd80f4dbb3ea/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -440,7 +436,6 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.18.6 h1:osqrAXbOQjkKIWDTjrqxWQ3w0GkKb1KA1XkUGHHYpeE= k8s.io/api v0.18.6/go.mod h1:eeyxr+cwCjMdLAmr2W3RyDI0VvTawSg/3RFFBEnmZGI= From 0d81f972a17ef8697479e93f4ecc6df052b70729 Mon Sep 17 00:00:00 2001 From: Jan Mussler Date: Wed, 12 Aug 2020 15:45:00 +0200 Subject: [PATCH 19/35] Added build and node directory to gitignore file. (#1102) --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 559c92499..b9a730ad8 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,8 @@ __pycache__/ # Distribution / packaging .Python +ui/app/node_modules +ui/operator_ui/static/build build/ develop-eggs/ dist/ From 3ddc56e5b9236de656cd5793b10029a567ff38c1 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 13 Aug 2020 16:36:22 +0200 Subject: [PATCH 20/35] allow delete only if annotations meet configured criteria (#1069) * define annotations for delete protection * change log level and reduce log lines for e2e tests * reduce wait_for_pod_start even further --- .../crds/operatorconfigurations.yaml | 4 + charts/postgres-operator/values-crd.yaml | 6 ++ charts/postgres-operator/values.yaml | 6 ++ docs/administrator.md | 76 +++++++++++++-- docs/reference/operator_parameters.md | 10 ++ e2e/requirements.txt | 4 +- e2e/tests/test_e2e.py | 96 +++++++++++++++++-- manifests/complete-postgres-manifest.yaml | 4 +- manifests/configmap.yaml | 2 + manifests/operatorconfiguration.crd.yaml | 4 + ...gresql-operator-default-configuration.yaml | 2 + pkg/apis/acid.zalan.do/v1/crds.go | 6 ++ .../v1/operator_configuration_type.go | 2 + pkg/controller/controller.go | 32 +++++++ pkg/controller/operator_config.go | 2 + pkg/controller/postgresql.go | 17 ++++ pkg/controller/postgresql_test.go | 87 +++++++++++++++++ pkg/util/config/config.go | 2 + 18 files changed, 343 insertions(+), 19 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 4dde1fc23..c3966b410 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -117,6 +117,10 @@ spec: type: object additionalProperties: type: string + delete_annotation_date_key: + type: string + delete_annotation_name_key: + type: string downscaler_annotations: type: array items: diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 44a7f315b..9f9100cab 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -67,6 +67,12 @@ configKubernetes: # keya: valuea # keyb: valueb + # key name for annotation that compares manifest value with current date + # delete_annotation_date_key: "delete-date" + + # key name for annotation that compares manifest value with cluster name + # delete_annotation_name_key: "delete-clustername" + # list of annotations propagated from cluster manifest to statefulset and deployment # downscaler_annotations: # - deployment-time diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index b64495bee..af918a67f 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -63,6 +63,12 @@ configKubernetes: # annotations attached to each database pod # custom_pod_annotations: "keya:valuea,keyb:valueb" + # key name for annotation that compares manifest value with current date + # delete_annotation_date_key: "delete-date" + + # key name for annotation that compares manifest value with cluster name + # delete_annotation_name_key: "delete-clustername" + # list of annotations propagated from cluster manifest to statefulset and deployment # downscaler_annotations: "deployment-time,downscaler/*" diff --git a/docs/administrator.md b/docs/administrator.md index b3d4d9efa..1a1b5e8f9 100644 --- a/docs/administrator.md +++ b/docs/administrator.md @@ -44,7 +44,7 @@ Once the validation is enabled it can only be disabled manually by editing or patching the CRD manifest: ```bash -zk8 patch crd postgresqls.acid.zalan.do -p '{"spec":{"validation": null}}' +kubectl patch crd postgresqls.acid.zalan.do -p '{"spec":{"validation": null}}' ``` ## Non-default cluster domain @@ -123,6 +123,68 @@ Every other Postgres cluster which lacks the annotation will be ignored by this operator. Conversely, operators without a defined `CONTROLLER_ID` will ignore clusters with defined ownership of another operator. +## Delete protection via annotations + +To avoid accidental deletes of Postgres clusters the operator can check the +manifest for two existing annotations containing the cluster name and/or the +current date (in YYYY-MM-DD format). The name of the annotation keys can be +defined in the configuration. By default, they are not set which disables the +delete protection. Thus, one could choose to only go with one annotation. + +**postgres-operator ConfigMap** + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-operator +data: + delete_annotation_date_key: "delete-date" + delete_annotation_name_key: "delete-clustername" +``` + +**OperatorConfiguration** + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: OperatorConfiguration +metadata: + name: postgresql-operator-configuration +configuration: + kubernetes: + delete_annotation_date_key: "delete-date" + delete_annotation_name_key: "delete-clustername" +``` + +Now, every cluster manifest must contain the configured annotation keys to +trigger the delete process when running `kubectl delete pg`. Note, that the +`Postgresql` resource would still get deleted as K8s' API server does not +block it. Only the operator logs will tell, that the delete criteria wasn't +met. + +**cluster manifest** + +```yaml +apiVersion: "acid.zalan.do/v1" +kind: postgresql +metadata: + name: demo-cluster + annotations: + delete-date: "2020-08-31" + delete-clustername: "demo-cluster" +spec: + ... +``` + +In case, the resource has been deleted accidentally or the annotations were +simply forgotten, it's safe to recreate the cluster with `kubectl create`. +Existing Postgres cluster are not replaced by the operator. But, as the +original cluster still exists the status will show `CreateFailed` at first. +On the next sync event it should change to `Running`. However, as it is in +fact a new resource for K8s, the UID will differ which can trigger a rolling +update of the pods because the UID is used as part of backup path to S3. + + ## Role-based access control for the operator The manifest [`operator-service-account-rbac.yaml`](../manifests/operator-service-account-rbac.yaml) @@ -586,11 +648,11 @@ The configuration paramaters that we will be using are: * `gcp_credentials` * `wal_gs_bucket` -### Generate a K8 secret resource +### Generate a K8s secret resource -Generate the K8 secret resource that will contain your service account's +Generate the K8s secret resource that will contain your service account's credentials. It's highly recommended to use a service account and limit its -scope to just the WAL-E bucket. +scope to just the WAL-E bucket. ```yaml apiVersion: v1 @@ -613,13 +675,13 @@ the operator's configuration is set up like the following: ... aws_or_gcp: additional_secret_mount: "pgsql-wale-creds" - additional_secret_mount_path: "/var/secrets/google" # or where ever you want to mount the file + additional_secret_mount_path: "/var/secrets/google" # or where ever you want to mount the file # aws_region: eu-central-1 # kube_iam_role: "" # log_s3_bucket: "" # wal_s3_bucket: "" - wal_gs_bucket: "postgres-backups-bucket-28302F2" # name of bucket on where to save the WAL-E logs - gcp_credentials: "/var/secrets/google/key.json" # combination of the mount path & key in the K8 resource. (i.e. key.json) + wal_gs_bucket: "postgres-backups-bucket-28302F2" # name of bucket on where to save the WAL-E logs + gcp_credentials: "/var/secrets/google/key.json" # combination of the mount path & key in the K8s resource. (i.e. key.json) ... ``` diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 20771078f..9fa622de8 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -200,6 +200,16 @@ configuration they are grouped under the `kubernetes` key. of a database created by the operator. If the annotation key is also provided by the database definition, the database definition value is used. +* **delete_annotation_date_key** + key name for annotation that compares manifest value with current date in the + YYYY-MM-DD format. Allowed pattern: `'([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]'`. + The default is empty which also disables this delete protection check. + +* **delete_annotation_name_key** + key name for annotation that compares manifest value with Postgres cluster name. + Allowed pattern: `'([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]'`. The default is + empty which also disables this delete protection check. + * **downscaler_annotations** An array of annotations that should be passed from Postgres CRD on to the statefulset and, if exists, to the connection pooler deployment as well. diff --git a/e2e/requirements.txt b/e2e/requirements.txt index 68a8775ff..4f6f5ac5f 100644 --- a/e2e/requirements.txt +++ b/e2e/requirements.txt @@ -1,3 +1,3 @@ -kubernetes==9.0.0 +kubernetes==11.0.0 timeout_decorator==0.4.1 -pyyaml==5.1 +pyyaml==5.3.1 diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 9f0b946c9..49e7da10d 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -7,6 +7,7 @@ import os import yaml +from datetime import datetime from kubernetes import client, config @@ -614,6 +615,71 @@ def test_infrastructure_roles(self): "Origin": 2, }) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_x_cluster_deletion(self): + ''' + Test deletion with configured protection + ''' + k8s = self.k8s + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' + + # configure delete protection + patch_delete_annotations = { + "data": { + "delete_annotation_date_key": "delete-date", + "delete_annotation_name_key": "delete-clustername" + } + } + k8s.update_config(patch_delete_annotations) + + # this delete attempt should be omitted because of missing annotations + k8s.api.custom_objects_api.delete_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster") + + # check that pods and services are still there + k8s.wait_for_running_pods(cluster_label, 2) + k8s.wait_for_service(cluster_label) + + # recreate Postgres cluster resource + k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml") + + # wait a little before proceeding + time.sleep(10) + + # add annotations to manifest + deleteDate = datetime.today().strftime('%Y-%m-%d') + pg_patch_delete_annotations = { + "metadata": { + "annotations": { + "delete-date": deleteDate, + "delete-clustername": "acid-minimal-cluster", + } + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_delete_annotations) + + # wait a little before proceeding + time.sleep(10) + k8s.wait_for_running_pods(cluster_label, 2) + k8s.wait_for_service(cluster_label) + + # now delete process should be triggered + k8s.api.custom_objects_api.delete_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster") + + # wait until cluster is deleted + time.sleep(120) + + # check if everything has been deleted + self.assertEqual(0, k8s.count_pods_with_label(cluster_label)) + self.assertEqual(0, k8s.count_services_with_label(cluster_label)) + self.assertEqual(0, k8s.count_endpoints_with_label(cluster_label)) + self.assertEqual(0, k8s.count_statefulsets_with_label(cluster_label)) + self.assertEqual(0, k8s.count_deployments_with_label(cluster_label)) + self.assertEqual(0, k8s.count_pdbs_with_label(cluster_label)) + self.assertEqual(0, k8s.count_secrets_with_label(cluster_label)) + def get_failover_targets(self, master_node, replica_nodes): ''' If all pods live on the same node, failover will happen to other worker(s) @@ -700,11 +766,12 @@ def __init__(self): self.apps_v1 = client.AppsV1Api() self.batch_v1_beta1 = client.BatchV1beta1Api() self.custom_objects_api = client.CustomObjectsApi() + self.policy_v1_beta1 = client.PolicyV1beta1Api() class K8s: ''' - Wraps around K8 api client and helper methods. + Wraps around K8s api client and helper methods. ''' RETRY_TIMEOUT_SEC = 10 @@ -755,14 +822,6 @@ def wait_for_pod_start(self, pod_labels, namespace='default'): if pods: pod_phase = pods[0].status.phase - if pods and pod_phase != 'Running': - pod_name = pods[0].metadata.name - response = self.api.core_v1.read_namespaced_pod( - name=pod_name, - namespace=namespace - ) - print("Pod description {}".format(response)) - time.sleep(self.RETRY_TIMEOUT_SEC) def get_service_type(self, svc_labels, namespace='default'): @@ -824,6 +883,25 @@ def get_services(): def count_pods_with_label(self, labels, namespace='default'): return len(self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items) + def count_services_with_label(self, labels, namespace='default'): + return len(self.api.core_v1.list_namespaced_service(namespace, label_selector=labels).items) + + def count_endpoints_with_label(self, labels, namespace='default'): + return len(self.api.core_v1.list_namespaced_endpoints(namespace, label_selector=labels).items) + + def count_secrets_with_label(self, labels, namespace='default'): + return len(self.api.core_v1.list_namespaced_secret(namespace, label_selector=labels).items) + + def count_statefulsets_with_label(self, labels, namespace='default'): + return len(self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=labels).items) + + def count_deployments_with_label(self, labels, namespace='default'): + return len(self.api.apps_v1.list_namespaced_deployment(namespace, label_selector=labels).items) + + def count_pdbs_with_label(self, labels, namespace='default'): + return len(self.api.policy_v1_beta1.list_namespaced_pod_disruption_budget( + namespace, label_selector=labels).items) + def wait_for_pod_failover(self, failover_targets, labels, namespace='default'): pod_phase = 'Failing over' new_pod_node = '' diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index e626d6b26..69f7a2d9f 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -6,6 +6,8 @@ metadata: # environment: demo # annotations: # "acid.zalan.do/controller": "second-operator" +# "delete-date": "2020-08-31" # can only be deleted on that day if "delete-date "key is configured +# "delete-clustername": "acid-test-cluster" # can only be deleted when name matches if "delete-clustername" key is configured spec: dockerImage: registry.opensource.zalan.do/acid/spilo-12:1.6-p3 teamId: "acid" @@ -34,7 +36,7 @@ spec: defaultUsers: false postgresql: version: "12" - parameters: # Expert section + parameters: # Expert section shared_buffers: "32MB" max_connections: "10" log_statement: "all" diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 1210d5015..3f3e331c4 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -29,6 +29,8 @@ data: # default_cpu_request: 100m # default_memory_limit: 500Mi # default_memory_request: 100Mi + # delete_annotation_date_key: delete-date + # delete_annotation_name_key: delete-clustername docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p3 # downscaler_annotations: "deployment-time,downscaler/*" # enable_admin_role_for_users: "true" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 95c4678a8..36db2dda8 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -113,6 +113,10 @@ spec: type: object additionalProperties: type: string + delete_annotation_date_key: + type: string + delete_annotation_name_key: + type: string downscaler_annotations: type: array items: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index c0dce42ee..7a029eccd 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -31,6 +31,8 @@ configuration: # custom_pod_annotations: # keya: valuea # keyb: valueb + # delete_annotation_date_key: delete-date + # delete_annotation_name_key: delete-clustername # downscaler_annotations: # - deployment-time # - downscaler/* diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index b5695bb4e..43c313c16 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -888,6 +888,12 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, }, }, + "delete_annotation_date_key": { + Type: "string", + }, + "delete_annotation_name_key": { + Type: "string", + }, "downscaler_annotations": { Type: "array", Items: &apiextv1beta1.JSONSchemaPropsOrArray{ diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index ea08f2ff3..157596123 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -66,6 +66,8 @@ type KubernetesMetaConfiguration struct { InheritedLabels []string `json:"inherited_labels,omitempty"` DownscalerAnnotations []string `json:"downscaler_annotations,omitempty"` ClusterNameLabel string `json:"cluster_name_label,omitempty"` + DeleteAnnotationDateKey string `json:"delete_annotation_date_key,omitempty"` + DeleteAnnotationNameKey string `json:"delete_annotation_name_key,omitempty"` NodeReadinessLabel map[string]string `json:"node_readiness_label,omitempty"` CustomPodAnnotations map[string]string `json:"custom_pod_annotations,omitempty"` // TODO: use a proper toleration structure? diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 10c817016..aa996288c 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "sync" + "time" "github.com/sirupsen/logrus" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" @@ -454,6 +455,37 @@ func (c *Controller) GetReference(postgresql *acidv1.Postgresql) *v1.ObjectRefer return ref } +func (c *Controller) meetsClusterDeleteAnnotations(postgresql *acidv1.Postgresql) error { + + deleteAnnotationDateKey := c.opConfig.DeleteAnnotationDateKey + currentTime := time.Now() + currentDate := currentTime.Format("2006-01-02") // go's reference date + + if deleteAnnotationDateKey != "" { + if deleteDate, ok := postgresql.Annotations[deleteAnnotationDateKey]; ok { + if deleteDate != currentDate { + return fmt.Errorf("annotation %s not matching the current date: got %s, expected %s", deleteAnnotationDateKey, deleteDate, currentDate) + } + } else { + return fmt.Errorf("annotation %s not set in manifest to allow cluster deletion", deleteAnnotationDateKey) + } + } + + deleteAnnotationNameKey := c.opConfig.DeleteAnnotationNameKey + + if deleteAnnotationNameKey != "" { + if clusterName, ok := postgresql.Annotations[deleteAnnotationNameKey]; ok { + if clusterName != postgresql.Name { + return fmt.Errorf("annotation %s not matching the cluster name: got %s, expected %s", deleteAnnotationNameKey, clusterName, postgresql.Name) + } + } else { + return fmt.Errorf("annotation %s not set in manifest to allow cluster deletion", deleteAnnotationNameKey) + } + } + + return nil +} + // hasOwnership returns true if the controller is the "owner" of the postgresql. // Whether it's owner is determined by the value of 'acid.zalan.do/controller' // annotation. If the value matches the controllerID then it owns it, or if the diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index d115aa118..aad9069b1 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -92,6 +92,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.InheritedLabels = fromCRD.Kubernetes.InheritedLabels result.DownscalerAnnotations = fromCRD.Kubernetes.DownscalerAnnotations result.ClusterNameLabel = util.Coalesce(fromCRD.Kubernetes.ClusterNameLabel, "cluster-name") + result.DeleteAnnotationDateKey = fromCRD.Kubernetes.DeleteAnnotationDateKey + result.DeleteAnnotationNameKey = fromCRD.Kubernetes.DeleteAnnotationNameKey result.NodeReadinessLabel = fromCRD.Kubernetes.NodeReadinessLabel result.PodPriorityClassName = fromCRD.Kubernetes.PodPriorityClassName result.PodManagementPolicy = util.Coalesce(fromCRD.Kubernetes.PodManagementPolicy, "ordered_ready") diff --git a/pkg/controller/postgresql.go b/pkg/controller/postgresql.go index a41eb0335..c7074c7e4 100644 --- a/pkg/controller/postgresql.go +++ b/pkg/controller/postgresql.go @@ -2,6 +2,7 @@ package controller import ( "context" + "encoding/json" "fmt" "reflect" "strings" @@ -420,6 +421,22 @@ func (c *Controller) queueClusterEvent(informerOldSpec, informerNewSpec *acidv1. clusterError = informerNewSpec.Error } + // only allow deletion if delete annotations are set and conditions are met + if eventType == EventDelete { + if err := c.meetsClusterDeleteAnnotations(informerOldSpec); err != nil { + c.logger.WithField("cluster-name", clusterName).Warnf( + "ignoring %q event for cluster %q - manifest does not fulfill delete requirements: %s", eventType, clusterName, err) + c.logger.WithField("cluster-name", clusterName).Warnf( + "please, recreate Postgresql resource %q and set annotations to delete properly", clusterName) + if currentManifest, marshalErr := json.Marshal(informerOldSpec); marshalErr != nil { + c.logger.WithField("cluster-name", clusterName).Warnf("could not marshal current manifest:\n%+v", informerOldSpec) + } else { + c.logger.WithField("cluster-name", clusterName).Warnf("%s\n", string(currentManifest)) + } + return + } + } + if clusterError != "" && eventType != EventDelete { c.logger.WithField("cluster-name", clusterName).Debugf("skipping %q event for the invalid cluster: %s", eventType, clusterError) diff --git a/pkg/controller/postgresql_test.go b/pkg/controller/postgresql_test.go index b36519c5a..71d23a264 100644 --- a/pkg/controller/postgresql_test.go +++ b/pkg/controller/postgresql_test.go @@ -1,8 +1,10 @@ package controller import ( + "fmt" "reflect" "testing" + "time" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/spec" @@ -90,3 +92,88 @@ func TestMergeDeprecatedPostgreSQLSpecParameters(t *testing.T) { } } } + +func TestMeetsClusterDeleteAnnotations(t *testing.T) { + // set delete annotations in configuration + postgresqlTestController.opConfig.DeleteAnnotationDateKey = "delete-date" + postgresqlTestController.opConfig.DeleteAnnotationNameKey = "delete-clustername" + + currentTime := time.Now() + today := currentTime.Format("2006-01-02") // go's reference date + clusterName := "acid-test-cluster" + + tests := []struct { + name string + pg *acidv1.Postgresql + error string + }{ + { + "Postgres cluster with matching delete annotations", + &acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Annotations: map[string]string{ + "delete-date": today, + "delete-clustername": clusterName, + }, + }, + }, + "", + }, + { + "Postgres cluster with violated delete date annotation", + &acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Annotations: map[string]string{ + "delete-date": "2020-02-02", + "delete-clustername": clusterName, + }, + }, + }, + fmt.Sprintf("annotation delete-date not matching the current date: got 2020-02-02, expected %s", today), + }, + { + "Postgres cluster with violated delete cluster name annotation", + &acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Annotations: map[string]string{ + "delete-date": today, + "delete-clustername": "acid-minimal-cluster", + }, + }, + }, + fmt.Sprintf("annotation delete-clustername not matching the cluster name: got acid-minimal-cluster, expected %s", clusterName), + }, + { + "Postgres cluster with missing delete annotations", + &acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Annotations: map[string]string{}, + }, + }, + "annotation delete-date not set in manifest to allow cluster deletion", + }, + { + "Postgres cluster with missing delete cluster name annotation", + &acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Annotations: map[string]string{ + "delete-date": today, + }, + }, + }, + "annotation delete-clustername not set in manifest to allow cluster deletion", + }, + } + for _, tt := range tests { + if err := postgresqlTestController.meetsClusterDeleteAnnotations(tt.pg); err != nil { + if !reflect.DeepEqual(err.Error(), tt.error) { + t.Errorf("Expected error %q, got: %v", tt.error, err) + } + } + } +} diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 4fe66910a..5f7559929 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -36,6 +36,8 @@ type Resources struct { InheritedLabels []string `name:"inherited_labels" default:""` DownscalerAnnotations []string `name:"downscaler_annotations"` ClusterNameLabel string `name:"cluster_name_label" default:"cluster-name"` + DeleteAnnotationDateKey string `name:"delete_annotation_date_key"` + DeleteAnnotationNameKey string `name:"delete_annotation_name_key"` PodRoleLabel string `name:"pod_role_label" default:"spilo-role"` PodToleration map[string]string `name:"toleration" default:""` DefaultCPURequest string `name:"default_cpu_request" default:"100m"` From dab704c566392b4978622ae8ad20f616e8411dc6 Mon Sep 17 00:00:00 2001 From: Peter Halliday Date: Wed, 26 Aug 2020 05:06:25 -0500 Subject: [PATCH 21/35] Add kustomize support to Postgres UI. (#1086) Co-authored-by: Peter Halliday --- ui/manifests/kustomization.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 ui/manifests/kustomization.yaml diff --git a/ui/manifests/kustomization.yaml b/ui/manifests/kustomization.yaml new file mode 100644 index 000000000..5803f854e --- /dev/null +++ b/ui/manifests/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- deployment.yaml +- ingress.yaml +- service.yaml +- ui-service-account-rbac.yaml From 248ce9fc7823d676d602b117ae36da5dd07bb1b6 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 26 Aug 2020 14:00:14 +0200 Subject: [PATCH 22/35] Update to go 1.14.7 (#1122) * update go version, dependencies, and client-go 1.18.8 --- Makefile | 2 +- delivery.yaml | 2 +- go.mod | 12 ++++++------ go.sum | 35 +++++++++++++++++++++++------------ 4 files changed, 31 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 69dd240db..7ecf54f7c 100644 --- a/Makefile +++ b/Makefile @@ -79,7 +79,7 @@ scm-source.json: .git tools: GO111MODULE=on go get -u honnef.co/go/tools/cmd/staticcheck - GO111MODULE=on go get k8s.io/client-go@kubernetes-1.16.3 + GO111MODULE=on go get k8s.io/client-go@kubernetes-1.18.8 GO111MODULE=on go mod tidy fmt: diff --git a/delivery.yaml b/delivery.yaml index d0884f982..07c768424 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -12,7 +12,7 @@ pipeline: - desc: 'Install go' cmd: | cd /tmp - wget -q https://storage.googleapis.com/golang/go1.14.linux-amd64.tar.gz -O go.tar.gz + wget -q https://storage.googleapis.com/golang/go1.14.7.linux-amd64.tar.gz -O go.tar.gz tar -xf go.tar.gz mv go /usr/local ln -s /usr/local/go/bin/go /usr/bin/go diff --git a/go.mod b/go.mod index 5d3a760df..74f8dc5e1 100644 --- a/go.mod +++ b/go.mod @@ -3,19 +3,19 @@ module github.com/zalando/postgres-operator go 1.14 require ( - github.com/aws/aws-sdk-go v1.34.1 + github.com/aws/aws-sdk-go v1.34.10 github.com/lib/pq v1.8.0 github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d github.com/r3labs/diff v1.1.0 github.com/sirupsen/logrus v1.6.0 github.com/stretchr/testify v1.5.1 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/tools v0.0.0-20200811032001-fd80f4dbb3ea // indirect + golang.org/x/tools v0.0.0-20200826040757-bc8aaaa29e06 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/yaml.v2 v2.2.8 - k8s.io/api v0.18.6 - k8s.io/apiextensions-apiserver v0.18.6 - k8s.io/apimachinery v0.18.6 + k8s.io/api v0.18.8 + k8s.io/apiextensions-apiserver v0.18.0 + k8s.io/apimachinery v0.18.8 k8s.io/client-go v0.18.6 - k8s.io/code-generator v0.18.6 + k8s.io/code-generator v0.18.8 ) diff --git a/go.sum b/go.sum index a12e65ce6..a7787e0fd 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,8 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.34.1 h1:jM0mJ9JSJyhujwxBNYKrNB8Iwp8N7J2WsQxTR4yPSck= -github.com/aws/aws-sdk-go v1.34.1/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.34.10 h1:VU78gcf/3wA4HNEDCHidK738l7K0Bals4SJnfnvXOtY= +github.com/aws/aws-sdk-go v1.34.10/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= @@ -64,6 +64,7 @@ github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v0.0.0-20200808040245-162e5629780b/go.mod h1:NAJj0yf/KaRKURN6nyi7A9IZydMivZEm9oQLWNjfKDc= github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -175,6 +176,7 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -391,8 +393,8 @@ golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200811032001-fd80f4dbb3ea h1:9ym67RBRK/wN50W0T3g8g1n8viM1D2ofgWufDlMfWe0= -golang.org/x/tools v0.0.0-20200811032001-fd80f4dbb3ea/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200826040757-bc8aaaa29e06 h1:ChBCbOHeLqK+j+znGPlWCcvx/t2PdxmyPBheVZxXbcc= +golang.org/x/tools v0.0.0-20200826040757-bc8aaaa29e06/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -436,19 +438,27 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.18.6 h1:osqrAXbOQjkKIWDTjrqxWQ3w0GkKb1KA1XkUGHHYpeE= +k8s.io/api v0.18.0/go.mod h1:q2HRQkfDzHMBZL9l/y9rH63PkQl4vae0xRT+8prbrK8= k8s.io/api v0.18.6/go.mod h1:eeyxr+cwCjMdLAmr2W3RyDI0VvTawSg/3RFFBEnmZGI= -k8s.io/apiextensions-apiserver v0.18.6 h1:vDlk7cyFsDyfwn2rNAO2DbmUbvXy5yT5GE3rrqOzaMo= -k8s.io/apiextensions-apiserver v0.18.6/go.mod h1:lv89S7fUysXjLZO7ke783xOwVTm6lKizADfvUM/SS/M= -k8s.io/apimachinery v0.18.6 h1:RtFHnfGNfd1N0LeSrKCUznz5xtUP1elRGvHJbL3Ntag= +k8s.io/api v0.18.8 h1:aIKUzJPb96f3fKec2lxtY7acZC9gQNDLVhfSGpxBAC4= +k8s.io/api v0.18.8/go.mod h1:d/CXqwWv+Z2XEG1LgceeDmHQwpUJhROPx16SlxJgERY= +k8s.io/apiextensions-apiserver v0.18.0 h1:HN4/P8vpGZFvB5SOMuPPH2Wt9Y/ryX+KRvIyAkchu1Q= +k8s.io/apiextensions-apiserver v0.18.0/go.mod h1:18Cwn1Xws4xnWQNC00FLq1E350b9lUF+aOdIWDOZxgo= +k8s.io/apimachinery v0.18.0/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= k8s.io/apimachinery v0.18.6/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= -k8s.io/apiserver v0.18.6/go.mod h1:Zt2XvTHuaZjBz6EFYzpp+X4hTmgWGy8AthNVnTdm3Wg= +k8s.io/apimachinery v0.18.8 h1:jimPrycCqgx2QPearX3to1JePz7wSbVLq+7PdBTTwQ0= +k8s.io/apimachinery v0.18.8/go.mod h1:6sQd+iHEqmOtALqOFjSWp2KZ9F0wlU/nWm0ZgsYWMig= +k8s.io/apiserver v0.18.0/go.mod h1:3S2O6FeBBd6XTo0njUrLxiqk8GNy6wWOftjhJcXYnjw= +k8s.io/client-go v0.18.0 h1:yqKw4cTUQraZK3fcVCMeSa+lqKwcjZ5wtcOIPnxQno4= +k8s.io/client-go v0.18.0/go.mod h1:uQSYDYs4WhVZ9i6AIoEZuwUggLVEF64HOD37boKAtF8= k8s.io/client-go v0.18.6 h1:I+oWqJbibLSGsZj8Xs8F0aWVXJVIoUHWaaJV3kUN/Zw= k8s.io/client-go v0.18.6/go.mod h1:/fwtGLjYMS1MaM5oi+eXhKwG+1UHidUEXRh6cNsdO0Q= -k8s.io/code-generator v0.18.6 h1:QdfvGfs4gUCS1dru+rLbCKIFxYEV0IRfF8MXwY/ozLk= -k8s.io/code-generator v0.18.6/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= -k8s.io/component-base v0.18.6/go.mod h1:knSVsibPR5K6EW2XOjEHik6sdU5nCvKMrzMt2D4In14= +k8s.io/code-generator v0.18.0/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= +k8s.io/code-generator v0.18.8 h1:lgO1P1wjikEtzNvj7ia+x1VC4svJ28a/r0wnOLhhOTU= +k8s.io/code-generator v0.18.8/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= +k8s.io/component-base v0.18.0/go.mod h1:u3BCg0z1uskkzrnAKFzulmYaEpZF7XC9Pf/uFyb1v2c= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200114144118-36b2048a9120 h1:RPscN6KhmG54S33L+lr3GS+oD1jmchIU0ll519K6FA4= k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= @@ -456,6 +466,7 @@ k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUc k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6 h1:Oh3Mzx5pJ+yIumsAD0MOECPVeXsVot0UkiaCGVyfGQY= k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 h1:d4vVOjXm687F1iLSP2q3lyPPuyvTUt3aVoBpi2DqRsU= From 30c86758a3ce3703648ee2529d4f2f48e6037538 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 28 Aug 2020 12:16:37 +0200 Subject: [PATCH 23/35] update kind and use with old storage class (#1121) * update kind and use with old storage class * specify standard storage class in minimal manifest * remove existing local storage class in kind * fix pod distribution test * exclude k8s master from nodes of interest --- e2e/Makefile | 2 +- e2e/run.sh | 9 +++------ e2e/tests/test_e2e.py | 23 ++++++++++++++--------- manifests/e2e-storage-class.yaml | 8 ++++++++ 4 files changed, 26 insertions(+), 16 deletions(-) create mode 100644 manifests/e2e-storage-class.yaml diff --git a/e2e/Makefile b/e2e/Makefile index 70a2ff4e9..16e3f2f99 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -42,7 +42,7 @@ push: docker tools: docker # install pinned version of 'kind' - GO111MODULE=on go get sigs.k8s.io/kind@v0.5.1 + GO111MODULE=on go get sigs.k8s.io/kind@v0.8.1 e2etest: ./run.sh diff --git a/e2e/run.sh b/e2e/run.sh index c7825bfd3..9d7e2eba7 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -35,25 +35,22 @@ function start_kind(){ kind delete cluster --name ${cluster_name} fi + export KUBECONFIG="${kubeconfig_path}" kind create cluster --name ${cluster_name} --config kind-cluster-postgres-operator-e2e-tests.yaml kind load docker-image "${operator_image}" --name ${cluster_name} kind load docker-image "${e2e_test_image}" --name ${cluster_name} - KUBECONFIG="$(kind get kubeconfig-path --name=${cluster_name})" - export KUBECONFIG } function set_kind_api_server_ip(){ # use the actual kubeconfig to connect to the 'kind' API server # but update the IP address of the API server to the one from the Docker 'bridge' network - cp "${KUBECONFIG}" /tmp readonly local kind_api_server_port=6443 # well-known in the 'kind' codebase - readonly local kind_api_server=$(docker inspect --format "{{ .NetworkSettings.IPAddress }}:${kind_api_server_port}" "${cluster_name}"-control-plane) + readonly local kind_api_server=$(docker inspect --format "{{ .NetworkSettings.Networks.kind.IPAddress }}:${kind_api_server_port}" "${cluster_name}"-control-plane) sed -i "s/server.*$/server: https:\/\/$kind_api_server/g" "${kubeconfig_path}" } function run_tests(){ - - docker run --rm --mount type=bind,source="$(readlink -f ${kubeconfig_path})",target=/root/.kube/config -e OPERATOR_IMAGE="${operator_image}" "${e2e_test_image}" + docker run --rm --network kind --mount type=bind,source="$(readlink -f ${kubeconfig_path})",target=/root/.kube/config -e OPERATOR_IMAGE="${operator_image}" "${e2e_test_image}" } function clean_up(){ diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 49e7da10d..182bdba4d 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -38,6 +38,9 @@ def setUpClass(cls): # set a single K8s wrapper for all tests k8s = cls.k8s = K8s() + # remove existing local storage class and create hostpath class + k8s.api.storage_v1_api.delete_storage_class("standard") + # operator deploys pod service account there on start up # needed for test_multi_namespace_support() cls.namespace = "test" @@ -54,7 +57,8 @@ def setUpClass(cls): "configmap.yaml", "postgres-operator.yaml", "infrastructure-roles.yaml", - "infrastructure-roles-new.yaml"]: + "infrastructure-roles-new.yaml", + "e2e-storage-class.yaml"]: result = k8s.create_with_kubectl("manifests/" + filename) print("stdout: {}, stderr: {}".format(result.stdout, result.stderr)) @@ -600,8 +604,8 @@ def test_infrastructure_roles(self): get_config_cmd = "wget --quiet -O - localhost:8080/config" result = k8s.exec_with_kubectl(operator_pod.metadata.name, get_config_cmd) roles_dict = (json.loads(result.stdout) - .get("controller", {}) - .get("InfrastructureRoles")) + .get("controller", {}) + .get("InfrastructureRoles")) self.assertTrue("robot_zmon_acid_monitoring_new" in roles_dict) role = roles_dict["robot_zmon_acid_monitoring_new"] @@ -685,12 +689,13 @@ def get_failover_targets(self, master_node, replica_nodes): If all pods live on the same node, failover will happen to other worker(s) ''' k8s = self.k8s + k8s_master_exclusion = 'kubernetes.io/hostname!=postgres-operator-e2e-tests-control-plane' failover_targets = [x for x in replica_nodes if x != master_node] if len(failover_targets) == 0: - nodes = k8s.api.core_v1.list_node() + nodes = k8s.api.core_v1.list_node(label_selector=k8s_master_exclusion) for n in nodes.items: - if "node-role.kubernetes.io/master" not in n.metadata.labels and n.metadata.name != master_node: + if n.metadata.name != master_node: failover_targets.append(n.metadata.name) return failover_targets @@ -738,8 +743,7 @@ def assert_distributed_pods(self, master_node, replica_nodes, cluster_label): } } k8s.update_config(patch_enable_antiaffinity) - self.assert_failover( - master_node, len(replica_nodes), failover_targets, cluster_label) + self.assert_failover(master_node, len(replica_nodes), failover_targets, cluster_label) # now disable pod anti affintiy again which will cause yet another failover patch_disable_antiaffinity = { @@ -767,6 +771,7 @@ def __init__(self): self.batch_v1_beta1 = client.BatchV1beta1Api() self.custom_objects_api = client.CustomObjectsApi() self.policy_v1_beta1 = client.PolicyV1beta1Api() + self.storage_v1_api = client.StorageV1Api() class K8s: @@ -944,8 +949,8 @@ def create_with_kubectl(self, path): def exec_with_kubectl(self, pod, cmd): return subprocess.run(["./exec.sh", pod, cmd], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) def get_effective_pod_image(self, pod_name, namespace='default'): ''' diff --git a/manifests/e2e-storage-class.yaml b/manifests/e2e-storage-class.yaml new file mode 100644 index 000000000..c8d941341 --- /dev/null +++ b/manifests/e2e-storage-class.yaml @@ -0,0 +1,8 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + namespace: kube-system + name: standard + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: kubernetes.io/host-path From 5e93aabea607b37002856b39d9e1b6368840350a Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 28 Aug 2020 14:57:19 +0200 Subject: [PATCH 24/35] improve e2e test debugging (#1107) * print operator log in most tests when they time out --- e2e/tests/test_e2e.py | 611 +++++++++++++++++++++++------------------- 1 file changed, 337 insertions(+), 274 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 182bdba4d..ce2d392e1 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -163,45 +163,96 @@ def test_enable_load_balancer(self): k8s = self.k8s cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' - # enable load balancer services - pg_patch_enable_lbs = { - "spec": { - "enableMasterLoadBalancer": True, - "enableReplicaLoadBalancer": True + try: + # enable load balancer services + pg_patch_enable_lbs = { + "spec": { + "enableMasterLoadBalancer": True, + "enableReplicaLoadBalancer": True + } } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_lbs) - # wait for service recreation - time.sleep(60) + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_lbs) + # wait for service recreation + time.sleep(60) + + master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') + self.assertEqual(master_svc_type, 'LoadBalancer', + "Expected LoadBalancer service type for master, found {}".format(master_svc_type)) + + repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') + self.assertEqual(repl_svc_type, 'LoadBalancer', + "Expected LoadBalancer service type for replica, found {}".format(repl_svc_type)) + + # disable load balancer services again + pg_patch_disable_lbs = { + "spec": { + "enableMasterLoadBalancer": False, + "enableReplicaLoadBalancer": False + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_lbs) + # wait for service recreation + time.sleep(60) - master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') - self.assertEqual(master_svc_type, 'LoadBalancer', - "Expected LoadBalancer service type for master, found {}".format(master_svc_type)) + master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') + self.assertEqual(master_svc_type, 'ClusterIP', + "Expected ClusterIP service type for master, found {}".format(master_svc_type)) - repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') - self.assertEqual(repl_svc_type, 'LoadBalancer', - "Expected LoadBalancer service type for replica, found {}".format(repl_svc_type)) + repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') + self.assertEqual(repl_svc_type, 'ClusterIP', + "Expected ClusterIP service type for replica, found {}".format(repl_svc_type)) - # disable load balancer services again - pg_patch_disable_lbs = { - "spec": { - "enableMasterLoadBalancer": False, - "enableReplicaLoadBalancer": False - } + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise + + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_infrastructure_roles(self): + ''' + Test using external secrets for infrastructure roles + ''' + k8s = self.k8s + # update infrastructure roles description + secret_name = "postgresql-infrastructure-roles" + roles = "secretname: postgresql-infrastructure-roles-new, userkey: user, rolekey: memberof, passwordkey: password, defaultrolevalue: robot_zmon" + patch_infrastructure_roles = { + "data": { + "infrastructure_roles_secret_name": secret_name, + "infrastructure_roles_secrets": roles, + }, } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_lbs) - # wait for service recreation - time.sleep(60) + k8s.update_config(patch_infrastructure_roles) + + # wait a little before proceeding + time.sleep(30) - master_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=master') - self.assertEqual(master_svc_type, 'ClusterIP', - "Expected ClusterIP service type for master, found {}".format(master_svc_type)) + try: + # check that new roles are represented in the config by requesting the + # operator configuration via API + operator_pod = k8s.get_operator_pod() + get_config_cmd = "wget --quiet -O - localhost:8080/config" + result = k8s.exec_with_kubectl(operator_pod.metadata.name, get_config_cmd) + roles_dict = (json.loads(result.stdout) + .get("controller", {}) + .get("InfrastructureRoles")) + + self.assertTrue("robot_zmon_acid_monitoring_new" in roles_dict) + role = roles_dict["robot_zmon_acid_monitoring_new"] + role.pop("Password", None) + self.assertDictEqual(role, { + "Name": "robot_zmon_acid_monitoring_new", + "Flags": None, + "MemberOf": ["robot_zmon"], + "Parameters": None, + "AdminRole": "", + "Origin": 2, + }) - repl_svc_type = k8s.get_service_type(cluster_label + ',spilo-role=replica') - self.assertEqual(repl_svc_type, 'ClusterIP', - "Expected ClusterIP service type for replica, found {}".format(repl_svc_type)) + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_lazy_spilo_upgrade(self): @@ -230,38 +281,44 @@ def test_lazy_spilo_upgrade(self): pod0 = 'acid-minimal-cluster-0' pod1 = 'acid-minimal-cluster-1' - # restart the pod to get a container with the new image - k8s.api.core_v1.delete_namespaced_pod(pod0, 'default') - time.sleep(60) - - # lazy update works if the restarted pod and older pods run different Spilo versions - new_image = k8s.get_effective_pod_image(pod0) - old_image = k8s.get_effective_pod_image(pod1) - self.assertNotEqual(new_image, old_image, "Lazy updated failed: pods have the same image {}".format(new_image)) - - # sanity check - assert_msg = "Image {} of a new pod differs from {} in operator conf".format(new_image, conf_image) - self.assertEqual(new_image, conf_image, assert_msg) - - # clean up - unpatch_lazy_spilo_upgrade = { - "data": { - "enable_lazy_spilo_upgrade": "false", + try: + # restart the pod to get a container with the new image + k8s.api.core_v1.delete_namespaced_pod(pod0, 'default') + time.sleep(60) + + # lazy update works if the restarted pod and older pods run different Spilo versions + new_image = k8s.get_effective_pod_image(pod0) + old_image = k8s.get_effective_pod_image(pod1) + self.assertNotEqual(new_image, old_image, + "Lazy updated failed: pods have the same image {}".format(new_image)) + + # sanity check + assert_msg = "Image {} of a new pod differs from {} in operator conf".format(new_image, conf_image) + self.assertEqual(new_image, conf_image, assert_msg) + + # clean up + unpatch_lazy_spilo_upgrade = { + "data": { + "enable_lazy_spilo_upgrade": "false", + } } - } - k8s.update_config(unpatch_lazy_spilo_upgrade) + k8s.update_config(unpatch_lazy_spilo_upgrade) - # at this point operator will complete the normal rolling upgrade - # so we additonally test if disabling the lazy upgrade - forcing the normal rolling upgrade - works + # at this point operator will complete the normal rolling upgrade + # so we additonally test if disabling the lazy upgrade - forcing the normal rolling upgrade - works - # XXX there is no easy way to wait until the end of Sync() - time.sleep(60) + # XXX there is no easy way to wait until the end of Sync() + time.sleep(60) + + image0 = k8s.get_effective_pod_image(pod0) + image1 = k8s.get_effective_pod_image(pod1) - image0 = k8s.get_effective_pod_image(pod0) - image1 = k8s.get_effective_pod_image(pod1) + assert_msg = "Disabling lazy upgrade failed: pods still have different images {} and {}".format(image0, image1) + self.assertEqual(image0, image1, assert_msg) - assert_msg = "Disabling lazy upgrade failed: pods still have different images {} and {}".format(image0, image1) - self.assertEqual(image0, image1, assert_msg) + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_logical_backup_cron_job(self): @@ -287,45 +344,51 @@ def test_logical_backup_cron_job(self): } k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_enable_backup) - k8s.wait_for_logical_backup_job_creation() - - jobs = k8s.get_logical_backup_job().items - self.assertEqual(1, len(jobs), "Expected 1 logical backup job, found {}".format(len(jobs))) - - job = jobs[0] - self.assertEqual(job.metadata.name, "logical-backup-acid-minimal-cluster", - "Expected job name {}, found {}" - .format("logical-backup-acid-minimal-cluster", job.metadata.name)) - self.assertEqual(job.spec.schedule, schedule, - "Expected {} schedule, found {}" - .format(schedule, job.spec.schedule)) - - # update the cluster-wide image of the logical backup pod - image = "test-image-name" - patch_logical_backup_image = { - "data": { - "logical_backup_docker_image": image, + + try: + k8s.wait_for_logical_backup_job_creation() + + jobs = k8s.get_logical_backup_job().items + self.assertEqual(1, len(jobs), "Expected 1 logical backup job, found {}".format(len(jobs))) + + job = jobs[0] + self.assertEqual(job.metadata.name, "logical-backup-acid-minimal-cluster", + "Expected job name {}, found {}" + .format("logical-backup-acid-minimal-cluster", job.metadata.name)) + self.assertEqual(job.spec.schedule, schedule, + "Expected {} schedule, found {}" + .format(schedule, job.spec.schedule)) + + # update the cluster-wide image of the logical backup pod + image = "test-image-name" + patch_logical_backup_image = { + "data": { + "logical_backup_docker_image": image, + } } - } - k8s.update_config(patch_logical_backup_image) + k8s.update_config(patch_logical_backup_image) - jobs = k8s.get_logical_backup_job().items - actual_image = jobs[0].spec.job_template.spec.template.spec.containers[0].image - self.assertEqual(actual_image, image, - "Expected job image {}, found {}".format(image, actual_image)) + jobs = k8s.get_logical_backup_job().items + actual_image = jobs[0].spec.job_template.spec.template.spec.containers[0].image + self.assertEqual(actual_image, image, + "Expected job image {}, found {}".format(image, actual_image)) - # delete the logical backup cron job - pg_patch_disable_backup = { - "spec": { - "enableLogicalBackup": False, + # delete the logical backup cron job + pg_patch_disable_backup = { + "spec": { + "enableLogicalBackup": False, + } } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_backup) - k8s.wait_for_logical_backup_job_deletion() - jobs = k8s.get_logical_backup_job().items - self.assertEqual(0, len(jobs), - "Expected 0 logical backup jobs, found {}".format(len(jobs))) + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_disable_backup) + k8s.wait_for_logical_backup_job_deletion() + jobs = k8s.get_logical_backup_job().items + self.assertEqual(0, len(jobs), + "Expected 0 logical backup jobs, found {}".format(len(jobs))) + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_min_resource_limits(self): @@ -365,20 +428,26 @@ def test_min_resource_limits(self): } k8s.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_resources) - k8s.wait_for_pod_failover(failover_targets, labels) - k8s.wait_for_pod_start('spilo-role=replica') - pods = k8s.api.core_v1.list_namespaced_pod( - 'default', label_selector=labels).items - self.assert_master_is_unique() - masterPod = pods[0] + try: + k8s.wait_for_pod_failover(failover_targets, labels) + k8s.wait_for_pod_start('spilo-role=replica') + + pods = k8s.api.core_v1.list_namespaced_pod( + 'default', label_selector=labels).items + self.assert_master_is_unique() + masterPod = pods[0] - self.assertEqual(masterPod.spec.containers[0].resources.limits['cpu'], minCPULimit, - "Expected CPU limit {}, found {}" - .format(minCPULimit, masterPod.spec.containers[0].resources.limits['cpu'])) - self.assertEqual(masterPod.spec.containers[0].resources.limits['memory'], minMemoryLimit, - "Expected memory limit {}, found {}" - .format(minMemoryLimit, masterPod.spec.containers[0].resources.limits['memory'])) + self.assertEqual(masterPod.spec.containers[0].resources.limits['cpu'], minCPULimit, + "Expected CPU limit {}, found {}" + .format(minCPULimit, masterPod.spec.containers[0].resources.limits['cpu'])) + self.assertEqual(masterPod.spec.containers[0].resources.limits['memory'], minMemoryLimit, + "Expected memory limit {}, found {}" + .format(minMemoryLimit, masterPod.spec.containers[0].resources.limits['memory'])) + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_multi_namespace_support(self): @@ -392,9 +461,14 @@ def test_multi_namespace_support(self): pg_manifest["metadata"]["namespace"] = self.namespace yaml.dump(pg_manifest, f, Dumper=yaml.Dumper) - k8s.create_with_kubectl("manifests/complete-postgres-manifest.yaml") - k8s.wait_for_pod_start("spilo-role=master", self.namespace) - self.assert_master_is_unique(self.namespace, "acid-test-cluster") + try: + k8s.create_with_kubectl("manifests/complete-postgres-manifest.yaml") + k8s.wait_for_pod_start("spilo-role=master", self.namespace) + self.assert_master_is_unique(self.namespace, "acid-test-cluster") + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_node_readiness_label(self): @@ -406,40 +480,45 @@ def test_node_readiness_label(self): readiness_label = 'lifecycle-status' readiness_value = 'ready' - # get nodes of master and replica(s) (expected target of new master) - current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) - num_replicas = len(current_replica_nodes) - failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) - - # add node_readiness_label to potential failover nodes - patch_readiness_label = { - "metadata": { - "labels": { - readiness_label: readiness_value + try: + # get nodes of master and replica(s) (expected target of new master) + current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) + num_replicas = len(current_replica_nodes) + failover_targets = self.get_failover_targets(current_master_node, current_replica_nodes) + + # add node_readiness_label to potential failover nodes + patch_readiness_label = { + "metadata": { + "labels": { + readiness_label: readiness_value + } } } - } - for failover_target in failover_targets: - k8s.api.core_v1.patch_node(failover_target, patch_readiness_label) + for failover_target in failover_targets: + k8s.api.core_v1.patch_node(failover_target, patch_readiness_label) - # define node_readiness_label in config map which should trigger a failover of the master - patch_readiness_label_config = { - "data": { - "node_readiness_label": readiness_label + ':' + readiness_value, + # define node_readiness_label in config map which should trigger a failover of the master + patch_readiness_label_config = { + "data": { + "node_readiness_label": readiness_label + ':' + readiness_value, + } } - } - k8s.update_config(patch_readiness_label_config) - new_master_node, new_replica_nodes = self.assert_failover( - current_master_node, num_replicas, failover_targets, cluster_label) + k8s.update_config(patch_readiness_label_config) + new_master_node, new_replica_nodes = self.assert_failover( + current_master_node, num_replicas, failover_targets, cluster_label) - # patch also node where master ran before - k8s.api.core_v1.patch_node(current_master_node, patch_readiness_label) + # patch also node where master ran before + k8s.api.core_v1.patch_node(current_master_node, patch_readiness_label) - # wait a little before proceeding with the pod distribution test - time.sleep(30) + # wait a little before proceeding with the pod distribution test + time.sleep(30) + + # toggle pod anti affinity to move replica away from master node + self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) - # toggle pod anti affinity to move replica away from master node - self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_scaling(self): @@ -449,13 +528,18 @@ def test_scaling(self): k8s = self.k8s labels = "application=spilo,cluster-name=acid-minimal-cluster" - k8s.wait_for_pg_to_scale(3) - self.assertEqual(3, k8s.count_pods_with_label(labels)) - self.assert_master_is_unique() + try: + k8s.wait_for_pg_to_scale(3) + self.assertEqual(3, k8s.count_pods_with_label(labels)) + self.assert_master_is_unique() - k8s.wait_for_pg_to_scale(2) - self.assertEqual(2, k8s.count_pods_with_label(labels)) - self.assert_master_is_unique() + k8s.wait_for_pg_to_scale(2) + self.assertEqual(2, k8s.count_pods_with_label(labels)) + self.assert_master_is_unique() + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_service_annotations(self): @@ -470,27 +554,32 @@ def test_service_annotations(self): } k8s.update_config(patch_custom_service_annotations) - pg_patch_custom_annotations = { - "spec": { - "serviceAnnotations": { - "annotation.key": "value", - "foo": "bar", + try: + pg_patch_custom_annotations = { + "spec": { + "serviceAnnotations": { + "annotation.key": "value", + "foo": "bar", + } } } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_custom_annotations) + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_custom_annotations) - # wait a little before proceeding - time.sleep(30) - annotations = { - "annotation.key": "value", - "foo": "bar", - } - self.assertTrue(k8s.check_service_annotations( - "cluster-name=acid-minimal-cluster,spilo-role=master", annotations)) - self.assertTrue(k8s.check_service_annotations( - "cluster-name=acid-minimal-cluster,spilo-role=replica", annotations)) + # wait a little before proceeding + time.sleep(30) + annotations = { + "annotation.key": "value", + "foo": "bar", + } + self.assertTrue(k8s.check_service_annotations( + "cluster-name=acid-minimal-cluster,spilo-role=master", annotations)) + self.assertTrue(k8s.check_service_annotations( + "cluster-name=acid-minimal-cluster,spilo-role=replica", annotations)) + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise # clean up unpatch_custom_service_annotations = { @@ -515,24 +604,29 @@ def test_statefulset_annotation_propagation(self): } k8s.update_config(patch_sset_propagate_annotations) - pg_crd_annotations = { - "metadata": { - "annotations": { - "deployment-time": "2020-04-30 12:00:00", - "downscaler/downtime_replicas": "0", - }, + try: + pg_crd_annotations = { + "metadata": { + "annotations": { + "deployment-time": "2020-04-30 12:00:00", + "downscaler/downtime_replicas": "0", + }, + } } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_crd_annotations) + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_crd_annotations) - # wait a little before proceeding - time.sleep(60) - annotations = { - "deployment-time": "2020-04-30 12:00:00", - "downscaler/downtime_replicas": "0", - } - self.assertTrue(k8s.check_statefulset_annotations(cluster_label, annotations)) + # wait a little before proceeding + time.sleep(60) + annotations = { + "deployment-time": "2020-04-30 12:00:00", + "downscaler/downtime_replicas": "0", + } + self.assertTrue(k8s.check_statefulset_annotations(cluster_label, annotations)) + + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_taint_based_eviction(self): @@ -559,65 +653,29 @@ def test_taint_based_eviction(self): } } - # patch node and test if master is failing over to one of the expected nodes - k8s.api.core_v1.patch_node(current_master_node, body) - new_master_node, new_replica_nodes = self.assert_failover( - current_master_node, num_replicas, failover_targets, cluster_label) - - # add toleration to pods - patch_toleration_config = { - "data": { - "toleration": "key:postgres,operator:Exists,effect:NoExecute" + try: + # patch node and test if master is failing over to one of the expected nodes + k8s.api.core_v1.patch_node(current_master_node, body) + new_master_node, new_replica_nodes = self.assert_failover( + current_master_node, num_replicas, failover_targets, cluster_label) + + # add toleration to pods + patch_toleration_config = { + "data": { + "toleration": "key:postgres,operator:Exists,effect:NoExecute" + } } - } - k8s.update_config(patch_toleration_config) - - # wait a little before proceeding with the pod distribution test - time.sleep(30) + k8s.update_config(patch_toleration_config) - # toggle pod anti affinity to move replica away from master node - self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) - - @timeout_decorator.timeout(TEST_TIMEOUT_SEC) - def test_infrastructure_roles(self): - ''' - Test using external secrets for infrastructure roles - ''' - k8s = self.k8s - # update infrastructure roles description - secret_name = "postgresql-infrastructure-roles" - roles = "secretname: postgresql-infrastructure-roles-new, userkey: user, rolekey: memberof, passwordkey: password, defaultrolevalue: robot_zmon" - patch_infrastructure_roles = { - "data": { - "infrastructure_roles_secret_name": secret_name, - "infrastructure_roles_secrets": roles, - }, - } - k8s.update_config(patch_infrastructure_roles) + # wait a little before proceeding with the pod distribution test + time.sleep(30) - # wait a little before proceeding - time.sleep(30) + # toggle pod anti affinity to move replica away from master node + self.assert_distributed_pods(new_master_node, new_replica_nodes, cluster_label) - # check that new roles are represented in the config by requesting the - # operator configuration via API - operator_pod = k8s.get_operator_pod() - get_config_cmd = "wget --quiet -O - localhost:8080/config" - result = k8s.exec_with_kubectl(operator_pod.metadata.name, get_config_cmd) - roles_dict = (json.loads(result.stdout) - .get("controller", {}) - .get("InfrastructureRoles")) - - self.assertTrue("robot_zmon_acid_monitoring_new" in roles_dict) - role = roles_dict["robot_zmon_acid_monitoring_new"] - role.pop("Password", None) - self.assertDictEqual(role, { - "Name": "robot_zmon_acid_monitoring_new", - "Flags": None, - "MemberOf": ["robot_zmon"], - "Parameters": None, - "AdminRole": "", - "Origin": 2, - }) + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_x_cluster_deletion(self): @@ -636,53 +694,58 @@ def test_x_cluster_deletion(self): } k8s.update_config(patch_delete_annotations) - # this delete attempt should be omitted because of missing annotations - k8s.api.custom_objects_api.delete_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster") + try: + # this delete attempt should be omitted because of missing annotations + k8s.api.custom_objects_api.delete_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster") + + # check that pods and services are still there + k8s.wait_for_running_pods(cluster_label, 2) + k8s.wait_for_service(cluster_label) + + # recreate Postgres cluster resource + k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml") + + # wait a little before proceeding + time.sleep(10) + + # add annotations to manifest + deleteDate = datetime.today().strftime('%Y-%m-%d') + pg_patch_delete_annotations = { + "metadata": { + "annotations": { + "delete-date": deleteDate, + "delete-clustername": "acid-minimal-cluster", + } + } + } + k8s.api.custom_objects_api.patch_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_delete_annotations) + + # wait a little before proceeding + time.sleep(10) + k8s.wait_for_running_pods(cluster_label, 2) + k8s.wait_for_service(cluster_label) - # check that pods and services are still there - k8s.wait_for_running_pods(cluster_label, 2) - k8s.wait_for_service(cluster_label) + # now delete process should be triggered + k8s.api.custom_objects_api.delete_namespaced_custom_object( + "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster") - # recreate Postgres cluster resource - k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml") + # wait until cluster is deleted + time.sleep(120) - # wait a little before proceeding - time.sleep(10) - - # add annotations to manifest - deleteDate = datetime.today().strftime('%Y-%m-%d') - pg_patch_delete_annotations = { - "metadata": { - "annotations": { - "delete-date": deleteDate, - "delete-clustername": "acid-minimal-cluster", - } - } - } - k8s.api.custom_objects_api.patch_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_delete_annotations) + # check if everything has been deleted + self.assertEqual(0, k8s.count_pods_with_label(cluster_label)) + self.assertEqual(0, k8s.count_services_with_label(cluster_label)) + self.assertEqual(0, k8s.count_endpoints_with_label(cluster_label)) + self.assertEqual(0, k8s.count_statefulsets_with_label(cluster_label)) + self.assertEqual(0, k8s.count_deployments_with_label(cluster_label)) + self.assertEqual(0, k8s.count_pdbs_with_label(cluster_label)) + self.assertEqual(0, k8s.count_secrets_with_label(cluster_label)) - # wait a little before proceeding - time.sleep(10) - k8s.wait_for_running_pods(cluster_label, 2) - k8s.wait_for_service(cluster_label) - - # now delete process should be triggered - k8s.api.custom_objects_api.delete_namespaced_custom_object( - "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster") - - # wait until cluster is deleted - time.sleep(120) - - # check if everything has been deleted - self.assertEqual(0, k8s.count_pods_with_label(cluster_label)) - self.assertEqual(0, k8s.count_services_with_label(cluster_label)) - self.assertEqual(0, k8s.count_endpoints_with_label(cluster_label)) - self.assertEqual(0, k8s.count_statefulsets_with_label(cluster_label)) - self.assertEqual(0, k8s.count_deployments_with_label(cluster_label)) - self.assertEqual(0, k8s.count_pdbs_with_label(cluster_label)) - self.assertEqual(0, k8s.count_secrets_with_label(cluster_label)) + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise def get_failover_targets(self, master_node, replica_nodes): ''' From e03e9f919a0f5b8effc6ab3a6ca91f051566c196 Mon Sep 17 00:00:00 2001 From: hlihhovac Date: Mon, 31 Aug 2020 12:28:52 +0200 Subject: [PATCH 25/35] add missing omitempty directive to the attributes of PostgresSpec (#1128) Co-authored-by: Pavlo Golub --- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 24ef24d63..b9ac6b660 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -53,7 +53,7 @@ type PostgresSpec struct { NumberOfInstances int32 `json:"numberOfInstances"` Users map[string]UserFlags `json:"users"` MaintenanceWindows []MaintenanceWindow `json:"maintenanceWindows,omitempty"` - Clone *CloneDescription `json:"clone"` + Clone *CloneDescription `json:"clone,omitempty"` ClusterName string `json:"-"` Databases map[string]string `json:"databases,omitempty"` PreparedDatabases map[string]PreparedDatabase `json:"preparedDatabases,omitempty"` @@ -64,10 +64,10 @@ type PostgresSpec struct { ShmVolume *bool `json:"enableShmVolume,omitempty"` EnableLogicalBackup bool `json:"enableLogicalBackup,omitempty"` LogicalBackupSchedule string `json:"logicalBackupSchedule,omitempty"` - StandbyCluster *StandbyDescription `json:"standby"` - PodAnnotations map[string]string `json:"podAnnotations"` - ServiceAnnotations map[string]string `json:"serviceAnnotations"` - TLS *TLSDescription `json:"tls"` + StandbyCluster *StandbyDescription `json:"standby,omitempty"` + PodAnnotations map[string]string `json:"podAnnotations,omitempty"` + ServiceAnnotations map[string]string `json:"serviceAnnotations,omitempty"` + TLS *TLSDescription `json:"tls,omitempty"` AdditionalVolumes []AdditionalVolume `json:"additionalVolumes,omitempty"` // deprecated json tags From 03437b63749e9b3bd51cdc9fd6b2f38bd6bfa8d6 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 3 Sep 2020 08:02:46 +0200 Subject: [PATCH 26/35] Update issue templates (#1051) * Update issue templates To help us helping them * update the template * some updates * or not on --- .../postgres-operator-issue-template.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/postgres-operator-issue-template.md diff --git a/.github/ISSUE_TEMPLATE/postgres-operator-issue-template.md b/.github/ISSUE_TEMPLATE/postgres-operator-issue-template.md new file mode 100644 index 000000000..ff7567d2d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/postgres-operator-issue-template.md @@ -0,0 +1,19 @@ +--- +name: Postgres Operator issue template +about: How are you using the operator? +title: '' +labels: '' +assignees: '' + +--- + +Please, answer some short questions which should help us to understand your problem / question better? + +- **Which image of the operator are you using?** e.g. registry.opensource.zalan.do/acid/postgres-operator:v1.5.0 +- **Where do you run it - cloud or metal? Kubernetes or OpenShift?** [AWS K8s | GCP ... | Bare Metal K8s] +- **Are you running Postgres Operator in production?** [yes | no] +- **Type of issue?** [Bug report, question, feature request, etc.] + +Some general remarks when posting a bug report: +- Please, check the operator, pod (Patroni) and postgresql logs first. When copy-pasting many log lines please do it in a separate GitHub gist together with your Postgres CRD and configuration manifest. +- If you feel this issue might be more related to the [Spilo](https://github.com/zalando/spilo/issues) docker image or [Patroni](https://github.com/zalando/patroni/issues), consider opening issues in the respective repos. From d8884a40038eef397aed0e504b5241820ef65c5f Mon Sep 17 00:00:00 2001 From: Igor Yanchenko <1504692+yanchenko-igor@users.noreply.github.com> Date: Tue, 15 Sep 2020 14:19:22 +0300 Subject: [PATCH 27/35] Allow to overwrite default ExternalTrafficPolicy for the service (#1136) * Allow to overwrite default ExternalTrafficPolicy for the service --- docs/reference/operator_parameters.md | 3 + pkg/apis/acid.zalan.do/v1/crds.go | 11 +++ .../v1/operator_configuration_type.go | 1 + pkg/cluster/k8sres.go | 1 + pkg/cluster/k8sres_test.go | 80 +++++++++++++++++++ pkg/controller/operator_config.go | 1 + pkg/util/config/config.go | 2 + 7 files changed, 99 insertions(+) diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 9fa622de8..a1ec0fab3 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -460,6 +460,9 @@ In the CRD-based configuration they are grouped under the `load_balancer` key. replaced with the hosted zone (the value of the `db_hosted_zone` parameter). No other placeholders are allowed. +* **external_traffic_policy** define external traffic policy for the load +balancer, it will default to `Cluster` if undefined. + ## AWS or GCP interaction The options in this group configure operator interactions with non-Kubernetes diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 43c313c16..d49399f6e 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -1129,6 +1129,17 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "replica_dns_name_format": { Type: "string", }, + "external_traffic_policy": { + Type: "string", + Enum: []apiextv1beta1.JSON{ + { + Raw: []byte(`"Cluster"`), + }, + { + Raw: []byte(`"Local"`), + }, + }, + }, }, }, "aws_or_gcp": { diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 157596123..2351b16aa 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -109,6 +109,7 @@ type LoadBalancerConfiguration struct { CustomServiceAnnotations map[string]string `json:"custom_service_annotations,omitempty"` MasterDNSNameFormat config.StringTemplate `json:"master_dns_name_format,omitempty"` ReplicaDNSNameFormat config.StringTemplate `json:"replica_dns_name_format,omitempty"` + ExternalTrafficPolicy string `json:"external_traffic_policy" default:"Cluster"` } // AWSGCPConfiguration defines the configuration for AWS diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index d7878942c..c7824e5ad 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1619,6 +1619,7 @@ func (c *Cluster) generateService(role PostgresRole, spec *acidv1.PostgresSpec) } c.logger.Debugf("final load balancer source ranges as seen in a service spec (not necessarily applied): %q", serviceSpec.LoadBalancerSourceRanges) + serviceSpec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyType(c.OpConfig.ExternalTrafficPolicy) serviceSpec.Type = v1.ServiceTypeLoadBalancer } else if role == Replica { // before PR #258, the replica service was only created if allocated a LB diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 1e474fbf5..5c92a788f 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -1742,3 +1742,83 @@ func TestSidecars(t *testing.T) { }) } + +func TestGenerateService(t *testing.T) { + var spec acidv1.PostgresSpec + var cluster *Cluster + var enableLB bool = true + spec = acidv1.PostgresSpec{ + TeamID: "myapp", NumberOfInstances: 1, + Resources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "1", Memory: "10"}, + }, + Volume: acidv1.Volume{ + Size: "1G", + }, + Sidecars: []acidv1.Sidecar{ + acidv1.Sidecar{ + Name: "cluster-specific-sidecar", + }, + acidv1.Sidecar{ + Name: "cluster-specific-sidecar-with-resources", + Resources: acidv1.Resources{ + ResourceRequests: acidv1.ResourceDescription{CPU: "210m", Memory: "0.8Gi"}, + ResourceLimits: acidv1.ResourceDescription{CPU: "510m", Memory: "1.4Gi"}, + }, + }, + acidv1.Sidecar{ + Name: "replace-sidecar", + DockerImage: "overwrite-image", + }, + }, + EnableMasterLoadBalancer: &enableLB, + } + + cluster = New( + Config{ + OpConfig: config.Config{ + PodManagementPolicy: "ordered_ready", + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + Resources: config.Resources{ + DefaultCPURequest: "200m", + DefaultCPULimit: "500m", + DefaultMemoryRequest: "0.7Gi", + DefaultMemoryLimit: "1.3Gi", + }, + SidecarImages: map[string]string{ + "deprecated-global-sidecar": "image:123", + }, + SidecarContainers: []v1.Container{ + v1.Container{ + Name: "global-sidecar", + }, + // will be replaced by a cluster specific sidecar with the same name + v1.Container{ + Name: "replace-sidecar", + Image: "replaced-image", + }, + }, + Scalyr: config.Scalyr{ + ScalyrAPIKey: "abc", + ScalyrImage: "scalyr-image", + ScalyrCPURequest: "220m", + ScalyrCPULimit: "520m", + ScalyrMemoryRequest: "0.9Gi", + // ise default memory limit + }, + ExternalTrafficPolicy: "Cluster", + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) + + service := cluster.generateService(Master, &spec) + assert.Equal(t, v1.ServiceExternalTrafficPolicyTypeCluster, service.Spec.ExternalTrafficPolicy) + cluster.OpConfig.ExternalTrafficPolicy = "Local" + service = cluster.generateService(Master, &spec) + assert.Equal(t, v1.ServiceExternalTrafficPolicyTypeLocal, service.Spec.ExternalTrafficPolicy) + +} diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index aad9069b1..51cd9737f 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -124,6 +124,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.CustomServiceAnnotations = fromCRD.LoadBalancer.CustomServiceAnnotations result.MasterDNSNameFormat = fromCRD.LoadBalancer.MasterDNSNameFormat result.ReplicaDNSNameFormat = fromCRD.LoadBalancer.ReplicaDNSNameFormat + result.ExternalTrafficPolicy = util.Coalesce(fromCRD.LoadBalancer.ExternalTrafficPolicy, "Cluster") // AWS or GCP config result.WALES3Bucket = fromCRD.AWSGCP.WALES3Bucket diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 5f7559929..3255e61bf 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -175,6 +175,8 @@ type Config struct { EnablePodAntiAffinity bool `name:"enable_pod_antiaffinity" default:"false"` PodAntiAffinityTopologyKey string `name:"pod_antiaffinity_topology_key" default:"kubernetes.io/hostname"` StorageResizeMode string `name:"storage_resize_mode" default:"ebs"` + // ExternalTrafficPolicy for load balancer + ExternalTrafficPolicy string `name:"external_traffic_policy" default:"Cluster"` // deprecated and kept for backward compatibility EnableLoadBalancer *bool `name:"enable_load_balancer"` MasterDNSNameFormat StringTemplate `name:"master_dns_name_format" default:"{cluster}.{team}.{hostedzone}"` From d09e418b56472411ee794726aa1c2dfa1b8f7443 Mon Sep 17 00:00:00 2001 From: Rico Berger Date: Tue, 15 Sep 2020 13:27:59 +0200 Subject: [PATCH 28/35] Set user and group in security context (#1083) * Set user and group in security context --- .../crds/operatorconfigurations.yaml | 4 +++ .../postgres-operator/crds/postgresqls.yaml | 4 +++ charts/postgres-operator/values-crd.yaml | 3 +++ charts/postgres-operator/values.yaml | 3 +++ docs/reference/cluster_manifest.md | 10 +++++++ docs/reference/operator_parameters.md | 10 +++++++ manifests/complete-postgres-manifest.yaml | 2 ++ manifests/configmap.yaml | 2 ++ manifests/operatorconfiguration.crd.yaml | 4 +++ ...gresql-operator-default-configuration.yaml | 2 ++ manifests/postgresql.crd.yaml | 4 +++ pkg/apis/acid.zalan.do/v1/crds.go | 12 +++++++++ .../v1/operator_configuration_type.go | 2 ++ pkg/apis/acid.zalan.do/v1/postgresql_type.go | 4 ++- .../acid.zalan.do/v1/zz_generated.deepcopy.go | 20 ++++++++++++++ pkg/cluster/k8sres.go | 26 ++++++++++++++++++- pkg/cluster/k8sres_test.go | 6 ++++- pkg/controller/operator_config.go | 2 ++ pkg/util/config/config.go | 2 ++ 19 files changed, 119 insertions(+), 3 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index c3966b410..24e476c11 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -200,6 +200,10 @@ spec: type: string secret_name_template: type: string + spilo_runasuser: + type: integer + spilo_runasgroup: + type: integer spilo_fsgroup: type: integer spilo_privileged: diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 6df2de723..0d444e568 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -374,6 +374,10 @@ spec: items: type: object additionalProperties: true + spiloRunAsUser: + type: integer + spiloRunAsGroup: + type: integer spiloFSGroup: type: integer standby: diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 9f9100cab..1aeff87ff 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -127,6 +127,9 @@ configKubernetes: pod_terminate_grace_period: 5m # template for database user secrets generated by the operator secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}" + # set user and group for the spilo container (required to run Spilo as non-root process) + # spilo_runasuser: "101" + # spilo_runasgroup: "103" # group ID with write-access to volumes (required to run Spilo as non-root process) # spilo_fsgroup: 103 diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index af918a67f..f72f375bf 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -118,6 +118,9 @@ configKubernetes: pod_terminate_grace_period: 5m # template for database user secrets generated by the operator secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}" + # set user and group for the spilo container (required to run Spilo as non-root process) + # spilo_runasuser: "101" + # spilo_runasgroup: "103" # group ID with write-access to volumes (required to run Spilo as non-root process) # spilo_fsgroup: "103" diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 576031543..70ab14855 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -65,6 +65,16 @@ These parameters are grouped directly under the `spec` key in the manifest. custom Docker image that overrides the **docker_image** operator parameter. It should be a [Spilo](https://github.com/zalando/spilo) image. Optional. +* **spiloRunAsUser** + sets the user ID which should be used in the container to run the process. + This must be set to run the container without root. By default the container + runs with root. This option only works for Spilo versions >= 1.6-p3. + +* **spiloRunAsGroup** + sets the group ID which should be used in the container to run the process. + This must be set to run the container without root. By default the container + runs with root. This option only works for Spilo versions >= 1.6-p3. + * **spiloFSGroup** the Persistent Volumes for the Spilo pods in the StatefulSet will be owned and writable by the group ID specified. This will override the **spilo_fsgroup** diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index a1ec0fab3..b21f6ac17 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -317,6 +317,16 @@ configuration they are grouped under the `kubernetes` key. that should be assigned to the Postgres pods. The priority class itself must be defined in advance. Default is empty (use the default priority class). +* **spilo_runasuser** + sets the user ID which should be used in the container to run the process. + This must be set to run the container without root. By default the container + runs with root. This option only works for Spilo versions >= 1.6-p3. + +* **spilo_runasgroup** + sets the group ID which should be used in the container to run the process. + This must be set to run the container without root. By default the container + runs with root. This option only works for Spilo versions >= 1.6-p3. + * **spilo_fsgroup** the Persistent Volumes for the Spilo pods in the StatefulSet will be owned and writable by the group ID specified. This is required to run Spilo as a diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 69f7a2d9f..79d1251e6 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -68,6 +68,8 @@ spec: # name: my-config-map enableShmVolume: true +# spiloRunAsUser: 101 +# spiloRunAsGroup: 103 # spiloFSGroup: 103 # podAnnotations: # annotation.key: value diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 3f3e331c4..db39ee33c 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -99,6 +99,8 @@ data: secret_name_template: "{username}.{cluster}.credentials" # sidecar_docker_images: "" # set_memory_request_to_limit: "false" + # spilo_runasuser: 101 + # spilo_runasgroup: 103 # spilo_fsgroup: 103 spilo_privileged: "false" # storage_resize_mode: "off" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 36db2dda8..23ab795ab 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -196,6 +196,10 @@ spec: type: string secret_name_template: type: string + spilo_runasuser: + type: integer + spilo_runasgroup: + type: integer spilo_fsgroup: type: integer spilo_privileged: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 7a029eccd..1fbfff529 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -68,6 +68,8 @@ configuration: # pod_service_account_role_binding_definition: "" pod_terminate_grace_period: 5m secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}" + # spilo_runasuser: 101 + # spilo_runasgroup: 103 # spilo_fsgroup: 103 spilo_privileged: false storage_resize_mode: ebs diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 1d42e7254..97b72a8ca 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -370,6 +370,10 @@ spec: items: type: object additionalProperties: true + spiloRunAsUser: + type: integer + spiloRunAsGroup: + type: integer spiloFSGroup: type: integer standby: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index d49399f6e..b67ee60e2 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -519,6 +519,12 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, }, }, + "spiloRunAsUser": { + Type: "integer", + }, + "spiloRunAsGroup": { + Type: "integer", + }, "spiloFSGroup": { Type: "integer", }, @@ -1018,6 +1024,12 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "secret_name_template": { Type: "string", }, + "spilo_runasuser": { + Type: "integer", + }, + "spilo_runasgroup": { + Type: "integer", + }, "spilo_fsgroup": { Type: "integer", }, diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 2351b16aa..ca3fa46d7 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -49,6 +49,8 @@ type KubernetesMetaConfiguration struct { PodServiceAccountRoleBindingDefinition string `json:"pod_service_account_role_binding_definition,omitempty"` PodTerminateGracePeriod Duration `json:"pod_terminate_grace_period,omitempty"` SpiloPrivileged bool `json:"spilo_privileged,omitempty"` + SpiloRunAsUser *int64 `json:"spilo_runasuser,omitempty"` + SpiloRunAsGroup *int64 `json:"spilo_runasgroup,omitempty"` SpiloFSGroup *int64 `json:"spilo_fsgroup,omitempty"` WatchedNamespace string `json:"watched_namespace,omitempty"` PDBNameFormat config.StringTemplate `json:"pdb_name_format,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index b9ac6b660..499a4cfda 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -35,7 +35,9 @@ type PostgresSpec struct { TeamID string `json:"teamId"` DockerImage string `json:"dockerImage,omitempty"` - SpiloFSGroup *int64 `json:"spiloFSGroup,omitempty"` + SpiloRunAsUser *int64 `json:"spiloRunAsUser,omitempty"` + SpiloRunAsGroup *int64 `json:"spiloRunAsGroup,omitempty"` + SpiloFSGroup *int64 `json:"spiloFSGroup,omitempty"` // vars that enable load balancers are pointers because it is important to know if any of them is omitted from the Postgres manifest // in that case the var evaluates to nil and the value is taken from the operator config diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index efc31d6b6..34e6b46e8 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -147,6 +147,16 @@ func (in *ConnectionPoolerConfiguration) DeepCopy() *ConnectionPoolerConfigurati // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfiguration) { *out = *in + if in.SpiloRunAsUser != nil { + in, out := &in.SpiloRunAsUser, &out.SpiloRunAsUser + *out = new(int64) + **out = **in + } + if in.SpiloRunAsGroup != nil { + in, out := &in.SpiloRunAsGroup, &out.SpiloRunAsGroup + *out = new(int64) + **out = **in + } if in.SpiloFSGroup != nil { in, out := &in.SpiloFSGroup, &out.SpiloFSGroup *out = new(int64) @@ -527,6 +537,16 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { *out = new(ConnectionPooler) (*in).DeepCopyInto(*out) } + if in.SpiloRunAsUser != nil { + in, out := &in.SpiloRunAsUser, &out.SpiloRunAsUser + *out = new(int64) + **out = **in + } + if in.SpiloRunAsGroup != nil { + in, out := &in.SpiloRunAsGroup, &out.SpiloRunAsGroup + *out = new(int64) + **out = **in + } if in.SpiloFSGroup != nil { in, out := &in.SpiloFSGroup, &out.SpiloFSGroup *out = new(int64) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index c7824e5ad..fef202538 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -557,6 +557,8 @@ func (c *Cluster) generatePodTemplate( initContainers []v1.Container, sidecarContainers []v1.Container, tolerationsSpec *[]v1.Toleration, + spiloRunAsUser *int64, + spiloRunAsGroup *int64, spiloFSGroup *int64, nodeAffinity *v1.Affinity, terminateGracePeriod int64, @@ -576,6 +578,14 @@ func (c *Cluster) generatePodTemplate( containers = append(containers, sidecarContainers...) securityContext := v1.PodSecurityContext{} + if spiloRunAsUser != nil { + securityContext.RunAsUser = spiloRunAsUser + } + + if spiloRunAsGroup != nil { + securityContext.RunAsGroup = spiloRunAsGroup + } + if spiloFSGroup != nil { securityContext.FSGroup = spiloFSGroup } @@ -1073,7 +1083,17 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef // pickup the docker image for the spilo container effectiveDockerImage := util.Coalesce(spec.DockerImage, c.OpConfig.DockerImage) - // determine the FSGroup for the spilo pod + // determine the User, Group and FSGroup for the spilo pod + effectiveRunAsUser := c.OpConfig.Resources.SpiloRunAsUser + if spec.SpiloRunAsUser != nil { + effectiveRunAsUser = spec.SpiloRunAsUser + } + + effectiveRunAsGroup := c.OpConfig.Resources.SpiloRunAsGroup + if spec.SpiloRunAsGroup != nil { + effectiveRunAsGroup = spec.SpiloRunAsGroup + } + effectiveFSGroup := c.OpConfig.Resources.SpiloFSGroup if spec.SpiloFSGroup != nil { effectiveFSGroup = spec.SpiloFSGroup @@ -1217,6 +1237,8 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef initContainers, sidecarContainers, &tolerationSpec, + effectiveRunAsUser, + effectiveRunAsGroup, effectiveFSGroup, nodeAffinity(c.OpConfig.NodeReadinessLabel), int64(c.OpConfig.PodTerminateGracePeriod.Seconds()), @@ -1897,6 +1919,8 @@ func (c *Cluster) generateLogicalBackupJob() (*batchv1beta1.CronJob, error) { []v1.Container{}, &[]v1.Toleration{}, nil, + nil, + nil, nodeAffinity(c.OpConfig.NodeReadinessLabel), int64(c.OpConfig.PodTerminateGracePeriod.Seconds()), c.OpConfig.PodServiceAccountName, diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 5c92a788f..f44b071bb 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -1302,6 +1302,8 @@ func TestTLS(t *testing.T) { var err error var spec acidv1.PostgresSpec var cluster *Cluster + var spiloRunAsUser = int64(101) + var spiloRunAsGroup = int64(103) var spiloFSGroup = int64(103) var additionalVolumes = spec.AdditionalVolumes @@ -1329,7 +1331,9 @@ func TestTLS(t *testing.T) { ReplicationUsername: replicationUserName, }, Resources: config.Resources{ - SpiloFSGroup: &spiloFSGroup, + SpiloRunAsUser: &spiloRunAsUser, + SpiloRunAsGroup: &spiloRunAsGroup, + SpiloFSGroup: &spiloFSGroup, }, }, }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger, eventRecorder) diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 51cd9737f..7e4880712 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -61,6 +61,8 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.PodEnvironmentSecret = fromCRD.Kubernetes.PodEnvironmentSecret result.PodTerminateGracePeriod = util.CoalesceDuration(time.Duration(fromCRD.Kubernetes.PodTerminateGracePeriod), "5m") result.SpiloPrivileged = fromCRD.Kubernetes.SpiloPrivileged + result.SpiloRunAsUser = fromCRD.Kubernetes.SpiloRunAsUser + result.SpiloRunAsGroup = fromCRD.Kubernetes.SpiloRunAsGroup result.SpiloFSGroup = fromCRD.Kubernetes.SpiloFSGroup result.ClusterDomain = util.Coalesce(fromCRD.Kubernetes.ClusterDomain, "cluster.local") result.WatchedNamespace = fromCRD.Kubernetes.WatchedNamespace diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 3255e61bf..2a2103f5a 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -28,6 +28,8 @@ type Resources struct { PodLabelWaitTimeout time.Duration `name:"pod_label_wait_timeout" default:"10m"` PodDeletionWaitTimeout time.Duration `name:"pod_deletion_wait_timeout" default:"10m"` PodTerminateGracePeriod time.Duration `name:"pod_terminate_grace_period" default:"5m"` + SpiloRunAsUser *int64 `json:"spilo_runasuser,omitempty"` + SpiloRunAsGroup *int64 `json:"spilo_runasgroup,omitempty"` SpiloFSGroup *int64 `name:"spilo_fsgroup"` PodPriorityClassName string `name:"pod_priority_class_name"` ClusterDomain string `name:"cluster_domain" default:"cluster.local"` From ab95eaa6ef9ea072a386c365e7b685b1de6b89a7 Mon Sep 17 00:00:00 2001 From: neelasha-09 <66790082+neelasha-09@users.noreply.github.com> Date: Tue, 22 Sep 2020 20:46:05 +0530 Subject: [PATCH 29/35] Fixes #1130 (#1139) * Fixes #1130 Co-authored-by: Felix Kunde --- pkg/cluster/sync.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 056e43043..fef5b7b66 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -696,12 +696,8 @@ func (c *Cluster) syncPreparedDatabases() error { if err := c.initDbConnWithName(preparedDbName); err != nil { return fmt.Errorf("could not init connection to database %s: %v", preparedDbName, err) } - defer func() { - if err := c.closeDbConn(); err != nil { - c.logger.Errorf("could not close database connection: %v", err) - } - }() + c.logger.Debugf("syncing prepared database %q", preparedDbName) // now, prepare defined schemas preparedSchemas := preparedDB.PreparedSchemas if len(preparedDB.PreparedSchemas) == 0 { @@ -715,6 +711,10 @@ func (c *Cluster) syncPreparedDatabases() error { if err := c.syncExtensions(preparedDB.Extensions); err != nil { return err } + + if err := c.closeDbConn(); err != nil { + c.logger.Errorf("could not close database connection: %v", err) + } } return nil From 2a21cc4393c8de4c2f1a0c666b27fbb3b5bb73f2 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Wed, 23 Sep 2020 17:26:56 +0200 Subject: [PATCH 30/35] Compare Postgres pod priority on Sync (#1144) * compare Postgres pod priority on Sync Co-authored-by: Sergey Dudoladov --- .travis.yml | 2 +- delivery.yaml | 4 ++++ manifests/configmap.yaml | 1 + manifests/postgres-pod-priority-class.yaml | 11 +++++++++++ pkg/cluster/cluster.go | 9 +++++++++ 5 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 manifests/postgres-pod-priority-class.yaml diff --git a/.travis.yml b/.travis.yml index a52769c91..1239596fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,4 +20,4 @@ script: - hack/verify-codegen.sh - travis_wait 20 go test -race -covermode atomic -coverprofile=profile.cov ./pkg/... -v - goveralls -coverprofile=profile.cov -service=travis-ci -v - - make e2e + - travis_wait 20 make e2e diff --git a/delivery.yaml b/delivery.yaml index 07c768424..d1eec8a2b 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -2,6 +2,10 @@ version: "2017-09-20" pipeline: - id: build-postgres-operator type: script + vm: large + cache: + paths: + - /go/pkg/mod commands: - desc: 'Update' cmd: | diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index db39ee33c..998e0e45f 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -85,6 +85,7 @@ data: pod_service_account_name: "postgres-pod" # pod_service_account_role_binding_definition: "" pod_terminate_grace_period: 5m + # pod_priority_class_name: "postgres-pod-priority" # postgres_superuser_teams: "postgres_superusers" # protected_role_names: "admin" ready_wait_interval: 3s diff --git a/manifests/postgres-pod-priority-class.yaml b/manifests/postgres-pod-priority-class.yaml new file mode 100644 index 000000000..f1b565f21 --- /dev/null +++ b/manifests/postgres-pod-priority-class.yaml @@ -0,0 +1,11 @@ +apiVersion: scheduling.k8s.io/v1 +description: 'This priority class must be used only for databases controlled by the + Postgres operator' +kind: PriorityClass +metadata: + labels: + application: postgres-operator + name: postgres-pod-priority +preemptionPolicy: PreemptLowerPriority +globalDefault: false +value: 1000000 diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 51c5d3809..9b8b51eb0 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -459,6 +459,15 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa } } + // we assume any change in priority happens by rolling out a new priority class + // changing the priority value in an existing class is not supproted + if c.Statefulset.Spec.Template.Spec.PriorityClassName != statefulSet.Spec.Template.Spec.PriorityClassName { + match = false + needsReplace = true + needsRollUpdate = true + reasons = append(reasons, "new statefulset's pod priority class in spec doesn't match the current one") + } + // lazy Spilo update: modify the image in the statefulset itself but let its pods run with the old image // until they are re-created for other reasons, for example node rotation if c.OpConfig.EnableLazySpiloUpgrade && !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.Containers[0].Image, statefulSet.Spec.Template.Spec.Containers[0].Image) { From ffdb47f53a3749050fe36168a70931b187f36ae1 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 25 Sep 2020 09:46:50 +0200 Subject: [PATCH 31/35] remove outdated GSOC info (#1148) Co-authored-by: Sergey Dudoladov --- README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.md b/README.md index f65a97a23..28532e246 100644 --- a/README.md +++ b/README.md @@ -76,12 +76,6 @@ There is a browser-friendly version of this documentation at * [Postgres manifest reference](docs/reference/cluster_manifest.md) * [Command-line options and environment variables](docs/reference/command_line_and_environment.md) -## Google Summer of Code - -The Postgres Operator made it to the [Google Summer of Code 2019](https://summerofcode.withgoogle.com/organizations/5429926902104064/)! -Check [our ideas](docs/gsoc-2019/ideas.md#google-summer-of-code-2019) -and start discussions in [the issue tracker](https://github.com/zalando/postgres-operator/issues). - ## Community There are two places to get in touch with the community: From 3b6dc4f92d6506ca28d1972ee3cc95144877dacb Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 25 Sep 2020 14:14:19 +0200 Subject: [PATCH 32/35] Improve e2e tests (#1111) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * icnrease vm size * cache deps * switch to the absolute cache path as cdp does not support shell expansion * do not pull non-existing image * manually install kind * add alias to kind * use full kind name * one more name change * install kind with other tools * add bind mounts instead of copying files * test fetching the runner image * build image for pierone * bump up the client-go version to match the master * bump up go version * install pinned version of kind before any test run * do not overwrite local ./manifests during test run * update the docs * fix kind name * update go.* files * fix deps * avoid unnecessary image upload * properly install kind * Change network to host to make it reachable within e2e runner. May not be the right solution though. * Small changes. Also use entrypoint vs cmd. * Bumping spilo. Load before test. * undo incorrect merge from the master Co-authored-by: Sergey Dudoladov Co-authored-by: Jan Mußler --- Makefile | 2 +- docs/developer.md | 8 ++-- e2e/Dockerfile | 20 ++++----- e2e/Makefile | 19 +++++--- ...d-cluster-postgres-operator-e2e-tests.yaml | 2 +- e2e/run.sh | 43 ++++++++++++------- e2e/tests/test_e2e.py | 11 +++-- go.mod | 4 +- go.sum | 21 ++------- manifests/configmap.yaml | 4 +- 10 files changed, 71 insertions(+), 63 deletions(-) diff --git a/Makefile b/Makefile index 7ecf54f7c..29bbb47e6 100644 --- a/Makefile +++ b/Makefile @@ -97,4 +97,4 @@ test: GO111MODULE=on go test ./... e2e: docker # build operator image to be tested - cd e2e; make tools e2etest clean + cd e2e; make e2etest diff --git a/docs/developer.md b/docs/developer.md index 6e0fc33c8..59fbe09a2 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -237,9 +237,11 @@ kubectl logs acid-minimal-cluster-0 ## End-to-end tests -The operator provides reference end-to-end tests (e2e) (as Docker image) to -ensure various infrastructure parts work smoothly together. Each e2e execution -tests a Postgres Operator image built from the current git branch. The test +The operator provides reference end-to-end (e2e) tests to +ensure various infrastructure parts work smoothly together. The test code is available at `e2e/tests`. +The special `registry.opensource.zalan.do/acid/postgres-operator-e2e-tests-runner` image is used to run the tests. The container mounts the local `e2e/tests` directory at runtime, so whatever you modify in your local copy of the tests will be executed by a test runner. By maintaining a separate test runner image we avoid the need to re-build the e2e test image on every build. + +Each e2e execution tests a Postgres Operator image built from the current git branch. The test runner creates a new local K8s cluster using [kind](https://kind.sigs.k8s.io/), utilizes provided manifest examples, and runs e2e tests contained in the `tests` folder. The K8s API client in the container connects to the `kind` cluster via diff --git a/e2e/Dockerfile b/e2e/Dockerfile index a250ea9cb..70e6f0a84 100644 --- a/e2e/Dockerfile +++ b/e2e/Dockerfile @@ -1,11 +1,12 @@ -# An image to perform the actual test. Do not forget to copy all necessary test -# files here. -FROM ubuntu:18.04 +# An image to run e2e tests. +# The image does not include the tests; all necessary files are bind-mounted when a container starts. +FROM ubuntu:20.04 LABEL maintainer="Team ACID @ Zalando " -COPY manifests ./manifests -COPY exec.sh ./exec.sh -COPY requirements.txt tests ./ +ENV TERM xterm-256color + +COPY requirements.txt ./ +COPY scm-source.json ./ RUN apt-get update \ && apt-get install --no-install-recommends -y \ @@ -14,13 +15,10 @@ RUN apt-get update \ python3-pip \ curl \ && pip3 install --no-cache-dir -r requirements.txt \ - && curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl \ + && curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl \ && chmod +x ./kubectl \ && mv ./kubectl /usr/local/bin/kubectl \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -ARG VERSION=dev -RUN sed -i "s/__version__ = .*/__version__ = '${VERSION}'/" ./__init__.py - -CMD ["python3", "-m", "unittest", "discover", "--start-directory", ".", "-v"] +ENTRYPOINT ["python3", "-m", "unittest", "discover", "--start-directory", ".", "-v"] diff --git a/e2e/Makefile b/e2e/Makefile index 16e3f2f99..05ea6a3d6 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -1,6 +1,6 @@ .PHONY: clean copy docker push tools test -BINARY ?= postgres-operator-e2e-tests +BINARY ?= postgres-operator-e2e-tests-runner BUILD_FLAGS ?= -v CGO_ENABLED ?= 0 ifeq ($(RACE),1) @@ -34,15 +34,20 @@ copy: clean mkdir manifests cp ../manifests -r . -docker: copy - docker build --build-arg "VERSION=$(VERSION)" -t "$(IMAGE):$(TAG)" . +docker: scm-source.json + docker build -t "$(IMAGE):$(TAG)" . + +scm-source.json: ../.git + echo '{\n "url": "git:$(GITURL)",\n "revision": "$(GITHEAD)",\n "author": "$(USER)",\n "status": "$(GITSTATUS)"\n}' > scm-source.json push: docker docker push "$(IMAGE):$(TAG)" -tools: docker +tools: # install pinned version of 'kind' - GO111MODULE=on go get sigs.k8s.io/kind@v0.8.1 + # go get must run outside of a dir with a (module-based) Go project ! + # otherwise go get updates project's dependencies and/or behaves differently + cd "/tmp" && GO111MODULE=on go get sigs.k8s.io/kind@v0.8.1 -e2etest: - ./run.sh +e2etest: tools copy clean + ./run.sh main diff --git a/e2e/kind-cluster-postgres-operator-e2e-tests.yaml b/e2e/kind-cluster-postgres-operator-e2e-tests.yaml index a59746fd3..752e993cd 100644 --- a/e2e/kind-cluster-postgres-operator-e2e-tests.yaml +++ b/e2e/kind-cluster-postgres-operator-e2e-tests.yaml @@ -1,5 +1,5 @@ kind: Cluster -apiVersion: kind.sigs.k8s.io/v1alpha3 +apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane - role: worker diff --git a/e2e/run.sh b/e2e/run.sh index 9d7e2eba7..74d842879 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -6,29 +6,29 @@ set -o nounset set -o pipefail IFS=$'\n\t' -cd $(dirname "$0"); - readonly cluster_name="postgres-operator-e2e-tests" readonly kubeconfig_path="/tmp/kind-config-${cluster_name}" +readonly spilo_image="registry.opensource.zalan.do/acid/spilo-12:1.6-p5" -function pull_images(){ +echo "Clustername: ${cluster_name}" +echo "Kubeconfig path: ${kubeconfig_path}" +function pull_images(){ operator_tag=$(git describe --tags --always --dirty) if [[ -z $(docker images -q registry.opensource.zalan.do/acid/postgres-operator:${operator_tag}) ]] then docker pull registry.opensource.zalan.do/acid/postgres-operator:latest fi - if [[ -z $(docker images -q registry.opensource.zalan.do/acid/postgres-operator-e2e-tests:${operator_tag}) ]] - then - docker pull registry.opensource.zalan.do/acid/postgres-operator-e2e-tests:latest - fi operator_image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator" --format "{{.Repository}}:{{.Tag}}" | head -1) - e2e_test_image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests" --format "{{.Repository}}:{{.Tag}}" | head -1) + + # this image does not contain the tests; a container mounts them from a local "./tests" dir at start time + e2e_test_runner_image="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests-runner:latest" + docker pull ${e2e_test_runner_image} } function start_kind(){ - + echo "Starting kind for e2e tests" # avoid interference with previous test runs if [[ $(kind get clusters | grep "^${cluster_name}*") != "" ]] then @@ -38,10 +38,12 @@ function start_kind(){ export KUBECONFIG="${kubeconfig_path}" kind create cluster --name ${cluster_name} --config kind-cluster-postgres-operator-e2e-tests.yaml kind load docker-image "${operator_image}" --name ${cluster_name} - kind load docker-image "${e2e_test_image}" --name ${cluster_name} + docker pull "${spilo_image}" + kind load docker-image "${spilo_image}" --name ${cluster_name} } function set_kind_api_server_ip(){ + echo "Setting up kind API server ip" # use the actual kubeconfig to connect to the 'kind' API server # but update the IP address of the API server to the one from the Docker 'bridge' network readonly local kind_api_server_port=6443 # well-known in the 'kind' codebase @@ -50,10 +52,21 @@ function set_kind_api_server_ip(){ } function run_tests(){ - docker run --rm --network kind --mount type=bind,source="$(readlink -f ${kubeconfig_path})",target=/root/.kube/config -e OPERATOR_IMAGE="${operator_image}" "${e2e_test_image}" + echo "Running tests..." + + # tests modify files in ./manifests, so we mount a copy of this directory done by the e2e Makefile + + docker run --rm --network=host -e "TERM=xterm-256color" \ + --mount type=bind,source="$(readlink -f ${kubeconfig_path})",target=/root/.kube/config \ + --mount type=bind,source="$(readlink -f manifests)",target=/manifests \ + --mount type=bind,source="$(readlink -f tests)",target=/tests \ + --mount type=bind,source="$(readlink -f exec.sh)",target=/exec.sh \ + -e OPERATOR_IMAGE="${operator_image}" "${e2e_test_runner_image}" + } function clean_up(){ + echo "Executing cleanup" unset KUBECONFIG kind delete cluster --name ${cluster_name} rm -rf ${kubeconfig_path} @@ -63,11 +76,11 @@ function main(){ trap "clean_up" QUIT TERM EXIT - pull_images - start_kind - set_kind_api_server_ip + time pull_images + time start_kind + time set_kind_api_server_ip run_tests exit 0 } -main "$@" +"$@" diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index ce2d392e1..550d3ced8 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -34,6 +34,7 @@ def setUpClass(cls): In the case of test failure the cluster will stay to enable manual examination; next invocation of "make test" will re-create it. ''' + print("Test Setup being executed") # set a single K8s wrapper for all tests k8s = cls.k8s = K8s() @@ -216,7 +217,8 @@ def test_infrastructure_roles(self): k8s = self.k8s # update infrastructure roles description secret_name = "postgresql-infrastructure-roles" - roles = "secretname: postgresql-infrastructure-roles-new, userkey: user, rolekey: memberof, passwordkey: password, defaultrolevalue: robot_zmon" + roles = "secretname: postgresql-infrastructure-roles-new, \ + userkey: user, rolekey: memberof, passwordkey: password, defaultrolevalue: robot_zmon" patch_infrastructure_roles = { "data": { "infrastructure_roles_secret_name": secret_name, @@ -313,7 +315,8 @@ def test_lazy_spilo_upgrade(self): image0 = k8s.get_effective_pod_image(pod0) image1 = k8s.get_effective_pod_image(pod1) - assert_msg = "Disabling lazy upgrade failed: pods still have different images {} and {}".format(image0, image1) + assert_msg = "Disabling lazy upgrade failed: pods still have different \ + images {} and {}".format(image0, image1) self.assertEqual(image0, image1, assert_msg) except timeout_decorator.TimeoutError: @@ -710,11 +713,11 @@ def test_x_cluster_deletion(self): time.sleep(10) # add annotations to manifest - deleteDate = datetime.today().strftime('%Y-%m-%d') + delete_date = datetime.today().strftime('%Y-%m-%d') pg_patch_delete_annotations = { "metadata": { "annotations": { - "delete-date": deleteDate, + "delete-date": delete_date, "delete-clustername": "acid-minimal-cluster", } } diff --git a/go.mod b/go.mod index 74f8dc5e1..91267bfad 100644 --- a/go.mod +++ b/go.mod @@ -10,12 +10,12 @@ require ( github.com/sirupsen/logrus v1.6.0 github.com/stretchr/testify v1.5.1 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/tools v0.0.0-20200826040757-bc8aaaa29e06 // indirect + golang.org/x/tools v0.0.0-20200828161849-5deb26317202 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/yaml.v2 v2.2.8 k8s.io/api v0.18.8 k8s.io/apiextensions-apiserver v0.18.0 k8s.io/apimachinery v0.18.8 - k8s.io/client-go v0.18.6 + k8s.io/client-go v0.18.8 k8s.io/code-generator v0.18.8 ) diff --git a/go.sum b/go.sum index a7787e0fd..1a59b280a 100644 --- a/go.sum +++ b/go.sum @@ -137,7 +137,6 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -146,7 +145,6 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -156,7 +154,6 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.1.0 h1:rVsPeBmXbYv4If/cumu1AzZPwV58q433hvONV1UEZoI= github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= @@ -181,7 +178,6 @@ github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -272,7 +268,6 @@ github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tL github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -281,7 +276,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= @@ -375,7 +369,6 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -393,11 +386,10 @@ golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200826040757-bc8aaaa29e06 h1:ChBCbOHeLqK+j+znGPlWCcvx/t2PdxmyPBheVZxXbcc= -golang.org/x/tools v0.0.0-20200826040757-bc8aaaa29e06/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200828161849-5deb26317202 h1:DrWbY9UUFi/sl/3HkNVoBjDbGfIPZZfgoGsGxOL1EU8= +golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -431,7 +423,6 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -441,20 +432,17 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.18.0/go.mod h1:q2HRQkfDzHMBZL9l/y9rH63PkQl4vae0xRT+8prbrK8= -k8s.io/api v0.18.6/go.mod h1:eeyxr+cwCjMdLAmr2W3RyDI0VvTawSg/3RFFBEnmZGI= k8s.io/api v0.18.8 h1:aIKUzJPb96f3fKec2lxtY7acZC9gQNDLVhfSGpxBAC4= k8s.io/api v0.18.8/go.mod h1:d/CXqwWv+Z2XEG1LgceeDmHQwpUJhROPx16SlxJgERY= k8s.io/apiextensions-apiserver v0.18.0 h1:HN4/P8vpGZFvB5SOMuPPH2Wt9Y/ryX+KRvIyAkchu1Q= k8s.io/apiextensions-apiserver v0.18.0/go.mod h1:18Cwn1Xws4xnWQNC00FLq1E350b9lUF+aOdIWDOZxgo= k8s.io/apimachinery v0.18.0/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= -k8s.io/apimachinery v0.18.6/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= k8s.io/apimachinery v0.18.8 h1:jimPrycCqgx2QPearX3to1JePz7wSbVLq+7PdBTTwQ0= k8s.io/apimachinery v0.18.8/go.mod h1:6sQd+iHEqmOtALqOFjSWp2KZ9F0wlU/nWm0ZgsYWMig= k8s.io/apiserver v0.18.0/go.mod h1:3S2O6FeBBd6XTo0njUrLxiqk8GNy6wWOftjhJcXYnjw= -k8s.io/client-go v0.18.0 h1:yqKw4cTUQraZK3fcVCMeSa+lqKwcjZ5wtcOIPnxQno4= k8s.io/client-go v0.18.0/go.mod h1:uQSYDYs4WhVZ9i6AIoEZuwUggLVEF64HOD37boKAtF8= -k8s.io/client-go v0.18.6 h1:I+oWqJbibLSGsZj8Xs8F0aWVXJVIoUHWaaJV3kUN/Zw= -k8s.io/client-go v0.18.6/go.mod h1:/fwtGLjYMS1MaM5oi+eXhKwG+1UHidUEXRh6cNsdO0Q= +k8s.io/client-go v0.18.8 h1:SdbLpIxk5j5YbFr1b7fq8S7mDgDjYmUxSbszyoesoDM= +k8s.io/client-go v0.18.8/go.mod h1:HqFqMllQ5NnQJNwjro9k5zMyfhZlOwpuTLVrxjkYSxU= k8s.io/code-generator v0.18.0/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= k8s.io/code-generator v0.18.8 h1:lgO1P1wjikEtzNvj7ia+x1VC4svJ28a/r0wnOLhhOTU= k8s.io/code-generator v0.18.8/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= @@ -475,7 +463,6 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E= sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= -sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 998e0e45f..fc374c754 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -15,7 +15,7 @@ data: # connection_pooler_default_cpu_request: "500m" # connection_pooler_default_memory_limit: 100Mi # connection_pooler_default_memory_request: 100Mi - connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-9" + connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-11" # connection_pooler_max_db_connections: 60 # connection_pooler_mode: "transaction" # connection_pooler_number_of_instances: 2 @@ -31,7 +31,7 @@ data: # default_memory_request: 100Mi # delete_annotation_date_key: delete-date # delete_annotation_name_key: delete-clustername - docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p3 + docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p5 # downscaler_annotations: "deployment-time,downscaler/*" # enable_admin_role_for_users: "true" # enable_crd_validation: "true" From 21475f4547d8ff62eb6cc1e085bc92e1401933a2 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 30 Sep 2020 17:24:14 +0200 Subject: [PATCH 33/35] Cleanup config examples (#1151) * post polishing for latest PRs * update travis and go modules * make deprecation comments in structs less confusing * have separate pod priority class es for operator and database pods --- .travis.yml | 2 +- .../crds/operatorconfigurations.yaml | 5 ++ .../templates/configmap.yaml | 3 + .../templates/operatorconfiguration.yaml | 3 + .../postgres-pod-priority-class.yaml | 15 +++++ charts/postgres-operator/values-crd.yaml | 6 ++ charts/postgres-operator/values.yaml | 6 ++ docs/reference/operator_parameters.md | 16 +++--- go.mod | 3 +- go.sum | 11 ++-- manifests/configmap.yaml | 3 +- manifests/operatorconfiguration.crd.yaml | 5 ++ ...gresql-operator-default-configuration.yaml | 9 +-- pkg/apis/acid.zalan.do/v1/crds.go | 12 ++-- .../v1/operator_configuration_type.go | 27 +++++---- pkg/util/config/config.go | 55 +++++++++---------- 16 files changed, 110 insertions(+), 71 deletions(-) create mode 100644 charts/postgres-operator/templates/postgres-pod-priority-class.yaml diff --git a/.travis.yml b/.travis.yml index 1239596fc..a52769c91 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,4 +20,4 @@ script: - hack/verify-codegen.sh - travis_wait 20 go test -race -covermode atomic -coverprofile=profile.cov ./pkg/... -v - goveralls -coverprofile=profile.cov -service=travis-ci -v - - travis_wait 20 make e2e + - make e2e diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 24e476c11..8b576822c 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -263,6 +263,11 @@ spec: type: boolean enable_replica_load_balancer: type: boolean + external_traffic_policy: + type: string + enum: + - "Cluster" + - "Local" master_dns_name_format: type: string replica_dns_name_format: diff --git a/charts/postgres-operator/templates/configmap.yaml b/charts/postgres-operator/templates/configmap.yaml index 64b55e0df..87fd752b1 100644 --- a/charts/postgres-operator/templates/configmap.yaml +++ b/charts/postgres-operator/templates/configmap.yaml @@ -9,6 +9,9 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} data: + {{- if .Values.podPriorityClassName }} + pod_priority_class_name: {{ .Values.podPriorityClassName }} + {{- end }} pod_service_account_name: {{ include "postgres-pod.serviceAccountName" . }} {{ toYaml .Values.configGeneral | indent 2 }} {{ toYaml .Values.configUsers | indent 2 }} diff --git a/charts/postgres-operator/templates/operatorconfiguration.yaml b/charts/postgres-operator/templates/operatorconfiguration.yaml index d28d68f9c..0625e1327 100644 --- a/charts/postgres-operator/templates/operatorconfiguration.yaml +++ b/charts/postgres-operator/templates/operatorconfiguration.yaml @@ -13,6 +13,9 @@ configuration: users: {{ toYaml .Values.configUsers | indent 4 }} kubernetes: + {{- if .Values.podPriorityClassName }} + pod_priority_class_name: {{ .Values.podPriorityClassName }} + {{- end }} pod_service_account_name: {{ include "postgres-pod.serviceAccountName" . }} oauth_token_secret_name: {{ template "postgres-operator.fullname" . }} {{ toYaml .Values.configKubernetes | indent 4 }} diff --git a/charts/postgres-operator/templates/postgres-pod-priority-class.yaml b/charts/postgres-operator/templates/postgres-pod-priority-class.yaml new file mode 100644 index 000000000..7ee0f2e55 --- /dev/null +++ b/charts/postgres-operator/templates/postgres-pod-priority-class.yaml @@ -0,0 +1,15 @@ +{{- if .Values.podPriorityClassName }} +apiVersion: scheduling.k8s.io/v1 +description: 'Use only for databases controlled by Postgres operator' +kind: PriorityClass +metadata: + labels: + app.kubernetes.io/name: {{ template "postgres-operator.name" . }} + helm.sh/chart: {{ template "postgres-operator.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + name: {{ .Values.podPriorityClassName }} +preemptionPolicy: PreemptLowerPriority +globalDefault: false +value: 1000000 +{{- end }} diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 1aeff87ff..ffa8b7f51 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -183,6 +183,8 @@ configLoadBalancer: enable_master_load_balancer: false # toggles service type load balancer pointing to the replica pod of the cluster enable_replica_load_balancer: false + # define external traffic policy for the load balancer + external_traffic_policy: "Cluster" # defines the DNS name string template for the master load balancer cluster master_dns_name_format: "{cluster}.{team}.{hostedzone}" # defines the DNS name string template for the replica load balancer cluster @@ -318,8 +320,12 @@ podServiceAccount: # If not set a name is generated using the fullname template and "-pod" suffix name: "postgres-pod" +# priority class for operator pod priorityClassName: "" +# priority class for database pods +podPriorityClassName: "" + resources: limits: cpu: 500m diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index f72f375bf..37eac4254 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -172,6 +172,8 @@ configLoadBalancer: enable_master_load_balancer: "false" # toggles service type load balancer pointing to the replica pod of the cluster enable_replica_load_balancer: "false" + # define external traffic policy for the load balancer + external_traffic_policy: "Cluster" # defines the DNS name string template for the master load balancer cluster master_dns_name_format: '{cluster}.{team}.{hostedzone}' # defines the DNS name string template for the replica load balancer cluster @@ -310,8 +312,12 @@ podServiceAccount: # If not set a name is generated using the fullname template and "-pod" suffix name: "postgres-pod" +# priority class for operator pod priorityClassName: "" +# priority class for database pods +podPriorityClassName: "" + resources: limits: cpu: 500m diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index b21f6ac17..465465432 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -434,6 +434,12 @@ CRD-based configuration. Those options affect the behavior of load balancers created by the operator. In the CRD-based configuration they are grouped under the `load_balancer` key. +* **custom_service_annotations** + This key/value map provides a list of annotations that get attached to each + service of a cluster created by the operator. If the annotation key is also + provided by the cluster definition, the manifest value is used. + Optional. + * **db_hosted_zone** DNS zone for the cluster DNS name when the load balancer is configured for the cluster. Only used when combined with @@ -450,11 +456,8 @@ In the CRD-based configuration they are grouped under the `load_balancer` key. cluster. Can be overridden by individual cluster settings. The default is `false`. -* **custom_service_annotations** - This key/value map provides a list of annotations that get attached to each - service of a cluster created by the operator. If the annotation key is also - provided by the cluster definition, the manifest value is used. - Optional. +* **external_traffic_policy** defines external traffic policy for load + balancers. Allowed values are `Cluster` (default) and `Local`. * **master_dns_name_format** defines the DNS name string template for the master load balancer cluster. The default is @@ -470,9 +473,6 @@ In the CRD-based configuration they are grouped under the `load_balancer` key. replaced with the hosted zone (the value of the `db_hosted_zone` parameter). No other placeholders are allowed. -* **external_traffic_policy** define external traffic policy for the load -balancer, it will default to `Cluster` if undefined. - ## AWS or GCP interaction The options in this group configure operator interactions with non-Kubernetes diff --git a/go.mod b/go.mod index 91267bfad..79c3b9be9 100644 --- a/go.mod +++ b/go.mod @@ -10,8 +10,7 @@ require ( github.com/sirupsen/logrus v1.6.0 github.com/stretchr/testify v1.5.1 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/tools v0.0.0-20200828161849-5deb26317202 // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + golang.org/x/tools v0.0.0-20200928201943-a0ef9b62deab // indirect gopkg.in/yaml.v2 v2.2.8 k8s.io/api v0.18.8 k8s.io/apiextensions-apiserver v0.18.0 diff --git a/go.sum b/go.sum index 1a59b280a..2d76a94ee 100644 --- a/go.sum +++ b/go.sum @@ -287,7 +287,7 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -333,8 +333,8 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -386,11 +386,10 @@ golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200828161849-5deb26317202 h1:DrWbY9UUFi/sl/3HkNVoBjDbGfIPZZfgoGsGxOL1EU8= -golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200928201943-a0ef9b62deab h1:CyH2SDm5ATQiX9gtbMYfvNNed97A9v+TJFnUX/fTaJY= +golang.org/x/tools v0.0.0-20200928201943-a0ef9b62deab/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index fc374c754..970f845bf 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -47,6 +47,7 @@ data: # enable_team_superuser: "false" enable_teams_api: "false" # etcd_host: "" + external_traffic_policy: "Cluster" # gcp_credentials: "" # kubernetes_use_configmaps: "false" # infrastructure_roles_secret_name: "postgresql-infrastructure-roles" @@ -80,12 +81,12 @@ data: # pod_environment_secret: "my-custom-secret" pod_label_wait_timeout: 10m pod_management_policy: "ordered_ready" + # pod_priority_class_name: "postgres-pod-priority" pod_role_label: spilo-role # pod_service_account_definition: "" pod_service_account_name: "postgres-pod" # pod_service_account_role_binding_definition: "" pod_terminate_grace_period: 5m - # pod_priority_class_name: "postgres-pod-priority" # postgres_superuser_teams: "postgres_superusers" # protected_role_names: "admin" ready_wait_interval: 3s diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 23ab795ab..515f87438 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -265,6 +265,11 @@ spec: type: boolean enable_replica_load_balancer: type: boolean + external_traffic_policy: + type: string + enum: + - "Cluster" + - "Local" master_dns_name_format: type: string replica_dns_name_format: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 1fbfff529..5fb77bf76 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -61,7 +61,7 @@ configuration: # pod_environment_configmap: "default/my-custom-config" # pod_environment_secret: "my-custom-secret" pod_management_policy: "ordered_ready" - # pod_priority_class_name: "" + # pod_priority_class_name: "postgres-pod-priority" pod_role_label: spilo-role # pod_service_account_definition: "" pod_service_account_name: postgres-pod @@ -90,12 +90,13 @@ configuration: resource_check_interval: 3s resource_check_timeout: 10m load_balancer: - # db_hosted_zone: "" - enable_master_load_balancer: false - enable_replica_load_balancer: false # custom_service_annotations: # keyx: valuex # keyy: valuey + # db_hosted_zone: "" + enable_master_load_balancer: false + enable_replica_load_balancer: false + external_traffic_policy: "Cluster" master_dns_name_format: "{cluster}.{team}.{hostedzone}" replica_dns_name_format: "{cluster}-repl.{team}.{hostedzone}" aws_or_gcp: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index b67ee60e2..2cfc28856 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -1135,12 +1135,6 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "enable_replica_load_balancer": { Type: "boolean", }, - "master_dns_name_format": { - Type: "string", - }, - "replica_dns_name_format": { - Type: "string", - }, "external_traffic_policy": { Type: "string", Enum: []apiextv1beta1.JSON{ @@ -1152,6 +1146,12 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, }, }, + "master_dns_name_format": { + Type: "string", + }, + "replica_dns_name_format": { + Type: "string", + }, }, }, "aws_or_gcp": { diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index ca3fa46d7..179b7e751 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -193,20 +193,19 @@ type OperatorLogicalBackupConfiguration struct { // OperatorConfigurationData defines the operation config type OperatorConfigurationData struct { - EnableCRDValidation *bool `json:"enable_crd_validation,omitempty"` - EnableLazySpiloUpgrade bool `json:"enable_lazy_spilo_upgrade,omitempty"` - EtcdHost string `json:"etcd_host,omitempty"` - KubernetesUseConfigMaps bool `json:"kubernetes_use_configmaps,omitempty"` - DockerImage string `json:"docker_image,omitempty"` - Workers uint32 `json:"workers,omitempty"` - MinInstances int32 `json:"min_instances,omitempty"` - MaxInstances int32 `json:"max_instances,omitempty"` - ResyncPeriod Duration `json:"resync_period,omitempty"` - RepairPeriod Duration `json:"repair_period,omitempty"` - SetMemoryRequestToLimit bool `json:"set_memory_request_to_limit,omitempty"` - ShmVolume *bool `json:"enable_shm_volume,omitempty"` - // deprecated in favour of SidecarContainers - SidecarImages map[string]string `json:"sidecar_docker_images,omitempty"` + EnableCRDValidation *bool `json:"enable_crd_validation,omitempty"` + EnableLazySpiloUpgrade bool `json:"enable_lazy_spilo_upgrade,omitempty"` + EtcdHost string `json:"etcd_host,omitempty"` + KubernetesUseConfigMaps bool `json:"kubernetes_use_configmaps,omitempty"` + DockerImage string `json:"docker_image,omitempty"` + Workers uint32 `json:"workers,omitempty"` + MinInstances int32 `json:"min_instances,omitempty"` + MaxInstances int32 `json:"max_instances,omitempty"` + ResyncPeriod Duration `json:"resync_period,omitempty"` + RepairPeriod Duration `json:"repair_period,omitempty"` + SetMemoryRequestToLimit bool `json:"set_memory_request_to_limit,omitempty"` + ShmVolume *bool `json:"enable_shm_volume,omitempty"` + SidecarImages map[string]string `json:"sidecar_docker_images,omitempty"` // deprecated in favour of SidecarContainers SidecarContainers []v1.Container `json:"sidecars,omitempty"` PostgresUsersConfiguration PostgresUsersConfiguration `json:"users"` Kubernetes KubernetesMetaConfiguration `json:"kubernetes"` diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 2a2103f5a..7a1ae8a41 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -143,14 +143,13 @@ type Config struct { LogicalBackup ConnectionPooler - WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' - KubernetesUseConfigMaps bool `name:"kubernetes_use_configmaps" default:"false"` - EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS - DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-12:1.6-p3"` - // deprecated in favour of SidecarContainers - SidecarImages map[string]string `name:"sidecar_docker_images"` - SidecarContainers []v1.Container `name:"sidecars"` - PodServiceAccountName string `name:"pod_service_account_name" default:"postgres-pod"` + WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' + KubernetesUseConfigMaps bool `name:"kubernetes_use_configmaps" default:"false"` + EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS + DockerImage string `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-12:1.6-p3"` + SidecarImages map[string]string `name:"sidecar_docker_images"` // deprecated in favour of SidecarContainers + SidecarContainers []v1.Container `name:"sidecars"` + PodServiceAccountName string `name:"pod_service_account_name" default:"postgres-pod"` // value of this string must be valid JSON or YAML; see initPodServiceAccount PodServiceAccountDefinition string `name:"pod_service_account_definition" default:""` PodServiceAccountRoleBindingDefinition string `name:"pod_service_account_role_binding_definition" default:""` @@ -177,27 +176,25 @@ type Config struct { EnablePodAntiAffinity bool `name:"enable_pod_antiaffinity" default:"false"` PodAntiAffinityTopologyKey string `name:"pod_antiaffinity_topology_key" default:"kubernetes.io/hostname"` StorageResizeMode string `name:"storage_resize_mode" default:"ebs"` - // ExternalTrafficPolicy for load balancer - ExternalTrafficPolicy string `name:"external_traffic_policy" default:"Cluster"` - // deprecated and kept for backward compatibility - EnableLoadBalancer *bool `name:"enable_load_balancer"` - MasterDNSNameFormat StringTemplate `name:"master_dns_name_format" default:"{cluster}.{team}.{hostedzone}"` - ReplicaDNSNameFormat StringTemplate `name:"replica_dns_name_format" default:"{cluster}-repl.{team}.{hostedzone}"` - PDBNameFormat StringTemplate `name:"pdb_name_format" default:"postgres-{cluster}-pdb"` - EnablePodDisruptionBudget *bool `name:"enable_pod_disruption_budget" default:"true"` - EnableInitContainers *bool `name:"enable_init_containers" default:"true"` - EnableSidecars *bool `name:"enable_sidecars" default:"true"` - Workers uint32 `name:"workers" default:"8"` - APIPort int `name:"api_port" default:"8080"` - RingLogLines int `name:"ring_log_lines" default:"100"` - ClusterHistoryEntries int `name:"cluster_history_entries" default:"1000"` - TeamAPIRoleConfiguration map[string]string `name:"team_api_role_configuration" default:"log_statement:all"` - PodTerminateGracePeriod time.Duration `name:"pod_terminate_grace_period" default:"5m"` - PodManagementPolicy string `name:"pod_management_policy" default:"ordered_ready"` - ProtectedRoles []string `name:"protected_role_names" default:"admin"` - PostgresSuperuserTeams []string `name:"postgres_superuser_teams" default:""` - SetMemoryRequestToLimit bool `name:"set_memory_request_to_limit" default:"false"` - EnableLazySpiloUpgrade bool `name:"enable_lazy_spilo_upgrade" default:"false"` + EnableLoadBalancer *bool `name:"enable_load_balancer"` // deprecated and kept for backward compatibility + ExternalTrafficPolicy string `name:"external_traffic_policy" default:"Cluster"` + MasterDNSNameFormat StringTemplate `name:"master_dns_name_format" default:"{cluster}.{team}.{hostedzone}"` + ReplicaDNSNameFormat StringTemplate `name:"replica_dns_name_format" default:"{cluster}-repl.{team}.{hostedzone}"` + PDBNameFormat StringTemplate `name:"pdb_name_format" default:"postgres-{cluster}-pdb"` + EnablePodDisruptionBudget *bool `name:"enable_pod_disruption_budget" default:"true"` + EnableInitContainers *bool `name:"enable_init_containers" default:"true"` + EnableSidecars *bool `name:"enable_sidecars" default:"true"` + Workers uint32 `name:"workers" default:"8"` + APIPort int `name:"api_port" default:"8080"` + RingLogLines int `name:"ring_log_lines" default:"100"` + ClusterHistoryEntries int `name:"cluster_history_entries" default:"1000"` + TeamAPIRoleConfiguration map[string]string `name:"team_api_role_configuration" default:"log_statement:all"` + PodTerminateGracePeriod time.Duration `name:"pod_terminate_grace_period" default:"5m"` + PodManagementPolicy string `name:"pod_management_policy" default:"ordered_ready"` + ProtectedRoles []string `name:"protected_role_names" default:"admin"` + PostgresSuperuserTeams []string `name:"postgres_superuser_teams" default:""` + SetMemoryRequestToLimit bool `name:"set_memory_request_to_limit" default:"false"` + EnableLazySpiloUpgrade bool `name:"enable_lazy_spilo_upgrade" default:"false"` } // MustMarshal marshals the config or panics From 38e15183a22a8bb6d1e7728ad32bad5001b82c48 Mon Sep 17 00:00:00 2001 From: Sergey Dudoladov Date: Fri, 2 Oct 2020 09:31:55 +0200 Subject: [PATCH 34/35] update kind (#1156) Co-authored-by: Sergey Dudoladov --- e2e/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/Makefile b/e2e/Makefile index 05ea6a3d6..a72c6bef0 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -47,7 +47,7 @@ tools: # install pinned version of 'kind' # go get must run outside of a dir with a (module-based) Go project ! # otherwise go get updates project's dependencies and/or behaves differently - cd "/tmp" && GO111MODULE=on go get sigs.k8s.io/kind@v0.8.1 + cd "/tmp" && GO111MODULE=on go get sigs.k8s.io/kind@v0.9.0 e2etest: tools copy clean ./run.sh main From 692c721854e4667102778e8e69e3dd12e47984b5 Mon Sep 17 00:00:00 2001 From: Alex Stockinger Date: Thu, 8 Oct 2020 15:32:15 +0200 Subject: [PATCH 35/35] Introduce ENABLE_JSON_LOGGING env variable (#1158) --- charts/postgres-operator/templates/deployment.yaml | 4 ++++ charts/postgres-operator/values.yaml | 3 +++ cmd/main.go | 7 ++++++- docs/reference/command_line_and_environment.md | 4 ++++ pkg/controller/controller.go | 3 +++ pkg/spec/types.go | 2 ++ 6 files changed, 22 insertions(+), 1 deletion(-) diff --git a/charts/postgres-operator/templates/deployment.yaml b/charts/postgres-operator/templates/deployment.yaml index 2d8eebcb3..9841bf1bc 100644 --- a/charts/postgres-operator/templates/deployment.yaml +++ b/charts/postgres-operator/templates/deployment.yaml @@ -37,6 +37,10 @@ spec: image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} env: + {{- if .Values.enableJsonLogging }} + - name: ENABLE_JSON_LOGGING + value: "true" + {{- end }} {{- if eq .Values.configTarget "ConfigMap" }} - name: CONFIG_MAP_NAME value: {{ template "postgres-operator.fullname" . }} diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 37eac4254..d4acfe1aa 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -15,6 +15,9 @@ podLabels: {} configTarget: "ConfigMap" +# JSON logging format +enableJsonLogging: false + # general configuration parameters configGeneral: # choose if deployment creates/updates CRDs with OpenAPIV3Validation diff --git a/cmd/main.go b/cmd/main.go index a178c187e..376df0bad 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -2,7 +2,7 @@ package main import ( "flag" - "log" + log "github.com/sirupsen/logrus" "os" "os/signal" "sync" @@ -36,6 +36,8 @@ func init() { flag.BoolVar(&config.NoTeamsAPI, "noteamsapi", false, "Disable all access to the teams API") flag.Parse() + config.EnableJsonLogging = os.Getenv("ENABLE_JSON_LOGGING") == "true" + configMapRawName := os.Getenv("CONFIG_MAP_NAME") if configMapRawName != "" { @@ -63,6 +65,9 @@ func init() { func main() { var err error + if config.EnableJsonLogging { + log.SetFormatter(&log.JSONFormatter{}) + } log.SetOutput(os.Stdout) log.Printf("Spilo operator %s\n", version) diff --git a/docs/reference/command_line_and_environment.md b/docs/reference/command_line_and_environment.md index ece29b094..35f47cabf 100644 --- a/docs/reference/command_line_and_environment.md +++ b/docs/reference/command_line_and_environment.md @@ -56,3 +56,7 @@ The following environment variables are accepted by the operator: * **CRD_READY_WAIT_INTERVAL** defines the interval between consecutive attempts waiting for the `postgresql` CRD to be created. The default is 5s. + +* **ENABLE_JSON_LOGGING** + Set to `true` for JSON formatted logging output. + The default is false. diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index aa996288c..8e9f02029 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -71,6 +71,9 @@ type Controller struct { // NewController creates a new controller func NewController(controllerConfig *spec.ControllerConfig, controllerId string) *Controller { logger := logrus.New() + if controllerConfig.EnableJsonLogging { + logger.SetFormatter(&logrus.JSONFormatter{}) + } var myComponentName = "postgres-operator" if controllerId != "" { diff --git a/pkg/spec/types.go b/pkg/spec/types.go index 7a2c0ddac..78c79e1b3 100644 --- a/pkg/spec/types.go +++ b/pkg/spec/types.go @@ -114,6 +114,8 @@ type ControllerConfig struct { CRDReadyWaitTimeout time.Duration ConfigMapName NamespacedName Namespace string + + EnableJsonLogging bool } // cached value for the GetOperatorNamespace