From 24bb45cf62b9e95d55aa513abf5677c008663c80 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Wed, 5 May 2021 22:24:15 +0200 Subject: [PATCH 1/2] feat: add ignored annotations --- cmd/main.go | 5 +- manifests/operatorconfiguration.crd.yaml | 1068 +++++++++-------- .../v1/operator_configuration_type.go | 1 + pkg/cluster/cluster.go | 11 +- .../comparison_test.go} | 29 +- pkg/cluster/sync.go | 2 +- pkg/cluster/util.go | 64 + pkg/controller/controller.go | 3 + pkg/controller/operator_config.go | 1 + pkg/spec/types.go | 1 + pkg/util/config/config.go | 1 + pkg/util/k8sutil/k8sutil.go | 51 - 12 files changed, 645 insertions(+), 592 deletions(-) rename pkg/{util/k8sutil/k8sutil_test.go => cluster/comparison_test.go} (93%) diff --git a/cmd/main.go b/cmd/main.go index 376df0bad..b943314e2 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -2,13 +2,15 @@ package main import ( "flag" - log "github.com/sirupsen/logrus" "os" "os/signal" + "strings" "sync" "syscall" "time" + log "github.com/sirupsen/logrus" + "github.com/zalando/postgres-operator/pkg/controller" "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util/k8sutil" @@ -36,6 +38,7 @@ func init() { flag.BoolVar(&config.NoTeamsAPI, "noteamsapi", false, "Disable all access to the teams API") flag.Parse() + config.IgnoredAnnotations = strings.Split(os.Getenv("IGNORED_ANNOTATIONS"), ",") config.EnableJsonLogging = os.Getenv("ENABLE_JSON_LOGGING") == "true" configMapRawName := os.Getenv("CONFIG_MAP_NAME") diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 47244e5d7..46593ab57 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -10,549 +10,553 @@ spec: plural: operatorconfigurations singular: operatorconfiguration shortNames: - - opconfig + - opconfig categories: - - all + - all scope: Namespaced versions: - - name: v1 - served: true - storage: true - subresources: - status: {} - additionalPrinterColumns: - - name: Image - type: string - description: Spilo image to be used for Pods - jsonPath: .configuration.docker_image - - name: Cluster-Label - type: string - description: Label for K8s resources created by operator - jsonPath: .configuration.kubernetes.cluster_name_label - - name: Service-Account - type: string - description: Name of service account to be used - jsonPath: .configuration.kubernetes.pod_service_account_name - - name: Min-Instances - type: integer - description: Minimum number of instances per Postgres cluster - jsonPath: .configuration.min_instances - - name: Age - type: date - jsonPath: .metadata.creationTimestamp - schema: - openAPIV3Schema: - type: object - required: - - kind - - apiVersion - - configuration - properties: - kind: - type: string - enum: - - OperatorConfiguration - apiVersion: - type: string - enum: - - acid.zalan.do/v1 - configuration: - type: object - properties: - docker_image: - type: string - default: "registry.opensource.zalan.do/acid/spilo-13:2.0-p6" - enable_crd_validation: - type: boolean - default: true - enable_lazy_spilo_upgrade: - type: boolean - default: false - enable_pgversion_env_var: - type: boolean - default: true - enable_shm_volume: - type: boolean - default: true - enable_spilo_wal_path_compat: - type: boolean - default: false - etcd_host: - type: string - default: "" - kubernetes_use_configmaps: - type: boolean - default: false - max_instances: - type: integer - minimum: -1 # -1 = disabled - default: -1 - min_instances: - type: integer - minimum: -1 # -1 = disabled - default: -1 - resync_period: - type: string - default: "30m" - repair_period: - type: string - default: "5m" - set_memory_request_to_limit: - type: boolean - default: false - sidecar_docker_images: - type: object - additionalProperties: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - name: Image + type: string + description: Spilo image to be used for Pods + jsonPath: .configuration.docker_image + - name: Cluster-Label + type: string + description: Label for K8s resources created by operator + jsonPath: .configuration.kubernetes.cluster_name_label + - name: Service-Account + type: string + description: Name of service account to be used + jsonPath: .configuration.kubernetes.pod_service_account_name + - name: Min-Instances + type: integer + description: Minimum number of instances per Postgres cluster + jsonPath: .configuration.min_instances + - name: Age + type: date + jsonPath: .metadata.creationTimestamp + schema: + openAPIV3Schema: + type: object + required: + - kind + - apiVersion + - configuration + properties: + kind: + type: string + enum: + - OperatorConfiguration + apiVersion: + type: string + enum: + - acid.zalan.do/v1 + configuration: + type: object + properties: + docker_image: type: string - sidecars: - type: array - nullable: true - items: + default: "registry.opensource.zalan.do/acid/spilo-13:2.0-p6" + enable_crd_validation: + type: boolean + default: true + enable_lazy_spilo_upgrade: + type: boolean + default: false + enable_pgversion_env_var: + type: boolean + default: true + enable_shm_volume: + type: boolean + default: true + enable_spilo_wal_path_compat: + type: boolean + default: false + etcd_host: + type: string + default: "" + kubernetes_use_configmaps: + type: boolean + default: false + max_instances: + type: integer + minimum: -1 # -1 = disabled + default: -1 + min_instances: + type: integer + minimum: -1 # -1 = disabled + default: -1 + resync_period: + type: string + default: "30m" + repair_period: + type: string + default: "5m" + set_memory_request_to_limit: + type: boolean + default: false + sidecar_docker_images: type: object - x-kubernetes-preserve-unknown-fields: true - workers: - type: integer - minimum: 1 - default: 8 - users: - type: object - properties: - replication_username: - type: string - default: standby - super_username: - type: string - default: postgres - major_version_upgrade: - type: object - properties: - major_version_upgrade_mode: - type: string - default: "off" - minimal_major_version: - type: string - default: "9.5" - target_major_version: - type: string - default: "13" - kubernetes: - type: object - properties: - additional_pod_capabilities: - type: array - items: - type: string - cluster_domain: + additionalProperties: type: string - default: "cluster.local" - cluster_labels: + sidecars: + type: array + nullable: true + items: type: object - additionalProperties: + x-kubernetes-preserve-unknown-fields: true + workers: + type: integer + minimum: 1 + default: 8 + users: + type: object + properties: + replication_username: type: string - default: - application: spilo - cluster_name_label: - type: string - default: "cluster-name" - custom_pod_annotations: - type: object - additionalProperties: + default: standby + super_username: type: string - delete_annotation_date_key: - type: string - delete_annotation_name_key: - type: string - downscaler_annotations: - type: array - items: - type: string - enable_init_containers: - type: boolean - default: true - enable_pod_antiaffinity: - type: boolean - default: false - enable_pod_disruption_budget: - type: boolean - default: true - enable_sidecars: - type: boolean - default: true - infrastructure_roles_secret_name: - type: string - infrastructure_roles_secrets: - type: array - nullable: true - items: + default: postgres + major_version_upgrade: + type: object + properties: + major_version_upgrade_mode: + type: string + default: "off" + minimal_major_version: + type: string + default: "9.5" + target_major_version: + type: string + default: "13" + kubernetes: + type: object + properties: + additional_pod_capabilities: + type: array + items: + type: string + cluster_domain: + type: string + default: "cluster.local" + cluster_labels: type: object - required: - - secretname - - userkey - - passwordkey - properties: - secretname: - type: string - userkey: - type: string - passwordkey: - type: string - rolekey: - type: string - defaultuservalue: - type: string - defaultrolevalue: - type: string - details: - type: string - template: - type: boolean - inherited_annotations: - type: array - items: - type: string - inherited_labels: - type: array - items: - type: string - master_pod_move_timeout: - type: string - default: "20m" - node_readiness_label: - type: object - additionalProperties: + additionalProperties: + type: string + default: + application: spilo + cluster_name_label: type: string - oauth_token_secret_name: - type: string - default: "postgresql-operator" - pdb_name_format: - type: string - default: "postgres-{cluster}-pdb" - pod_antiaffinity_topology_key: - type: string - default: "kubernetes.io/hostname" - pod_environment_configmap: - type: string - pod_environment_secret: - type: string - pod_management_policy: - type: string - enum: - - "ordered_ready" - - "parallel" - default: "ordered_ready" - pod_priority_class_name: - type: string - pod_role_label: - type: string - default: "spilo-role" - pod_service_account_definition: - type: string - default: "" - pod_service_account_name: - type: string - default: "postgres-pod" - pod_service_account_role_binding_definition: - type: string - default: "" - pod_terminate_grace_period: - type: string - default: "5m" - secret_name_template: - type: string - default: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}" - spilo_allow_privilege_escalation: - type: boolean - default: true - spilo_runasuser: - type: integer - spilo_runasgroup: - type: integer - spilo_fsgroup: - type: integer - spilo_privileged: - type: boolean - default: false - storage_resize_mode: - type: string - enum: - - "ebs" - - "pvc" - - "off" - default: "pvc" - toleration: - type: object - additionalProperties: + default: "cluster-name" + ignored_annotations: + type: array + items: + type: string + custom_pod_annotations: + type: object + additionalProperties: + type: string + delete_annotation_date_key: type: string - watched_namespace: - type: string - postgres_pod_resources: - type: object - properties: - default_cpu_limit: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default: "1" - default_cpu_request: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default: "100m" - default_memory_limit: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - default: "500Mi" - default_memory_request: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - default: "100Mi" - min_cpu_limit: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default: "250m" - min_memory_limit: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - default: "250Mi" - timeouts: - type: object - properties: - pod_label_wait_timeout: - type: string - default: "10m" - pod_deletion_wait_timeout: - type: string - default: "10m" - ready_wait_interval: - type: string - default: "4s" - ready_wait_timeout: - type: string - default: "30s" - resource_check_interval: - type: string - default: "3s" - resource_check_timeout: - type: string - default: "10m" - load_balancer: - type: object - properties: - custom_service_annotations: - type: object - additionalProperties: + delete_annotation_name_key: type: string - db_hosted_zone: - type: string - default: "db.example.com" - enable_master_load_balancer: - type: boolean - default: true - enable_replica_load_balancer: - type: boolean - default: false - external_traffic_policy: - type: string - enum: - - "Cluster" - - "Local" - default: "Cluster" - master_dns_name_format: - type: string - default: "{cluster}.{team}.{hostedzone}" - replica_dns_name_format: - type: string - default: "{cluster}-repl.{team}.{hostedzone}" - aws_or_gcp: - type: object - properties: - additional_secret_mount: - type: string - additional_secret_mount_path: - type: string - default: "/meta/credentials" - aws_region: - type: string - default: "eu-central-1" - enable_ebs_gp3_migration: - type: boolean - default: false - enable_ebs_gp3_migration_max_size: - type: integer - default: 1000 - gcp_credentials: - type: string - kube_iam_role: - type: string - log_s3_bucket: - type: string - wal_gs_bucket: - type: string - wal_s3_bucket: - type: string - logical_backup: - type: object - properties: - logical_backup_docker_image: - type: string - default: "registry.opensource.zalan.do/acid/logical-backup:v1.6.2" - logical_backup_google_application_credentials: - type: string - logical_backup_job_prefix: - type: string - default: "logical-backup-" - logical_backup_provider: - type: string - default: "s3" - logical_backup_s3_access_key_id: - type: string - logical_backup_s3_bucket: - type: string - logical_backup_s3_endpoint: - type: string - logical_backup_s3_region: - type: string - logical_backup_s3_secret_access_key: - type: string - logical_backup_s3_sse: - type: string - logical_backup_schedule: - type: string - pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' - default: "30 00 * * *" - debug: - type: object - properties: - debug_logging: - type: boolean - default: true - enable_database_access: - type: boolean - default: true - teams_api: - type: object - properties: - enable_admin_role_for_users: - type: boolean - default: true - enable_postgres_team_crd: - type: boolean - default: true - enable_postgres_team_crd_superusers: - type: boolean - default: false - enable_team_superuser: - type: boolean - default: false - enable_teams_api: - type: boolean - default: true - pam_configuration: - type: string - default: "https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees" - pam_role_name: - type: string - default: "zalandos" - postgres_superuser_teams: - type: array - items: - type: string - protected_role_names: - type: array - items: - type: string - default: - - admin - team_admin_role: - type: string - default: "admin" - team_api_role_configuration: - type: object - additionalProperties: + downscaler_annotations: + type: array + items: + type: string + enable_init_containers: + type: boolean + default: true + enable_pod_antiaffinity: + type: boolean + default: false + enable_pod_disruption_budget: + type: boolean + default: true + enable_sidecars: + type: boolean + default: true + infrastructure_roles_secret_name: type: string - default: - log_statement: all - teams_api_url: - type: string - default: "https://teams.example.com/api/" - logging_rest_api: - type: object - properties: - api_port: - type: integer - default: 8080 - cluster_history_entries: - type: integer - default: 1000 - ring_log_lines: - type: integer - default: 100 - scalyr: # deprecated - type: object - properties: - scalyr_api_key: - type: string - scalyr_cpu_limit: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default: "1" - scalyr_cpu_request: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default: "100m" - scalyr_image: - type: string - scalyr_memory_limit: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - default: "500Mi" - scalyr_memory_request: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - default: "50Mi" - scalyr_server_url: - type: string - default: "https://upload.eu.scalyr.com" - connection_pooler: - type: object - properties: - connection_pooler_schema: - type: string - default: "pooler" - connection_pooler_user: - type: string - default: "pooler" - connection_pooler_image: - type: string - default: "registry.opensource.zalan.do/acid/pgbouncer:master-16" - connection_pooler_max_db_connections: - type: integer - default: 60 - connection_pooler_mode: - type: string - enum: - - "session" - - "transaction" - default: "transaction" - connection_pooler_number_of_instances: - type: integer - minimum: 1 - default: 2 - connection_pooler_default_cpu_limit: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default: "1" - connection_pooler_default_cpu_request: - type: string - pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - default: "500m" - connection_pooler_default_memory_limit: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - default: "100Mi" - connection_pooler_default_memory_request: - type: string - pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - default: "100Mi" - status: - type: object - additionalProperties: - type: string + infrastructure_roles_secrets: + type: array + nullable: true + items: + type: object + required: + - secretname + - userkey + - passwordkey + properties: + secretname: + type: string + userkey: + type: string + passwordkey: + type: string + rolekey: + type: string + defaultuservalue: + type: string + defaultrolevalue: + type: string + details: + type: string + template: + type: boolean + inherited_annotations: + type: array + items: + type: string + inherited_labels: + type: array + items: + type: string + master_pod_move_timeout: + type: string + default: "20m" + node_readiness_label: + type: object + additionalProperties: + type: string + oauth_token_secret_name: + type: string + default: "postgresql-operator" + pdb_name_format: + type: string + default: "postgres-{cluster}-pdb" + pod_antiaffinity_topology_key: + type: string + default: "kubernetes.io/hostname" + pod_environment_configmap: + type: string + pod_environment_secret: + type: string + pod_management_policy: + type: string + enum: + - "ordered_ready" + - "parallel" + default: "ordered_ready" + pod_priority_class_name: + type: string + pod_role_label: + type: string + default: "spilo-role" + pod_service_account_definition: + type: string + default: "" + pod_service_account_name: + type: string + default: "postgres-pod" + pod_service_account_role_binding_definition: + type: string + default: "" + pod_terminate_grace_period: + type: string + default: "5m" + secret_name_template: + type: string + default: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}" + spilo_allow_privilege_escalation: + type: boolean + default: true + spilo_runasuser: + type: integer + spilo_runasgroup: + type: integer + spilo_fsgroup: + type: integer + spilo_privileged: + type: boolean + default: false + storage_resize_mode: + type: string + enum: + - "ebs" + - "pvc" + - "off" + default: "pvc" + toleration: + type: object + additionalProperties: + type: string + watched_namespace: + type: string + postgres_pod_resources: + type: object + properties: + default_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default: "1" + default_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default: "100m" + default_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + default: "500Mi" + default_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + default: "100Mi" + min_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default: "250m" + min_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + default: "250Mi" + timeouts: + type: object + properties: + pod_label_wait_timeout: + type: string + default: "10m" + pod_deletion_wait_timeout: + type: string + default: "10m" + ready_wait_interval: + type: string + default: "4s" + ready_wait_timeout: + type: string + default: "30s" + resource_check_interval: + type: string + default: "3s" + resource_check_timeout: + type: string + default: "10m" + load_balancer: + type: object + properties: + custom_service_annotations: + type: object + additionalProperties: + type: string + db_hosted_zone: + type: string + default: "db.example.com" + enable_master_load_balancer: + type: boolean + default: true + enable_replica_load_balancer: + type: boolean + default: false + external_traffic_policy: + type: string + enum: + - "Cluster" + - "Local" + default: "Cluster" + master_dns_name_format: + type: string + default: "{cluster}.{team}.{hostedzone}" + replica_dns_name_format: + type: string + default: "{cluster}-repl.{team}.{hostedzone}" + aws_or_gcp: + type: object + properties: + additional_secret_mount: + type: string + additional_secret_mount_path: + type: string + default: "/meta/credentials" + aws_region: + type: string + default: "eu-central-1" + enable_ebs_gp3_migration: + type: boolean + default: false + enable_ebs_gp3_migration_max_size: + type: integer + default: 1000 + gcp_credentials: + type: string + kube_iam_role: + type: string + log_s3_bucket: + type: string + wal_gs_bucket: + type: string + wal_s3_bucket: + type: string + logical_backup: + type: object + properties: + logical_backup_docker_image: + type: string + default: "registry.opensource.zalan.do/acid/logical-backup:v1.6.2" + logical_backup_google_application_credentials: + type: string + logical_backup_job_prefix: + type: string + default: "logical-backup-" + logical_backup_provider: + type: string + default: "s3" + logical_backup_s3_access_key_id: + type: string + logical_backup_s3_bucket: + type: string + logical_backup_s3_endpoint: + type: string + logical_backup_s3_region: + type: string + logical_backup_s3_secret_access_key: + type: string + logical_backup_s3_sse: + type: string + logical_backup_schedule: + type: string + pattern: '^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$' + default: "30 00 * * *" + debug: + type: object + properties: + debug_logging: + type: boolean + default: true + enable_database_access: + type: boolean + default: true + teams_api: + type: object + properties: + enable_admin_role_for_users: + type: boolean + default: true + enable_postgres_team_crd: + type: boolean + default: true + enable_postgres_team_crd_superusers: + type: boolean + default: false + enable_team_superuser: + type: boolean + default: false + enable_teams_api: + type: boolean + default: true + pam_configuration: + type: string + default: "https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees" + pam_role_name: + type: string + default: "zalandos" + postgres_superuser_teams: + type: array + items: + type: string + protected_role_names: + type: array + items: + type: string + default: + - admin + team_admin_role: + type: string + default: "admin" + team_api_role_configuration: + type: object + additionalProperties: + type: string + default: + log_statement: all + teams_api_url: + type: string + default: "https://teams.example.com/api/" + logging_rest_api: + type: object + properties: + api_port: + type: integer + default: 8080 + cluster_history_entries: + type: integer + default: 1000 + ring_log_lines: + type: integer + default: 100 + scalyr: # deprecated + type: object + properties: + scalyr_api_key: + type: string + scalyr_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default: "1" + scalyr_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default: "100m" + scalyr_image: + type: string + scalyr_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + default: "500Mi" + scalyr_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + default: "50Mi" + scalyr_server_url: + type: string + default: "https://upload.eu.scalyr.com" + connection_pooler: + type: object + properties: + connection_pooler_schema: + type: string + default: "pooler" + connection_pooler_user: + type: string + default: "pooler" + connection_pooler_image: + type: string + default: "registry.opensource.zalan.do/acid/pgbouncer:master-16" + connection_pooler_max_db_connections: + type: integer + default: 60 + connection_pooler_mode: + type: string + enum: + - "session" + - "transaction" + default: "transaction" + connection_pooler_number_of_instances: + type: integer + minimum: 1 + default: 2 + connection_pooler_default_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default: "1" + connection_pooler_default_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + default: "500m" + connection_pooler_default_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + default: "100Mi" + connection_pooler_default_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + default: "100Mi" + status: + type: object + additionalProperties: + type: string diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 78b618b78..7bebd704d 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -74,6 +74,7 @@ type KubernetesMetaConfiguration struct { InfrastructureRolesDefs []*config.InfrastructureRole `json:"infrastructure_roles_secrets,omitempty"` PodRoleLabel string `json:"pod_role_label,omitempty"` ClusterLabels map[string]string `json:"cluster_labels,omitempty"` + IgnoredAnnotations []string `json:"ignored_annotations,omitempty"` InheritedLabels []string `json:"inherited_labels,omitempty"` InheritedAnnotations []string `json:"inherited_annotations,omitempty"` DownscalerAnnotations []string `json:"downscaler_annotations,omitempty"` diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 512f6a6c9..c44ff8750 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -359,10 +359,10 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa match = false reasons = append(reasons, "new statefulset's number of replicas does not match the current one") } - if !reflect.DeepEqual(c.Statefulset.Annotations, statefulSet.Annotations) { + if changed, reason := c.compareAnnotations(c.Statefulset.Annotations, statefulSet.Annotations); changed { match = false needsReplace = true - reasons = append(reasons, "new statefulset's annotations do not match the current one") + reasons = append(reasons, "new statefulset's annotations do not match "+reason) } needsRollUpdate, reasons = c.compareContainers("initContainers", c.Statefulset.Spec.Template.Spec.InitContainers, statefulSet.Spec.Template.Spec.InitContainers, needsRollUpdate, reasons) @@ -411,11 +411,11 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *appsv1.StatefulSet) *compa } } - if !reflect.DeepEqual(c.Statefulset.Spec.Template.Annotations, statefulSet.Spec.Template.Annotations) { + if changed, reason := c.compareAnnotations(c.Statefulset.Spec.Template.Annotations, statefulSet.Spec.Template.Annotations); changed { match = false needsReplace = true needsRollUpdate = true - reasons = append(reasons, "new statefulset's pod template metadata annotations does not match the current one") + reasons = append(reasons, "new statefulset's pod template metadata annotations does not match "+reason) } if !reflect.DeepEqual(c.Statefulset.Spec.Template.Spec.SecurityContext, statefulSet.Spec.Template.Spec.SecurityContext) { match = false @@ -689,7 +689,8 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { updateFailed = true return } - if syncStatefulSet || !reflect.DeepEqual(oldSs, newSs) || !reflect.DeepEqual(oldSpec.Annotations, newSpec.Annotations) { + annotationsChanged, _ := c.compareAnnotations(oldSpec.Annotations, newSpec.Annotations) + if syncStatefulSet || !reflect.DeepEqual(oldSs, newSs) || annotationsChanged { c.logger.Debugf("syncing statefulsets") syncStatefulSet = false // TODO: avoid generating the StatefulSet object twice by passing it to syncStatefulSet diff --git a/pkg/util/k8sutil/k8sutil_test.go b/pkg/cluster/comparison_test.go similarity index 93% rename from pkg/util/k8sutil/k8sutil_test.go rename to pkg/cluster/comparison_test.go index b3e768501..e57049199 100644 --- a/pkg/util/k8sutil/k8sutil_test.go +++ b/pkg/cluster/comparison_test.go @@ -1,9 +1,10 @@ -package k8sutil +package cluster import ( "strings" "testing" + "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" v1 "k8s.io/api/core/v1" @@ -21,6 +22,15 @@ func newsService(ann map[string]string, svcT v1.ServiceType, lbSr []string) *v1. } func TestSameService(t *testing.T) { + cluster := Cluster{ + Config: Config{ + OpConfig: config.Config{ + IgnoredAnnotations: []string{ + "k8s.v1.cni.cncf.io/network-status", + }, + }, + }, + } tests := []struct { about string current *v1.Service @@ -288,11 +298,26 @@ func TestSameService(t *testing.T) { // Test just the prefix to avoid flakiness and map sorting reason: `new service's annotations does not match the current one: Added `, }, + { + about: "ignored annotations", + current: newsService( + map[string]string{}, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + new: newsService( + map[string]string{ + "k8s.v1.cni.cncf.io/network-status": "up", + }, + v1.ServiceTypeLoadBalancer, + []string{"128.141.0.0/16", "137.138.0.0/16"}), + match: true, + }, } for _, tt := range tests { t.Run(tt.about, func(t *testing.T) { - match, reason := SameService(tt.current, tt.new) + match, reason := cluster.SameService(tt.current, tt.new) if match && !tt.match { + t.Logf("match=%v current=%v, old=%v reason=%s", match, tt.current.Annotations, tt.new.Annotations, reason) t.Errorf("expected services to do not match: '%q' and '%q'", tt.current, tt.new) return } diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 3f08cfb4d..f3f2d10e9 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -153,7 +153,7 @@ func (c *Cluster) syncService(role PostgresRole) error { if svc, err = c.KubeClient.Services(c.Namespace).Get(context.TODO(), c.serviceName(role), metav1.GetOptions{}); err == nil { c.Services[role] = svc desiredSvc := c.generateService(role, &c.Spec) - if match, reason := k8sutil.SameService(svc, desiredSvc); !match { + if match, reason := c.SameService(svc, desiredSvc); !match { c.logServiceChanges(role, svc, desiredSvc, false, reason) if err = c.updateService(role, desiredSvc); err != nil { return fmt.Errorf("could not update %s service to match desired state: %v", role, err) diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index fa8a52a1b..9ec6f4e8d 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -186,6 +186,70 @@ func logNiceDiff(log *logrus.Entry, old, new interface{}) { } } +func (c *Cluster) compareAnnotations(old, new map[string]string) (bool, string) { + ignored := make(map[string]bool) + for _, ignore := range c.OpConfig.IgnoredAnnotations { + ignored[ignore] = true + } + + // changed := false + reason := "" + + for key := range old { + if _, ok := ignored[key]; ok { + continue + } + if _, ok := new[key]; !ok { + reason += fmt.Sprintf(" Removed '%s'.", key) + } + } + + for key := range new { + if _, ok := ignored[key]; ok { + continue + } + + v, ok := old[key] + if !ok { + reason += fmt.Sprintf(" Added '%s' with value '%s'.", key, new[key]) + } else if v != new[key] { + reason += fmt.Sprintf(" '%s' changed from '%s' to '%s'.", key, v, new[key]) + } + } + + if reason != "" { + return true, reason + } + return false, "" + +} + +// SameService compares the Services +func (c *Cluster) SameService(old, new *v1.Service) (bool, string) { + //TODO: improve comparison + if old.Spec.Type != new.Spec.Type { + return false, fmt.Sprintf("new service's type %q does not match the current one %q", + new.Spec.Type, old.Spec.Type) + } + + oldSourceRanges := old.Spec.LoadBalancerSourceRanges + newSourceRanges := new.Spec.LoadBalancerSourceRanges + + /* work around Kubernetes 1.6 serializing [] as nil. See https://github.com/kubernetes/kubernetes/issues/43203 */ + if (len(oldSourceRanges) != 0) || (len(newSourceRanges) != 0) { + if !reflect.DeepEqual(oldSourceRanges, newSourceRanges) { + return false, "new service's LoadBalancerSourceRange does not match the current one" + } + } + + if changed, reason := c.compareAnnotations(old.Annotations, new.Annotations); changed { + return !changed, "new service's annotations does not match the current one:" + reason + } + + return true, "" + +} + func (c *Cluster) logStatefulSetChanges(old, new *appsv1.StatefulSet, isUpdate bool, reasons []string) { if isUpdate { c.logger.Infof("statefulset %s has been changed", util.NameFromMeta(old.ObjectMeta)) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index f992ff782..4a3ba0ec8 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -184,6 +184,9 @@ func (c *Controller) modifyConfigFromEnvironment() { if c.config.NoTeamsAPI { c.opConfig.EnableTeamsAPI = false } + + c.opConfig.IgnoredAnnotations = append(c.opConfig.IgnoredAnnotations, c.config.IgnoredAnnotations...) + scalyrAPIKey := os.Getenv("SCALYR_API_KEY") if scalyrAPIKey != "" { c.opConfig.ScalyrAPIKey = scalyrAPIKey diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 3c0302cab..6c51d6bcc 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -60,6 +60,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.TargetMajorVersion = util.Coalesce(fromCRD.MajorVersionUpgrade.TargetMajorVersion, "13") // kubernetes config + result.IgnoredAnnotations = fromCRD.Kubernetes.IgnoredAnnotations result.CustomPodAnnotations = fromCRD.Kubernetes.CustomPodAnnotations result.PodServiceAccountName = util.Coalesce(fromCRD.Kubernetes.PodServiceAccountName, "postgres-pod") result.PodServiceAccountDefinition = fromCRD.Kubernetes.PodServiceAccountDefinition diff --git a/pkg/spec/types.go b/pkg/spec/types.go index 78c79e1b3..8ecffb299 100644 --- a/pkg/spec/types.go +++ b/pkg/spec/types.go @@ -114,6 +114,7 @@ type ControllerConfig struct { CRDReadyWaitTimeout time.Duration ConfigMapName NamespacedName Namespace string + IgnoredAnnotations []string EnableJsonLogging bool } diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index a5d144051..4a5ef917c 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -181,6 +181,7 @@ type Config struct { EnablePostgresTeamCRDSuperusers bool `name:"enable_postgres_team_crd_superusers" default:"false"` EnableMasterLoadBalancer bool `name:"enable_master_load_balancer" default:"true"` EnableReplicaLoadBalancer bool `name:"enable_replica_load_balancer" default:"false"` + IgnoredAnnotations []string `json:"ignored_annotations"` CustomServiceAnnotations map[string]string `name:"custom_service_annotations"` CustomPodAnnotations map[string]string `name:"custom_pod_annotations"` EnablePodAntiAffinity bool `name:"enable_pod_antiaffinity" default:"false"` diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index dd6ec1e8b..7e204cb89 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -197,57 +197,6 @@ func (client *KubernetesClient) SetPostgresCRDStatus(clusterName spec.Namespaced return pg, nil } -// SameService compares the Services -func SameService(cur, new *v1.Service) (match bool, reason string) { - //TODO: improve comparison - if cur.Spec.Type != new.Spec.Type { - return false, fmt.Sprintf("new service's type %q does not match the current one %q", - new.Spec.Type, cur.Spec.Type) - } - - oldSourceRanges := cur.Spec.LoadBalancerSourceRanges - newSourceRanges := new.Spec.LoadBalancerSourceRanges - - /* work around Kubernetes 1.6 serializing [] as nil. See https://github.com/kubernetes/kubernetes/issues/43203 */ - if (len(oldSourceRanges) != 0) || (len(newSourceRanges) != 0) { - if !reflect.DeepEqual(oldSourceRanges, newSourceRanges) { - return false, "new service's LoadBalancerSourceRange does not match the current one" - } - } - - match = true - - reasonPrefix := "new service's annotations does not match the current one:" - for ann := range cur.Annotations { - if _, ok := new.Annotations[ann]; !ok { - match = false - if len(reason) == 0 { - reason = reasonPrefix - } - reason += fmt.Sprintf(" Removed '%s'.", ann) - } - } - - for ann := range new.Annotations { - v, ok := cur.Annotations[ann] - if !ok { - if len(reason) == 0 { - reason = reasonPrefix - } - reason += fmt.Sprintf(" Added '%s' with value '%s'.", ann, new.Annotations[ann]) - match = false - } else if v != new.Annotations[ann] { - if len(reason) == 0 { - reason = reasonPrefix - } - reason += fmt.Sprintf(" '%s' changed from '%s' to '%s'.", ann, v, new.Annotations[ann]) - match = false - } - } - - return match, reason -} - // SamePDB compares the PodDisruptionBudgets func SamePDB(cur, new *policybeta1.PodDisruptionBudget) (match bool, reason string) { //TODO: improve comparison From 3ecd1e94a0083e6f18cb26e8381ffd84440cfc24 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Wed, 12 May 2021 17:11:12 +0200 Subject: [PATCH 2/2] feat: add ignored annotations Co-authored-by: Felix Kunde --- pkg/util/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 4a5ef917c..241e109a2 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -181,7 +181,7 @@ type Config struct { EnablePostgresTeamCRDSuperusers bool `name:"enable_postgres_team_crd_superusers" default:"false"` EnableMasterLoadBalancer bool `name:"enable_master_load_balancer" default:"true"` EnableReplicaLoadBalancer bool `name:"enable_replica_load_balancer" default:"false"` - IgnoredAnnotations []string `json:"ignored_annotations"` + IgnoredAnnotations []string `name:"ignored_annotations"` CustomServiceAnnotations map[string]string `name:"custom_service_annotations"` CustomPodAnnotations map[string]string `name:"custom_pod_annotations"` EnablePodAntiAffinity bool `name:"enable_pod_antiaffinity" default:"false"`