Add ServiceAnnotations cluster config

charts/postgres-operator/crds/postgresqls.yaml

@@ -266,6 +266,10 @@ spec:
                       pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$'
                       # Note: the value specified here must not be zero or be higher
                       # than the corresponding limit.
+            serviceAnnotations:
+              type: object
+              additionalProperties:
+                type: string
             sidecars:
               type: array
               nullable: true

docs/administrator.md

@@ -376,6 +376,17 @@ cluster manifest. In the case any of these variables are omitted from the
 manifest, the operator configuration settings `enable_master_load_balancer` and
 `enable_replica_load_balancer` apply. Note that the operator settings affect
 all Postgresql services running in all namespaces watched by the operator.
+If load balancing is enabled two default annotations will be applied to its
+services:
+
+- `external-dns.alpha.kubernetes.io/hostname` with the value defined by the
+  operator configs `master_dns_name_format` and `replica_dns_name_format`.
+  This value can't be overwritten. If any changing in its value is needed, it
+  MUST be done changing the DNS format operator config parameters; and
+- `service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout` with
+  a default value of "3600". This value can be overwritten with the operator
+  config parameter `custom_service_annotations` or the  cluster parameter
+  `serviceAnnotations`.
 
 To limit the range of IP addresses that can reach a load balancer, specify the
 desired ranges in the `allowedSourceRanges` field (applies to both master and

docs/reference/cluster_manifest.md

@@ -122,6 +122,11 @@ These parameters are grouped directly under  the `spec` key in the manifest.
   A map of key value pairs that gets attached as [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/)
   to each pod created for the database.
 
+* **serviceAnnotations**
+  A map of key value pairs that gets attached as [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/)
+  to the services created for the database cluster. Check the
+  [administrator docs](https://github.com/zalando/postgres-operator/blob/master/docs/administrator.md#load-balancers-and-allowed-ip-ranges)
+  for more information regarding default values and overwrite rules.
 
 * **enableShmVolume**
   Start a database pod without limitations on shm memory. By default Docker

docs/reference/operator_parameters.md

@@ -380,8 +380,9 @@ In the CRD-based configuration they are grouped under the `load_balancer` key.
   `false`.
 
 * **custom_service_annotations**
-  when load balancing is enabled, LoadBalancer service is created and
-  this parameter takes service annotations that are applied to service.
+  This key/value map provides a list of annotations that get attached to each
+  service of a cluster created by the operator. If the annotation key is also
+  provided by the cluster definition, the manifest value is used.
   Optional.
 
 * **master_dns_name_format** defines the DNS name string template for the

e2e/README.md

@@ -44,3 +44,4 @@ The current tests are all bundled in [`test_e2e.py`](tests/test_e2e.py):
 * taint-based eviction of Postgres pods
 * invoking logical backup cron job
 * uniqueness of master pod
+* custom service annotations

e2e/tests/test_e2e.py

@@ -161,8 +161,8 @@ def test_logical_backup_cron_job(self):
         schedule = "7 7 7 7 *"
         pg_patch_enable_backup = {
             "spec": {
-               "enableLogicalBackup": True,
-               "logicalBackupSchedule": schedule
+                "enableLogicalBackup": True,
+                "logicalBackupSchedule": schedule
             }
         }
         k8s.api.custom_objects_api.patch_namespaced_custom_object(
@@ -184,7 +184,7 @@ def test_logical_backup_cron_job(self):
         image = "test-image-name"
         patch_logical_backup_image = {
             "data": {
-               "logical_backup_docker_image": image,
+                "logical_backup_docker_image": image,
             }
         }
         k8s.update_config(patch_logical_backup_image)
@@ -197,7 +197,7 @@ def test_logical_backup_cron_job(self):
         # delete the logical backup cron job
         pg_patch_disable_backup = {
             "spec": {
-               "enableLogicalBackup": False,
+                "enableLogicalBackup": False,
             }
         }
         k8s.api.custom_objects_api.patch_namespaced_custom_object(
@@ -207,6 +207,37 @@ def test_logical_backup_cron_job(self):
         self.assertEqual(0, len(jobs),
                          "Expected 0 logical backup jobs, found {}".format(len(jobs)))
 
+    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
+    def test_service_annotations(self):
+        '''
+        Create a Postgres cluster with service annotations and check them.
+        '''
+        k8s = self.k8s
+        patch_custom_service_annotations = {
+            "data": {
+                "custom_service_annotations": "foo:bar",
+            }
+        }
+        k8s.update_config(patch_custom_service_annotations)
+
+        k8s.create_with_kubectl("manifests/postgres-manifest-with-service-annotations.yaml")
+        annotations = {
+            "annotation.key": "value",
+            "foo": "bar",
+        }
+        self.assertTrue(k8s.check_service_annotations(
+            "version=acid-service-annotations,spilo-role=master", annotations))
+        self.assertTrue(k8s.check_service_annotations(
+            "version=acid-service-annotations,spilo-role=replica", annotations))
+
+        # clean up
+        unpatch_custom_service_annotations = {
+            "data": {
+                "custom_service_annotations": "",
+            }
+        }
+        k8s.update_config(unpatch_custom_service_annotations)
+
     def assert_master_is_unique(self, namespace='default', version="acid-minimal-cluster"):
         """
            Check that there is a single pod in the k8s cluster with the label "spilo-role=master"
@@ -272,6 +303,16 @@ def wait_for_pod_start(self, pod_labels, namespace='default'):
                 pod_phase = pods[0].status.phase
             time.sleep(self.RETRY_TIMEOUT_SEC)
 
+    def check_service_annotations(self, svc_labels, annotations, namespace='default'):
+        svcs = self.api.core_v1.list_namespaced_service(namespace, label_selector=svc_labels, limit=1).items
+        for svc in svcs:
+            if len(svc.metadata.annotations) != len(annotations):
+                return False
+            for key in svc.metadata.annotations:
+                if svc.metadata.annotations[key] != annotations[key]:
+                    return False
+        return True
+
     def wait_for_pg_to_scale(self, number_of_instances, namespace='default'):
 
         body = {
@@ -280,7 +321,7 @@ def wait_for_pg_to_scale(self, number_of_instances, namespace='default'):
             }
         }
         _ = self.api.custom_objects_api.patch_namespaced_custom_object(
-                    "acid.zalan.do", "v1", namespace, "postgresqls", "acid-minimal-cluster", body)
+            "acid.zalan.do", "v1", namespace, "postgresqls", "acid-minimal-cluster", body)
 
         labels = 'version=acid-minimal-cluster'
         while self.count_pods_with_label(labels) != number_of_instances:

manifests/complete-postgres-manifest.yaml

@@ -32,6 +32,8 @@ spec:
 #  spiloFSGroup: 103
 #  podAnnotations:
 #    annotation.key: value
+#  serviceAnnotations:
+#    annotation.key: value
 #  podPriorityClassName: "spilo-pod-priority"
 #  tolerations:
 #  - key: postgres

manifests/postgres-manifest-with-service-annotations.yaml

@@ -0,0 +1,20 @@
+apiVersion: "acid.zalan.do/v1"
+kind: postgresql
+metadata:
+  name: acid-service-annotations
+spec:
+  teamId: "acid"
+  volume:
+    size: 1Gi
+  numberOfInstances: 2
+  users:
+    zalando:  # database owner
+    - superuser
+    - createdb
+    foo_user: []  # role for application foo
+  databases:
+    foo: zalando  # dbname: owner
+  postgresql:
+    version: "11"
+  serviceAnnotations:
+    annotation.key: value

manifests/postgresql.crd.yaml

@@ -230,6 +230,10 @@ spec:
                       pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$'
                       # Note: the value specified here must not be zero or be higher
                       # than the corresponding limit.
+            serviceAnnotations:
+              type: object
+              additionalProperties:
+                type: string
             sidecars:
               type: array
               nullable: true

pkg/apis/acid.zalan.do/v1/crds.go

@@ -383,6 +383,14 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{
 							},
 						},
 					},
+					"serviceAnnotations": {
+						Type: "object",
+						AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{
+							Schema: &apiextv1beta1.JSONSchemaProps{
+								Type: "string",
+							},
+						},
+					},
 					"sidecars": {
 						Type: "array",
 						Items: &apiextv1beta1.JSONSchemaPropsOrArray{

pkg/apis/acid.zalan.do/v1/postgresql_type.go

@@ -60,6 +60,7 @@ type PostgresSpec struct {
 	LogicalBackupSchedule string               `json:"logicalBackupSchedule,omitempty"`
 	StandbyCluster        *StandbyDescription  `json:"standby"`
 	PodAnnotations        map[string]string    `json:"podAnnotations"`
+	ServiceAnnotations    map[string]string    `json:"serviceAnnotations"`
 
 	// deprecated json tags
 	InitContainersOld       []v1.Container `json:"init_containers,omitempty"`

pkg/apis/acid.zalan.do/v1/util_test.go

@@ -456,18 +456,84 @@ var postgresqlList = []struct {
 		PostgresqlList{},
 		errors.New("unexpected end of JSON input")}}
 
-var annotations = []struct {
+var podAnnotations = []struct {
 	about       string
 	in          []byte
 	annotations map[string]string
 	err         error
 }{{
-	about:       "common annotations",
-	in:          []byte(`{"kind": "Postgresql","apiVersion": "acid.zalan.do/v1","metadata": {"name": "acid-testcluster1"}, "spec": {"podAnnotations": {"foo": "bar"},"teamId": "acid", "clone": {"cluster": "team-batman"}}}`),
+	about: "common annotations",
+	in: []byte(`{
+		"kind": "Postgresql",
+		"apiVersion": "acid.zalan.do/v1",
+		"metadata": {
+			"name": "acid-testcluster1"
+		},
+		"spec": {
+			"podAnnotations": {
+				"foo": "bar"
+			},
+			"teamId": "acid",
+			"clone": {
+				"cluster": "team-batman"
+			}
+		}
+	}`),
 	annotations: map[string]string{"foo": "bar"},
 	err:         nil},
 }
 
+var serviceAnnotations = []struct {
+	about       string
+	in          []byte
+	annotations map[string]string
+	err         error
+}{
+	{
+		about: "common single annotation",
+		in: []byte(`{
+			"kind": "Postgresql",
+			"apiVersion": "acid.zalan.do/v1",
+			"metadata": {
+				"name": "acid-testcluster1"
+			},
+			"spec": {
+				"serviceAnnotations": {
+					"foo": "bar"
+				},
+				"teamId": "acid",
+				"clone": {
+					"cluster": "team-batman"
+				}
+			}
+		}`),
+		annotations: map[string]string{"foo": "bar"},
+		err:         nil,
+	},
+	{
+		about: "common two annotations",
+		in: []byte(`{
+			"kind": "Postgresql",
+			"apiVersion": "acid.zalan.do/v1",
+			"metadata": {
+				"name": "acid-testcluster1"
+			},
+			"spec": {
+				"serviceAnnotations": {
+					"foo": "bar",
+					"post": "gres"
+				},
+				"teamId": "acid",
+				"clone": {
+					"cluster": "team-batman"
+				}
+			}
+		}`),
+		annotations: map[string]string{"foo": "bar", "post": "gres"},
+		err:         nil,
+	},
+}
+
 func mustParseTime(s string) metav1.Time {
 	v, err := time.Parse("15:04", s)
 	if err != nil {
@@ -517,21 +583,42 @@ func TestWeekdayTime(t *testing.T) {
 	}
 }
 
-func TestClusterAnnotations(t *testing.T) {
-	for _, tt := range annotations {
+func TestPodAnnotations(t *testing.T) {
+	for _, tt := range podAnnotations {
 		t.Run(tt.about, func(t *testing.T) {
 			var cluster Postgresql
 			err := cluster.UnmarshalJSON(tt.in)
 			if err != nil {
 				if tt.err == nil || err.Error() != tt.err.Error() {
-					t.Errorf("Unable to marshal cluster with annotations: expected %v got %v", tt.err, err)
+					t.Errorf("Unable to marshal cluster with podAnnotations: expected %v got %v", tt.err, err)
 				}
 				return
 			}
 			for k, v := range cluster.Spec.PodAnnotations {
 				found, expected := v, tt.annotations[k]
 				if found != expected {
-					t.Errorf("Didn't find correct value for key %v in for podAnnotations:  Expected %v found %v", k, expected, found)
+					t.Errorf("Didn't find correct value for key %v in for podAnnotations: Expected %v found %v", k, expected, found)
+				}
+			}
+		})
+	}
+}
+
+func TestServiceAnnotations(t *testing.T) {
+	for _, tt := range serviceAnnotations {
+		t.Run(tt.about, func(t *testing.T) {
+			var cluster Postgresql
+			err := cluster.UnmarshalJSON(tt.in)
+			if err != nil {
+				if tt.err == nil || err.Error() != tt.err.Error() {
+					t.Errorf("Unable to marshal cluster with serviceAnnotations: expected %v got %v", tt.err, err)
+				}
+				return
+			}
+			for k, v := range cluster.Spec.ServiceAnnotations {
+				found, expected := v, tt.annotations[k]
+				if found != expected {
+					t.Errorf("Didn't find correct value for key %v in for serviceAnnotations: Expected %v found %v", k, expected, found)
 				}
 			}
 		})