update to latest

.github/ISSUE_TEMPLATE/postgres-operator-issue-template.md

@@ -0,0 +1,19 @@
+---
+name: Postgres Operator issue template
+about: How are you using the operator?
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+Please, answer some short questions which should help us to understand your problem / question better?
+
+- **Which image of the operator are you using?** e.g. registry.opensource.zalan.do/acid/postgres-operator:v1.5.0
+- **Where do you run it - cloud or metal? Kubernetes or OpenShift?** [AWS K8s | GCP ... | Bare Metal K8s]
+- **Are you running Postgres Operator in production?** [yes | no]
+- **Type of issue?** [Bug report, question, feature request, etc.]
+
+Some general remarks when posting a bug report:
+- Please, check the operator, pod (Patroni) and postgresql logs first. When copy-pasting many log lines please do it in a separate GitHub gist together with your Postgres CRD and configuration manifest.
+- If you feel this issue might be more related to the [Spilo](https://github.com/zalando/spilo/issues) docker image or [Patroni](https://github.com/zalando/patroni/issues), consider opening issues in the respective repos.

.gitignore

@@ -30,6 +30,7 @@ _testmain.go
 /docker/build/
 /github.com/
 .idea
+.vscode
 
 scm-source.json
 
@@ -47,6 +48,8 @@ __pycache__/
 
 # Distribution / packaging
 .Python
+ui/app/node_modules
+ui/operator_ui/static/build
 build/
 develop-eggs/
 dist/

.travis.yml

@@ -18,5 +18,6 @@ install:
 
 script:
   - hack/verify-codegen.sh
-  - travis_wait 20 goveralls -service=travis-ci -package ./pkg/... -v
+  - travis_wait 20 go test -race -covermode atomic -coverprofile=profile.cov ./pkg/... -v
+  - goveralls -coverprofile=profile.cov -service=travis-ci -v
   - make e2e

Makefile

@@ -79,7 +79,7 @@ scm-source.json: .git
 
 tools:
 	GO111MODULE=on go get -u honnef.co/go/tools/cmd/staticcheck
-	GO111MODULE=on go get k8s.io/client-go@kubernetes-1.16.3
+	GO111MODULE=on go get k8s.io/client-go@kubernetes-1.18.8
 	GO111MODULE=on go mod tidy
 
 fmt:
@@ -97,4 +97,4 @@ test:
 	GO111MODULE=on go test ./...
 
 e2e: docker # build operator image to be tested
-	cd e2e; make tools e2etest clean
+	cd e2e; make e2etest

README.md

@@ -76,12 +76,6 @@ There is a browser-friendly version of this documentation at
 * [Postgres manifest reference](docs/reference/cluster_manifest.md)
 * [Command-line options and environment variables](docs/reference/command_line_and_environment.md)
 
-## Google Summer of Code
-
-The Postgres Operator made it to the [Google Summer of Code 2019](https://summerofcode.withgoogle.com/organizations/5429926902104064/)!
-Check [our ideas](docs/gsoc-2019/ideas.md#google-summer-of-code-2019)
-and start discussions in [the issue tracker](https://github.com/zalando/postgres-operator/issues).
-
 ## Community
 
 There are two places to get in touch with the community:

charts/postgres-operator-ui/templates/deployment.yaml

@@ -46,7 +46,7 @@ spec:
             - name: "RESOURCES_VISIBLE"
               value: "{{ .Values.envs.resourcesVisible }}"
             - name: "TARGET_NAMESPACE"
-              value: {{ .Values.envs.targetNamespace }}
+              value: "{{ .Values.envs.targetNamespace }}"
             - name: "TEAMS"
               value: |-
                 [

charts/postgres-operator/crds/operatorconfigurations.yaml

@@ -117,6 +117,10 @@ spec:
                   type: object
                   additionalProperties:
                     type: string
+                delete_annotation_date_key:
+                  type: string
+                delete_annotation_name_key:
+                  type: string
                 downscaler_annotations:
                   type: array
                   items:
@@ -131,6 +135,32 @@ spec:
                   type: boolean
                 infrastructure_roles_secret_name:
                   type: string
+                infrastructure_roles_secrets:
+                  type: array
+                  nullable: true
+                  items:
+                    type: object
+                    required:
+                      - secretname
+                      - userkey
+                      - passwordkey
+                    properties:
+                      secretname:
+                        type: string
+                      userkey:
+                        type: string
+                      passwordkey:
+                        type: string
+                      rolekey:
+                        type: string
+                      defaultuservalue:
+                        type: string
+                      defaultrolevalue:
+                        type: string
+                      details:
+                        type: string
+                      template:
+                        type: boolean
                 inherited_labels:
                   type: array
                   items:
@@ -149,6 +179,8 @@ spec:
                   type: string
                 pod_environment_configmap:
                   type: string
+                pod_environment_secret:
+                  type: string
                 pod_management_policy:
                   type: string
                   enum:
@@ -168,6 +200,10 @@ spec:
                   type: string
                 secret_name_template:
                   type: string
+                spilo_runasuser:
+                  type: integer
+                spilo_runasgroup:
+                  type: integer
                 spilo_fsgroup:
                   type: integer
                 spilo_privileged:
@@ -227,6 +263,11 @@ spec:
                   type: boolean
                 enable_replica_load_balancer:
                   type: boolean
+                external_traffic_policy:
+                  type: string
+                  enum:
+                    - "Cluster"
+                    - "Local"
                 master_dns_name_format:
                   type: string
                 replica_dns_name_format:

charts/postgres-operator/crds/postgresqls.yaml

@@ -374,6 +374,10 @@ spec:
               items:
                 type: object
                 additionalProperties: true
+            spiloRunAsUser:
+              type: integer
+            spiloRunAsGroup:
+              type: integer
             spiloFSGroup:
               type: integer
             standby:

charts/postgres-operator/templates/configmap.yaml

@@ -9,6 +9,9 @@ metadata:
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     app.kubernetes.io/instance: {{ .Release.Name }}
 data:
+  {{- if .Values.podPriorityClassName }}
+  pod_priority_class_name: {{ .Values.podPriorityClassName }}
+  {{- end }}
   pod_service_account_name: {{ include "postgres-pod.serviceAccountName" . }}
 {{ toYaml .Values.configGeneral | indent 2 }}
 {{ toYaml .Values.configUsers | indent 2 }}

charts/postgres-operator/templates/deployment.yaml

@@ -37,6 +37,10 @@ spec:
         image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}"
         imagePullPolicy: {{ .Values.image.pullPolicy }}
         env:
+      {{- if .Values.enableJsonLogging }}
+        - name: ENABLE_JSON_LOGGING
+          value: "true"
+      {{- end }}
       {{- if eq .Values.configTarget "ConfigMap" }}
         - name: CONFIG_MAP_NAME
           value: {{ template "postgres-operator.fullname" . }}

charts/postgres-operator/templates/operatorconfiguration.yaml

@@ -13,6 +13,9 @@ configuration:
   users:
 {{ toYaml .Values.configUsers | indent 4 }}
   kubernetes:
+    {{- if .Values.podPriorityClassName }}
+    pod_priority_class_name: {{ .Values.podPriorityClassName }}
+    {{- end }}
     pod_service_account_name: {{ include "postgres-pod.serviceAccountName" . }}
     oauth_token_secret_name: {{ template "postgres-operator.fullname" . }}
 {{ toYaml .Values.configKubernetes | indent 4 }}

charts/postgres-operator/templates/postgres-pod-priority-class.yaml

@@ -0,0 +1,15 @@
+{{- if .Values.podPriorityClassName }}
+apiVersion: scheduling.k8s.io/v1
+description: 'Use only for databases controlled by Postgres operator'
+kind: PriorityClass
+metadata:
+  labels:
+    app.kubernetes.io/name: {{ template "postgres-operator.name" . }}
+    helm.sh/chart: {{ template "postgres-operator.chart" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+  name: {{ .Values.podPriorityClassName }}
+preemptionPolicy: PreemptLowerPriority
+globalDefault: false
+value: 1000000
+{{- end }}

charts/postgres-operator/values-crd.yaml

@@ -67,6 +67,12 @@ configKubernetes:
   #   keya: valuea
   #   keyb: valueb
 
+  # key name for annotation that compares manifest value with current date
+  # delete_annotation_date_key: "delete-date"
+
+  # key name for annotation that compares manifest value with cluster name
+  # delete_annotation_name_key: "delete-clustername"
+
   # list of annotations propagated from cluster manifest to statefulset and deployment
   # downscaler_annotations:
   #   - deployment-time
@@ -104,6 +110,8 @@ configKubernetes:
   pod_antiaffinity_topology_key: "kubernetes.io/hostname"
   # namespaced name of the ConfigMap with environment variables to populate on every pod
   # pod_environment_configmap: "default/my-custom-config"
+  # name of the Secret (in cluster namespace) with environment variables to populate on every pod
+  # pod_environment_secret: "my-custom-secret"
 
   # specify the pod management policy of stateful sets of Postgres clusters
   pod_management_policy: "ordered_ready"
@@ -119,11 +127,16 @@ configKubernetes:
   pod_terminate_grace_period: 5m
   # template for database user secrets generated by the operator
   secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}"
+  # set user and group for the spilo container (required to run Spilo as non-root process)
+  # spilo_runasuser: "101"
+  # spilo_runasgroup: "103"
   # group ID with write-access to volumes (required to run Spilo as non-root process)
   # spilo_fsgroup: 103
 
   # whether the Spilo container should run in privileged mode
   spilo_privileged: false
+  # storage resize strategy, available options are: ebs, pvc, off
+  storage_resize_mode: ebs
   # operator watches for postgres objects in the given namespace
   watched_namespace: "*"  # listen to all namespaces
 
@@ -170,6 +183,8 @@ configLoadBalancer:
   enable_master_load_balancer: false
   # toggles service type load balancer pointing to the replica pod of the cluster
   enable_replica_load_balancer: false
+  # define external traffic policy for the load balancer
+  external_traffic_policy: "Cluster"
   # defines the DNS name string template for the master load balancer cluster
   master_dns_name_format: "{cluster}.{team}.{hostedzone}"
   # defines the DNS name string template for the replica load balancer cluster
@@ -271,7 +286,7 @@ configConnectionPooler:
   # db user for pooler to use
   connection_pooler_user: "pooler"
   # docker image
-  connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-8"
+  connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-9"
   # max db connections the pooler should hold
   connection_pooler_max_db_connections: 60
   # default pooling mode
@@ -305,8 +320,12 @@ podServiceAccount:
   # If not set a name is generated using the fullname template and "-pod" suffix
   name: "postgres-pod"
 
+# priority class for operator pod
 priorityClassName: ""
 
+# priority class for database pods
+podPriorityClassName: ""
+
 resources:
   limits:
     cpu: 500m

charts/postgres-operator/values.yaml

@@ -15,6 +15,9 @@ podLabels: {}
 
 configTarget: "ConfigMap"
 
+# JSON logging format
+enableJsonLogging: false
+
 # general configuration parameters
 configGeneral:
   # choose if deployment creates/updates CRDs with OpenAPIV3Validation
@@ -63,6 +66,12 @@ configKubernetes:
   # annotations attached to each database pod
   # custom_pod_annotations: "keya:valuea,keyb:valueb"
 
+  # key name for annotation that compares manifest value with current date
+  # delete_annotation_date_key: "delete-date"
+
+  # key name for annotation that compares manifest value with cluster name
+  # delete_annotation_name_key: "delete-clustername"
+
   # list of annotations propagated from cluster manifest to statefulset and deployment
   # downscaler_annotations: "deployment-time,downscaler/*"
 
@@ -95,6 +104,8 @@ configKubernetes:
   pod_antiaffinity_topology_key: "kubernetes.io/hostname"
   # namespaced name of the ConfigMap with environment variables to populate on every pod
   # pod_environment_configmap: "default/my-custom-config"
+  # name of the Secret (in cluster namespace) with environment variables to populate on every pod
+  # pod_environment_secret: "my-custom-secret"
 
   # specify the pod management policy of stateful sets of Postgres clusters
   pod_management_policy: "ordered_ready"
@@ -110,11 +121,16 @@ configKubernetes:
   pod_terminate_grace_period: 5m
   # template for database user secrets generated by the operator
   secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}"
+  # set user and group for the spilo container (required to run Spilo as non-root process)
+  # spilo_runasuser: "101"
+  # spilo_runasgroup: "103"
   # group ID with write-access to volumes (required to run Spilo as non-root process)
   # spilo_fsgroup: "103"
 
   # whether the Spilo container should run in privileged mode
   spilo_privileged: "false"
+  # storage resize strategy, available options are: ebs, pvc, off
+  storage_resize_mode: ebs
   # operator watches for postgres objects in the given namespace
   watched_namespace: "*"  # listen to all namespaces
 
@@ -159,6 +175,8 @@ configLoadBalancer:
   enable_master_load_balancer: "false"
   # toggles service type load balancer pointing to the replica pod of the cluster
   enable_replica_load_balancer: "false"
+  # define external traffic policy for the load balancer
+  external_traffic_policy: "Cluster"
   # defines the DNS name string template for the master load balancer cluster
   master_dns_name_format: '{cluster}.{team}.{hostedzone}'
   # defines the DNS name string template for the replica load balancer cluster
@@ -263,7 +281,7 @@ configConnectionPooler:
   # db user for pooler to use
   connection_pooler_user: "pooler"
   # docker image
-  connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-8"
+  connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-9"
   # max db connections the pooler should hold
   connection_pooler_max_db_connections: "60"
   # default pooling mode
@@ -297,8 +315,12 @@ podServiceAccount:
   # If not set a name is generated using the fullname template and "-pod" suffix
   name: "postgres-pod"
 
+# priority class for operator pod
 priorityClassName: ""
 
+# priority class for database pods
+podPriorityClassName: ""
+
 resources:
   limits:
     cpu: 500m

cmd/main.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	"flag"
-	"log"
+	log "github.com/sirupsen/logrus"
 	"os"
 	"os/signal"
 	"sync"
@@ -36,6 +36,8 @@ func init() {
 	flag.BoolVar(&config.NoTeamsAPI, "noteamsapi", false, "Disable all access to the teams API")
 	flag.Parse()
 
+	config.EnableJsonLogging = os.Getenv("ENABLE_JSON_LOGGING") == "true"
+
 	configMapRawName := os.Getenv("CONFIG_MAP_NAME")
 	if configMapRawName != "" {
 
@@ -63,6 +65,9 @@ func init() {
 func main() {
 	var err error
 
+	if config.EnableJsonLogging {
+		log.SetFormatter(&log.JSONFormatter{})
+	}
 	log.SetOutput(os.Stdout)
 	log.Printf("Spilo operator %s\n", version)