Merge branch 'master' into release-0.34

astronomer · Apr 1, 2024 · 2478379 · 2478379
2 parents 8bb87b0 + 08704b9
commit 2478379
Show file tree

Hide file tree

Showing 17 changed files with 263 additions and 68 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -360,14 +360,14 @@ workflows:
                 - quay.io/astronomer/ap-alertmanager:0.27.0
                 - quay.io/astronomer/ap-astro-ui:0.34.4
                 - quay.io/astronomer/ap-auth-sidecar:1.25.4
-                - quay.io/astronomer/ap-awsesproxy:1.5.0-7
+                - quay.io/astronomer/ap-awsesproxy:1.5.0-8
                 - quay.io/astronomer/ap-base:3.18.6-1
-                - quay.io/astronomer/ap-blackbox-exporter:0.24.0-6
+                - quay.io/astronomer/ap-blackbox-exporter:0.24.0-7
                 - quay.io/astronomer/ap-cli-install:0.26.23
-                - quay.io/astronomer/ap-commander:0.34.0
+                - quay.io/astronomer/ap-commander:0.34.1
                 - quay.io/astronomer/ap-configmap-reloader:0.12.0
-                - quay.io/astronomer/ap-curator:8.0.9
-                - quay.io/astronomer/ap-db-bootstrapper:0.31.11
+                - quay.io/astronomer/ap-curator:8.0.13-1
+                - quay.io/astronomer/ap-db-bootstrapper:0.31.12
                 - quay.io/astronomer/ap-default-backend:0.28.24
                 - quay.io/astronomer/ap-elasticsearch-exporter:1.7.0
                 - quay.io/astronomer/ap-elasticsearch:8.11.4
@@ -377,18 +377,18 @@ workflows:
                 - quay.io/astronomer/ap-init:3.18.6-1
                 - quay.io/astronomer/ap-kibana:8.11.4
                 - quay.io/astronomer/ap-kube-state:2.10.1
-                - quay.io/astronomer/ap-nats-exporter:0.14.0-1
-                - quay.io/astronomer/ap-nats-server:2.10.9-2
-                - quay.io/astronomer/ap-nats-streaming:0.25.6-1
+                - quay.io/astronomer/ap-nats-exporter:0.14.0-2
+                - quay.io/astronomer/ap-nats-server:2.10.10
+                - quay.io/astronomer/ap-nats-streaming:0.25.6-2
                 - quay.io/astronomer/ap-nginx-es:1.25.4
                 - quay.io/astronomer/ap-nginx:1.9.6
                 - quay.io/astronomer/ap-node-exporter:1.7.0
                 - quay.io/astronomer/ap-openresty:1.25.3-1
-                - quay.io/astronomer/ap-pgbouncer-krb:1.17.0-10
-                - quay.io/astronomer/ap-postgres-exporter:0.15.0-4
-                - quay.io/astronomer/ap-postgresql:15.6.0
+                - quay.io/astronomer/ap-pgbouncer-krb:1.17.0-11
+                - quay.io/astronomer/ap-postgres-exporter:0.15.0-5
+                - quay.io/astronomer/ap-postgresql:15.6.0-1
                 - quay.io/astronomer/ap-prometheus:2.45.4
-                - quay.io/astronomer/ap-registry:3.18.6
+                - quay.io/astronomer/ap-registry:3.18.6-2
                 - quay.io/astronomer/ap-vector:0.32.2-4
           context:
             - slack_team-software-infra-bot
@@ -399,14 +399,14 @@ workflows:
                 - quay.io/astronomer/ap-alertmanager:0.27.0
                 - quay.io/astronomer/ap-astro-ui:0.34.4
                 - quay.io/astronomer/ap-auth-sidecar:1.25.4
-                - quay.io/astronomer/ap-awsesproxy:1.5.0-7
+                - quay.io/astronomer/ap-awsesproxy:1.5.0-8
                 - quay.io/astronomer/ap-base:3.18.6-1
-                - quay.io/astronomer/ap-blackbox-exporter:0.24.0-6
+                - quay.io/astronomer/ap-blackbox-exporter:0.24.0-7
                 - quay.io/astronomer/ap-cli-install:0.26.23
-                - quay.io/astronomer/ap-commander:0.34.0
+                - quay.io/astronomer/ap-commander:0.34.1
                 - quay.io/astronomer/ap-configmap-reloader:0.12.0
-                - quay.io/astronomer/ap-curator:8.0.9
-                - quay.io/astronomer/ap-db-bootstrapper:0.31.11
+                - quay.io/astronomer/ap-curator:8.0.13-1
+                - quay.io/astronomer/ap-db-bootstrapper:0.31.12
                 - quay.io/astronomer/ap-default-backend:0.28.24
                 - quay.io/astronomer/ap-elasticsearch-exporter:1.7.0
                 - quay.io/astronomer/ap-elasticsearch:8.11.4
@@ -416,18 +416,18 @@ workflows:
                 - quay.io/astronomer/ap-init:3.18.6-1
                 - quay.io/astronomer/ap-kibana:8.11.4
                 - quay.io/astronomer/ap-kube-state:2.10.1
-                - quay.io/astronomer/ap-nats-exporter:0.14.0-1
-                - quay.io/astronomer/ap-nats-server:2.10.9-2
-                - quay.io/astronomer/ap-nats-streaming:0.25.6-1
+                - quay.io/astronomer/ap-nats-exporter:0.14.0-2
+                - quay.io/astronomer/ap-nats-server:2.10.10
+                - quay.io/astronomer/ap-nats-streaming:0.25.6-2
                 - quay.io/astronomer/ap-nginx-es:1.25.4
                 - quay.io/astronomer/ap-nginx:1.9.6
                 - quay.io/astronomer/ap-node-exporter:1.7.0
                 - quay.io/astronomer/ap-openresty:1.25.3-1
-                - quay.io/astronomer/ap-pgbouncer-krb:1.17.0-10
-                - quay.io/astronomer/ap-postgres-exporter:0.15.0-4
-                - quay.io/astronomer/ap-postgresql:15.6.0
+                - quay.io/astronomer/ap-pgbouncer-krb:1.17.0-11
+                - quay.io/astronomer/ap-postgres-exporter:0.15.0-5
+                - quay.io/astronomer/ap-postgresql:15.6.0-1
                 - quay.io/astronomer/ap-prometheus:2.45.4
-                - quay.io/astronomer/ap-registry:3.18.6
+                - quay.io/astronomer/ap-registry:3.18.6-2
                 - quay.io/astronomer/ap-vector:0.32.2-4
           context:
             - twistcli

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,5 +1,5 @@
 ---
-exclude: '(venv|\.vscode|tests/k8s_schema)' # regex
+exclude: '(venv|\.vscode|tests/k8s_schema|tests/chart_tests/test_data)' # regex
 repos:
   - repo: local
     hooks:

diff --git a/charts/astronomer/values.yaml b/charts/astronomer/values.yaml
@@ -15,11 +15,11 @@ tolerations: []
 images:
   commander:
     repository: quay.io/astronomer/ap-commander
-    tag: 0.34.0
+    tag: 0.34.1
     pullPolicy: IfNotPresent
   registry:
     repository: quay.io/astronomer/ap-registry
-    tag: 3.18.6
+    tag: 3.18.6-2
     pullPolicy: IfNotPresent
     # httpSecret: ~
   houston:
@@ -32,7 +32,7 @@ images:
     pullPolicy: IfNotPresent
   dbBootstrapper:
     repository: quay.io/astronomer/ap-db-bootstrapper
-    tag: 0.31.11
+    tag: 0.31.12
     pullPolicy: IfNotPresent
   cliInstall:
     repository: quay.io/astronomer/ap-cli-install

diff --git a/charts/elasticsearch/values.yaml b/charts/elasticsearch/values.yaml
@@ -18,7 +18,7 @@ images:
     pullPolicy: IfNotPresent
   curator:
     repository: quay.io/astronomer/ap-curator
-    tag: 8.0.9
+    tag: 8.0.13-1
     pullPolicy: IfNotPresent
   exporter:
     repository: quay.io/astronomer/ap-elasticsearch-exporter

diff --git a/charts/external-es-proxy/templates/_helpers.tpl b/charts/external-es-proxy/templates/_helpers.tpl
@@ -62,15 +62,21 @@ Create the name of the service account to use
 {{- end }}
 
 
+{{/*
+Option to add trust certs when privateCA or self signed certs are used
+with hosted elastic search. By defaults it is off when trustCaCerts are
+provided it will use that certs to trust the connection
+*/}}
+
 {{- define "external-es-proxy-trustcerts" -}}
 {{- if .Values.global.customLogging.trustCaCerts  }}
 {{- $secret_name := .Values.global.customLogging.trustCaCerts }}
-  proxy_ssl_trusted_certificate /etc/ssl/certs/{{ $secret_name }}.pem;
-  proxy_ssl_verify              on;
-  proxy_ssl_verify_depth        2;
-  proxy_ssl_session_reuse on;
+proxy_ssl_trusted_certificate /etc/ssl/certs/{{ $secret_name }}.pem;
+proxy_ssl_verify              on;
+proxy_ssl_verify_depth        2;
+proxy_ssl_session_reuse on;
 {{- else }}
-  proxy_ssl_verify              off;
+proxy_ssl_verify              off;
 {{- end }}
 {{- end }}
 
@@ -101,3 +107,20 @@ imagePullSecrets:
 {{ .Values.images.awsproxy.repository }}:{{ .Values.images.awsproxy.tag }}
 {{- end }}
 {{- end }}
+
+
+{{/*
+Switches the elasticsearch configuratiob based on customLogging
+when aws managed elastic search is confired awsesproxy settings is required
+to authenticate with aws managed elastic search or opensearch
+*/}}
+
+{{- define "external-es-proxy-nginx-location-common" -}}
+{{- if  or .Values.global.customLogging.awsSecretName  .Values.global.customLogging.awsServiceAccountAnnotation .Values.global.customLogging.awsIAMRole }}
+proxy_pass http://localhost:{{ .Values.service.awsproxy }};
+{{- else }}
+access_by_lua_file  /usr/local/openresty/nginx/conf/setenv.lua;
+proxy_pass {{.Values.global.customLogging.scheme}}://{{.Values.global.customLogging.host}}:{{.Values.global.customLogging.port}};
+{{- include "external-es-proxy-trustcerts" . }}
+{{- end }}
+{{- end }}
diff --git a/charts/external-es-proxy/templates/external-es-proxy-configmap.yaml b/charts/external-es-proxy/templates/external-es-proxy-configmap.yaml
@@ -31,27 +31,23 @@ data:
         proxy_set_header Connection "Keep-Alive";
         proxy_set_header Proxy-Connection "Keep-Alive";
 
-        location ~ ^/ {
-          {{- if  or .Values.global.customLogging.awsSecretName  .Values.global.customLogging.awsServiceAccountAnnotation .Values.global.customLogging.awsIAMRole }}
-          proxy_pass http://localhost:{{ .Values.service.awsproxy }};
-          {{- else }}
-          access_by_lua_file  /usr/local/openresty/nginx/conf/setenv.lua;
-          proxy_pass {{.Values.global.customLogging.scheme}}://{{.Values.global.customLogging.host}}:{{.Values.global.customLogging.port}};
-          {{- include "external-es-proxy-trustcerts" . | indent 8 }}
-          {{- end }}
+        # The following "location" rules limit airflow interactions to only their indices. Any further
+        # additions should follow this pattern.
+        location ~* /_count$ {
+          rewrite /_count(.*) /{{ include "logging.indexNamePrefix" . }}.$remote_user.*/_count$1 break;
+          {{- include "external-es-proxy-nginx-location-common" . | indent 10 }}
+        }
+
+        location ~* /_bulk$ {
+          rewrite /_bulk(.*) /{{ include "logging.indexNamePrefix" . }}.$remote_user.*/_bulk$1 break;
+          {{- include "external-es-proxy-nginx-location-common" . | indent 10 }}
         }
 
         location = /_search {
           # This combined with disabling explicit index searching downstream
           # prevents any deployment from being able to query any other indexes.
           rewrite ^/(.*) /{{ include "logging.indexNamePrefix" . }}.$remote_user.*/$1 break;
-          {{- if  or .Values.global.customLogging.awsSecretName  .Values.global.customLogging.awsServiceAccountAnnotation .Values.global.customLogging.awsIAMRole }}
-          proxy_pass http://localhost:{{ .Values.service.awsproxy }};
-          {{- else }}
-          access_by_lua_file  /usr/local/openresty/nginx/conf/setenv.lua;
-          proxy_pass {{.Values.global.customLogging.scheme}}://{{.Values.global.customLogging.host}}:{{.Values.global.customLogging.port}};
-          {{- include "external-es-proxy-trustcerts" . | indent 8 }}
-          {{- end }}
+          {{- include "external-es-proxy-nginx-location-common" . | indent 10 }}
         }
 
         location = /auth {
@@ -60,6 +56,15 @@ data:
           proxy_set_header Content-Length "";
           proxy_set_header X-Original-URI $request_uri;
         }
+
+        location = /_cluster/state/version {
+          {{- include "external-es-proxy-nginx-location-common" . | indent 10 }}
+        }
+
+        location = /_cluster/health {
+          {{- include "external-es-proxy-nginx-location-common" . | indent 10 }}
+        }
+
       }
       server {
         listen {{ .Values.service.http }};
@@ -68,13 +73,7 @@ data:
         proxy_set_header Proxy-Connection "Keep-Alive";
 
         location ~ ^/ {
-          {{- if  or .Values.global.customLogging.awsSecretName  .Values.global.customLogging.awsServiceAccountAnnotation .Values.global.customLogging.awsIAMRole }}
-          proxy_pass http://localhost:{{ .Values.service.awsproxy }};
-          {{- else }}
-          access_by_lua_file  /usr/local/openresty/nginx/conf/setenv.lua;
-          proxy_pass {{.Values.global.customLogging.scheme}}://{{.Values.global.customLogging.host}}:{{.Values.global.customLogging.port}};
-          {{- include "external-es-proxy-trustcerts" . | indent 8 }}
-          {{- end }}
+          {{- include "external-es-proxy-nginx-location-common" . | indent 10 }}
         }
       }
     }
diff --git a/charts/external-es-proxy/values.yaml b/charts/external-es-proxy/values.yaml
@@ -11,7 +11,7 @@ images:
     pullPolicy: IfNotPresent
   awsproxy:
     repository: quay.io/astronomer/ap-awsesproxy
-    tag: 1.5.0-7
+    tag: 1.5.0-8
     pullPolicy: IfNotPresent
 
 imagePullSecrets: []

diff --git a/charts/grafana/values.yaml b/charts/grafana/values.yaml
@@ -13,7 +13,7 @@ images:
     pullPolicy: IfNotPresent
   dbBootstrapper:
     repository: quay.io/astronomer/ap-db-bootstrapper
-    tag: 0.31.11
+    tag: 0.31.12
     pullPolicy: IfNotPresent
 
 securityContext:

diff --git a/charts/nats/values.yaml b/charts/nats/values.yaml
@@ -6,15 +6,15 @@
 images:
   nats:
     repository: quay.io/astronomer/ap-nats-server
-    tag: 2.10.9-2
+    tag: 2.10.10
     pullPolicy: IfNotPresent
   exporter:
     repository: quay.io/astronomer/ap-nats-exporter
-    tag: 0.14.0-1
+    tag: 0.14.0-2
     pullPolicy: IfNotPresent
   dbBootstrapper:
     repository: quay.io/astronomer/ap-db-bootstrapper
-    tag: 0.31.11
+    tag: 0.31.12
     pullPolicy: IfNotPresent
 
 

diff --git a/charts/pgbouncer/values.yaml b/charts/pgbouncer/values.yaml
@@ -3,7 +3,7 @@
 #############################
 image:
   repository: quay.io/astronomer/ap-pgbouncer-krb
-  tag: 1.17.0-10
+  tag: 1.17.0-11
   pullPolicy: IfNotPresent
 
 securityContext:

diff --git a/charts/postgresql/values.yaml b/charts/postgresql/values.yaml
@@ -15,7 +15,7 @@ global:
 image:
   registry: quay.io
   repository: astronomer/ap-postgresql
-  tag: 15.6.0
+  tag: 15.6.0-1
   ## Specify a imagePullPolicy
   ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
   ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images

diff --git a/charts/prometheus-blackbox-exporter/values.yaml b/charts/prometheus-blackbox-exporter/values.yaml
@@ -14,7 +14,7 @@ strategy:
 
 image:
   repository: quay.io/astronomer/ap-blackbox-exporter
-  tag: 0.24.0-6
+  tag: 0.24.0-7
   pullPolicy: IfNotPresent
 
   ## Optionally specify an array of imagePullSecrets.

diff --git a/charts/prometheus-postgres-exporter/values.yaml b/charts/prometheus-postgres-exporter/values.yaml
@@ -9,7 +9,7 @@ replicaCount: 2
 
 image:
   repository: quay.io/astronomer/ap-postgres-exporter
-  tag: 0.15.0-4
+  tag: 0.15.0-5
   pullPolicy: IfNotPresent
 
 

diff --git a/charts/stan/values.yaml b/charts/stan/values.yaml
@@ -10,11 +10,11 @@ images:
     pullPolicy: IfNotPresent
   stan:
     repository: quay.io/astronomer/ap-nats-streaming
-    tag: 0.25.6-1
+    tag: 0.25.6-2
     pullPolicy: IfNotPresent
   exporter:
     repository: quay.io/astronomer/ap-nats-exporter
-    tag: 0.14.0-1
+    tag: 0.14.0-2
     pullPolicy: IfNotPresent