diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index a435fe6fe65cc..b9a7587b6b28f 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -50,6 +50,14 @@ If release name contains chart name it will be used as a full name. {{- end }} {{- end }} +{{- define "airflow.tplDict" -}} + {{- $rendered := dict -}} + {{- range $key, $value := .values }} + {{- $_ := set $rendered $key (tpl (toString $value) $.context) -}} + {{- end }} + {{- toYaml $rendered -}} +{{- end }} + {{/* Standard Airflow environment variables */}} {{- define "standard_airflow_environment" }} # Hard Coded Airflow Envs @@ -497,8 +505,8 @@ If release name contains chart name it will be used as a full name. {{ $pgMetadataHost := .Values.data.metadataConnection.host | default (printf "%s-%s.%s" .Release.Name "postgresql" .Release.Namespace) }} {{ $pgResultBackendHost := $resultBackendConnection.host | default (printf "%s-%s.%s" .Release.Name "postgresql" .Release.Namespace) }} [databases] -{{ .Release.Name }}-metadata = host={{ $pgMetadataHost }} dbname={{ .Values.data.metadataConnection.db }} port={{ .Values.data.metadataConnection.port }} pool_size={{ .Values.pgbouncer.metadataPoolSize }} {{ .Values.pgbouncer.extraIniMetadata | default "" }} -{{ .Release.Name }}-result-backend = host={{ $pgResultBackendHost }} dbname={{ $resultBackendConnection.db }} port={{ $resultBackendConnection.port }} pool_size={{ .Values.pgbouncer.resultBackendPoolSize }} {{ .Values.pgbouncer.extraIniResultBackend | default "" }} +{{ .Release.Name }}-metadata = host={{ $pgMetadataHost }} dbname={{ tpl .Values.data.metadataConnection.db . }} port={{ .Values.data.metadataConnection.port }} pool_size={{ .Values.pgbouncer.metadataPoolSize }} {{ .Values.pgbouncer.extraIniMetadata | default "" }} +{{ .Release.Name }}-result-backend = host={{ $pgResultBackendHost }} dbname={{ tpl $resultBackendConnection.db . }} port={{ $resultBackendConnection.port }} pool_size={{ .Values.pgbouncer.resultBackendPoolSize }} {{ .Values.pgbouncer.extraIniResultBackend | default "" }} [pgbouncer] pool_mode = transaction @@ -506,7 +514,7 @@ listen_port = {{ .Values.ports.pgbouncer }} listen_addr = * auth_type = {{ .Values.pgbouncer.auth_type }} auth_file = {{ .Values.pgbouncer.auth_file }} -stats_users = {{ .Values.data.metadataConnection.user }} +stats_users = {{ tpl .Values.data.metadataConnection.user . }} ignore_startup_parameters = extra_float_digits max_client_conn = {{ .Values.pgbouncer.maxClientConn }} verbose = {{ .Values.pgbouncer.verbose }} @@ -533,8 +541,8 @@ server_tls_key_file = /etc/pgbouncer/server.key {{ define "pgbouncer_users" }} {{- $resultBackendConnection := .Values.data.resultBackendConnection | default .Values.data.metadataConnection }} -{{ .Values.data.metadataConnection.user | quote }} {{ .Values.data.metadataConnection.pass | quote }} -{{ $resultBackendConnection.user | quote }} {{ $resultBackendConnection.pass | quote }} +{{ tpl .Values.data.metadataConnection.user . | quote }} {{ .Values.data.metadataConnection.pass | quote }} +{{ tpl $resultBackendConnection.user . | quote }} {{ $resultBackendConnection.pass | quote }} {{- end }} {{- define "airflow_logs" -}} @@ -599,7 +607,11 @@ server_tls_key_file = /etc/pgbouncer/server.key {{- end }} {{- define "airflow_webserver_config_configmap_name" -}} - {{- default (printf "%s-webserver-config" (include "airflow.fullname" .)) .Values.webserver.webserverConfigConfigMapName }} + {{- if .Values.webserver.webserverConfigConfigMapName }} + {{- tpl .Values.webserver.webserverConfigConfigMapName . }} + {{- else }} + {{- printf "%s-webserver-config" (include "airflow.fullname" .) }} + {{- end }} {{- end }} {{- define "airflow_webserver_config_mount" -}} @@ -610,7 +622,11 @@ server_tls_key_file = /etc/pgbouncer/server.key {{- end }} {{- define "airflow_api_server_config_configmap_name" -}} - {{- default (printf "%s-api-server-config" (include "airflow.fullname" .)) .Values.apiServer.apiServerConfigConfigMapName }} + {{- if .Values.apiServer.apiServerConfigConfigMapName }} + {{- tpl .Values.apiServer.apiServerConfigConfigMapName . }} + {{- else }} + {{- printf "%s-api-server-config" (include "airflow.fullname" .) }} + {{- end }} {{- end }} {{- define "airflow_api_server_config_mount" -}} diff --git a/chart/templates/api-server/api-server-serviceaccount.yaml b/chart/templates/api-server/api-server-serviceaccount.yaml index 0cd9984df96ec..f9bbaf8df5775 100644 --- a/chart/templates/api-server/api-server-serviceaccount.yaml +++ b/chart/templates/api-server/api-server-serviceaccount.yaml @@ -36,6 +36,7 @@ metadata: {{- mustMerge .Values.apiServer.labels .Values.labels | toYaml | nindent 4 }} {{- end }} {{- with .Values.apiServer.serviceAccount.annotations }} - annotations: {{- toYaml . | nindent 4 }} + annotations: + {{- include "airflow.tplDict" (dict "values" . "context" $) | nindent 4 }} {{- end }} {{- end }} diff --git a/chart/templates/dag-processor/dag-processor-serviceaccount.yaml b/chart/templates/dag-processor/dag-processor-serviceaccount.yaml index e47511617f12f..89b71eb40f45e 100644 --- a/chart/templates/dag-processor/dag-processor-serviceaccount.yaml +++ b/chart/templates/dag-processor/dag-processor-serviceaccount.yaml @@ -40,6 +40,7 @@ metadata: {{- mustMerge .Values.dagProcessor.labels .Values.labels | toYaml | nindent 4 }} {{- end }} {{- with .Values.dagProcessor.serviceAccount.annotations}} - annotations: {{- toYaml . | nindent 4 }} + annotations: + {{- include "airflow.tplDict" (dict "values" . "context" $) | nindent 4 }} {{- end }} {{- end }} diff --git a/chart/templates/pgbouncer/pgbouncer-serviceaccount.yaml b/chart/templates/pgbouncer/pgbouncer-serviceaccount.yaml index c9f757eb10390..53712ff3884b4 100644 --- a/chart/templates/pgbouncer/pgbouncer-serviceaccount.yaml +++ b/chart/templates/pgbouncer/pgbouncer-serviceaccount.yaml @@ -36,6 +36,7 @@ metadata: {{- mustMerge .Values.pgbouncer.labels .Values.labels | toYaml | nindent 4 }} {{- end }} {{- with .Values.pgbouncer.serviceAccount.annotations }} - annotations: {{- toYaml . | nindent 4 }} + annotations: + {{- include "airflow.tplDict" (dict "values" . "context" $) | nindent 4 }} {{- end }} {{- end }} diff --git a/chart/templates/scheduler/scheduler-serviceaccount.yaml b/chart/templates/scheduler/scheduler-serviceaccount.yaml index b87a5149d4f47..0142c367c9a03 100644 --- a/chart/templates/scheduler/scheduler-serviceaccount.yaml +++ b/chart/templates/scheduler/scheduler-serviceaccount.yaml @@ -39,8 +39,6 @@ metadata: {{- end }} {{- with .Values.scheduler.serviceAccount.annotations }} annotations: - {{- range $key, $value := . }} - {{- printf "%s: %s" $key (tpl $value $ | quote) | nindent 4 }} - {{- end }} + {{- include "airflow.tplDict" (dict "values" . "context" $) | nindent 4 }} {{- end }} {{- end }} diff --git a/chart/templates/secrets/metadata-connection-secret.yaml b/chart/templates/secrets/metadata-connection-secret.yaml index d64637d805eab..71441755d4bae 100644 --- a/chart/templates/secrets/metadata-connection-secret.yaml +++ b/chart/templates/secrets/metadata-connection-secret.yaml @@ -27,7 +27,8 @@ {{- $host := ternary $pgbouncerHost $metadataHost .Values.pgbouncer.enabled }} {{- $metadataPort := .Values.data.metadataConnection.port | toString }} {{- $port := (ternary .Values.ports.pgbouncer $metadataPort .Values.pgbouncer.enabled) | toString }} -{{- $metadataDatabase := .Values.data.metadataConnection.db }} +{{- $metadataUser := tpl .Values.data.metadataConnection.user . }} +{{- $metadataDatabase := tpl .Values.data.metadataConnection.db . }} {{- $database := ternary (printf "%s-%s" .Release.Name "metadata") $metadataDatabase .Values.pgbouncer.enabled }} {{- $query := ternary (printf "sslmode=%s" .Values.data.metadataConnection.sslmode) "" (eq .Values.data.metadataConnection.protocol "postgresql") }} {{- $kedaEnabled := .Values.workers.keda.enabled }} @@ -55,15 +56,15 @@ metadata: type: Opaque data: {{- with .Values.data.metadataConnection }} - connection: {{ urlJoin (dict "scheme" .protocol "userinfo" (printf "%s:%s" (.user | urlquery) (.pass | urlquery) ) "host" (printf "%s:%s" $host $port) "path" (printf "/%s" $database) "query" $query) | b64enc | quote }} + connection: {{ urlJoin (dict "scheme" .protocol "userinfo" (printf "%s:%s" ($metadataUser | urlquery) (.pass | urlquery) ) "host" (printf "%s:%s" $host $port) "path" (printf "/%s" $database) "query" $query) | b64enc | quote }} {{- end }} {{- if and $kedaEnabled .Values.pgbouncer.enabled (not $kedaUsePgBouncer) }} {{- with .Values.data.metadataConnection }} - kedaConnection: {{ urlJoin (dict "scheme" .protocol "userinfo" (printf "%s:%s" (.user | urlquery) (.pass | urlquery) ) "host" (printf "%s:%s" $metadataHost $metadataPort) "path" (printf "/%s" $metadataDatabase) "query" $query) | b64enc | quote }} + kedaConnection: {{ urlJoin (dict "scheme" .protocol "userinfo" (printf "%s:%s" ($metadataUser | urlquery) (.pass | urlquery) ) "host" (printf "%s:%s" $metadataHost $metadataPort) "path" (printf "/%s" $metadataDatabase) "query" $query) | b64enc | quote }} {{- end }} {{- else if and (or $kedaEnabled .Values.triggerer.keda.enabled) (eq .Values.data.metadataConnection.protocol "mysql") }} {{- with .Values.data.metadataConnection }} - kedaConnection: {{ urlJoin (dict "userinfo" (printf "%s:%s" (.user | urlquery) (.pass | urlquery) ) "host" (printf "tcp(%s:%s)" $metadataHost $metadataPort) "path" (printf "/%s" $metadataDatabase) "query" $query) | trimPrefix "//" | b64enc | quote }} + kedaConnection: {{ urlJoin (dict "userinfo" (printf "%s:%s" ($metadataUser | urlquery) (.pass | urlquery) ) "host" (printf "tcp(%s:%s)" $metadataHost $metadataPort) "path" (printf "/%s" $metadataDatabase) "query" $query) | trimPrefix "//" | b64enc | quote }} {{- end }} {{- end }} {{- end }} diff --git a/chart/templates/triggerer/triggerer-serviceaccount.yaml b/chart/templates/triggerer/triggerer-serviceaccount.yaml index 27fd76d08021e..e2fed78e3329a 100644 --- a/chart/templates/triggerer/triggerer-serviceaccount.yaml +++ b/chart/templates/triggerer/triggerer-serviceaccount.yaml @@ -36,6 +36,7 @@ metadata: {{- mustMerge .Values.triggerer.labels .Values.labels | toYaml | nindent 4 }} {{- end }} {{- with .Values.triggerer.serviceAccount.annotations}} - annotations: {{- toYaml . | nindent 4 }} + annotations: + {{- include "airflow.tplDict" (dict "values" . "context" $) | nindent 4 }} {{- end }} {{- end }} diff --git a/chart/templates/webserver/webserver-serviceaccount.yaml b/chart/templates/webserver/webserver-serviceaccount.yaml index e105dbde0a039..c3ae6aff97838 100644 --- a/chart/templates/webserver/webserver-serviceaccount.yaml +++ b/chart/templates/webserver/webserver-serviceaccount.yaml @@ -36,6 +36,7 @@ metadata: {{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 4 }} {{- end }} {{- with .Values.webserver.serviceAccount.annotations }} - annotations: {{- toYaml . | nindent 4 }} + annotations: + {{- include "airflow.tplDict" (dict "values" . "context" $) | nindent 4 }} {{- end }} {{- end }} diff --git a/chart/templates/workers/worker-serviceaccount.yaml b/chart/templates/workers/worker-serviceaccount.yaml index d8e377f56f401..cbcf95381e834 100644 --- a/chart/templates/workers/worker-serviceaccount.yaml +++ b/chart/templates/workers/worker-serviceaccount.yaml @@ -49,7 +49,8 @@ metadata: {{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }} {{- end }} {{- with .Values.workers.serviceAccount.annotations}} - annotations: {{- toYaml . | nindent 4 }} + annotations: + {{- include "airflow.tplDict" (dict "values" . "context" $) | nindent 4 }} {{- end }} {{- end }} {{- end }} diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_annotations.py b/helm-tests/tests/helm_tests/airflow_aux/test_annotations.py index b10cc00cf38e3..1bbf800c46af3 100644 --- a/helm-tests/tests/helm_tests/airflow_aux/test_annotations.py +++ b/helm-tests/tests/helm_tests/airflow_aux/test_annotations.py @@ -361,6 +361,158 @@ def test_annotations_are_added(self, values, show_only, expected_annotations): assert k in obj["metadata"]["annotations"] assert v == obj["metadata"]["annotations"][k] + @pytest.mark.parametrize( + ("values_key", "show_only"), + [ + ("scheduler", "templates/scheduler/scheduler-serviceaccount.yaml"), + ("triggerer", "templates/triggerer/triggerer-serviceaccount.yaml"), + ], + ) + def test_tpl_rendered_annotations_airflow_2(self, values_key, show_only): + """Test SA annotations support tpl rendering for Airflow 2.x components.""" + k8s_objects = render_chart( + values={ + "airflowVersion": "2.11.0", + values_key: { + "serviceAccount": { + "annotations": { + "iam.gke.io/gcp-service-account": "{{ .Release.Name }}-sa@project.iam", + }, + }, + }, + }, + show_only=[show_only], + ) + assert len(k8s_objects) == 1 + annotations = k8s_objects[0]["metadata"]["annotations"] + assert annotations["iam.gke.io/gcp-service-account"] == "release-name-sa@project.iam" + + def test_tpl_rendered_multiple_annotations(self): + """Test that multiple annotations render correctly with tpl.""" + k8s_objects = render_chart( + values={ + "airflowVersion": "2.11.0", + "scheduler": { + "serviceAccount": { + "annotations": { + "iam.gke.io/gcp-service-account": "{{ .Release.Name }}-sa@project.iam", + "another-annotation": "{{ .Release.Name }}-other", + "plain-annotation": "no-template", + }, + }, + }, + }, + show_only=["templates/scheduler/scheduler-serviceaccount.yaml"], + ) + assert len(k8s_objects) == 1 + annotations = k8s_objects[0]["metadata"]["annotations"] + assert annotations["iam.gke.io/gcp-service-account"] == "release-name-sa@project.iam" + assert annotations["another-annotation"] == "release-name-other" + assert annotations["plain-annotation"] == "no-template" + + def test_tpl_rendered_annotations_pgbouncer(self): + """Test pgbouncer SA annotations support tpl rendering.""" + k8s_objects = render_chart( + values={ + "airflowVersion": "2.11.0", + "pgbouncer": { + "enabled": True, + "serviceAccount": { + "annotations": { + "iam.gke.io/gcp-service-account": "{{ .Release.Name }}-sa@project.iam", + }, + }, + }, + }, + show_only=["templates/pgbouncer/pgbouncer-serviceaccount.yaml"], + ) + assert len(k8s_objects) == 1 + annotations = k8s_objects[0]["metadata"]["annotations"] + assert annotations["iam.gke.io/gcp-service-account"] == "release-name-sa@project.iam" + + @pytest.mark.parametrize( + ("values_key", "show_only"), + [ + ("dagProcessor", "templates/dag-processor/dag-processor-serviceaccount.yaml"), + ("apiServer", "templates/api-server/api-server-serviceaccount.yaml"), + ], + ) + def test_tpl_rendered_annotations_airflow_3(self, values_key, show_only): + """Test SA annotations support tpl rendering for Airflow 3.x components.""" + k8s_objects = render_chart( + values={ + values_key: { + "serviceAccount": { + "annotations": { + "iam.gke.io/gcp-service-account": "{{ .Release.Name }}-sa@project.iam", + }, + }, + }, + }, + show_only=[show_only], + ) + assert len(k8s_objects) == 1 + annotations = k8s_objects[0]["metadata"]["annotations"] + assert annotations["iam.gke.io/gcp-service-account"] == "release-name-sa@project.iam" + + def test_tpl_rendered_annotations_celery_worker(self): + """Test Celery worker SA annotations support tpl rendering.""" + k8s_objects = render_chart( + values={ + "workers": { + "celery": { + "serviceAccount": { + "annotations": { + "iam.gke.io/gcp-service-account": "{{ .Release.Name }}-worker@project.iam", + }, + }, + }, + }, + }, + show_only=["templates/workers/worker-serviceaccount.yaml"], + ) + assert len(k8s_objects) == 1 + annotations = k8s_objects[0]["metadata"]["annotations"] + assert annotations["iam.gke.io/gcp-service-account"] == "release-name-worker@project.iam" + + def test_tpl_rendered_annotations_kubernetes_worker(self): + """Test KubernetesExecutor worker SA annotations support tpl rendering.""" + k8s_objects = render_chart( + values={ + "executor": "KubernetesExecutor", + "workers": { + "serviceAccount": { + "annotations": { + "iam.gke.io/gcp-service-account": "{{ .Release.Name }}-worker@project.iam", + }, + }, + }, + }, + show_only=["templates/workers/worker-serviceaccount.yaml"], + ) + assert len(k8s_objects) == 1 + annotations = k8s_objects[0]["metadata"]["annotations"] + assert annotations["iam.gke.io/gcp-service-account"] == "release-name-worker@project.iam" + + def test_tpl_rendered_annotations_webserver(self): + """Test webserver SA annotations support tpl rendering (Airflow 2.x only).""" + k8s_objects = render_chart( + values={ + "airflowVersion": "2.11.0", + "webserver": { + "serviceAccount": { + "annotations": { + "iam.gke.io/gcp-service-account": "{{ .Release.Name }}-web@project.iam", + }, + }, + }, + }, + show_only=["templates/webserver/webserver-serviceaccount.yaml"], + ) + assert len(k8s_objects) == 1 + annotations = k8s_objects[0]["metadata"]["annotations"] + assert annotations["iam.gke.io/gcp-service-account"] == "release-name-web@project.iam" + def test_annotations_on_webserver(self): """Test annotations are added on webserver for Airflow 2""" k8s_objects = render_chart( diff --git a/helm-tests/tests/helm_tests/security/test_metadata_connection_secret.py b/helm-tests/tests/helm_tests/security/test_metadata_connection_secret.py index d2479835faff1..d5727db485efd 100644 --- a/helm-tests/tests/helm_tests/security/test_metadata_connection_secret.py +++ b/helm-tests/tests/helm_tests/security/test_metadata_connection_secret.py @@ -125,6 +125,46 @@ def test_should_correctly_handle_password_with_special_characters(self): "somedb?sslmode=disable" ) + def test_tpl_rendered_user_and_db(self): + """Test that metadataConnection.user and .db support tpl rendering.""" + connection = self._get_connection( + { + "data": { + "metadataConnection": { + "user": "{{ .Release.Name }}-dbuser", + "pass": "", + "host": "localhost", + "port": 5432, + "db": "{{ .Release.Name }}-mydb", + "protocol": "postgresql", + "sslmode": "disable", + } + } + } + ) + assert "release-name-dbuser" in connection + assert "release-name-mydb" in connection + + def test_tpl_rendered_user_and_db_plain_values(self): + """Test that plain (non-template) user and db still work after tpl rendering.""" + connection = self._get_connection( + { + "data": { + "metadataConnection": { + "user": "plainuser", + "pass": "plainpass", + "host": "localhost", + "port": 5432, + "db": "plaindb", + "protocol": "postgresql", + "sslmode": "disable", + } + } + } + ) + assert "plainuser" in connection + assert "plaindb" in connection + def test_should_add_annotations_to_metadata_connection_secret(self): docs = render_chart( values={