From 38f4447d69069a65cfb36e08f31baee9b746dca9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Mirowski?= <17602603+Miretpl@users.noreply.github.com> Date: Sun, 12 Apr 2026 12:04:48 +0200 Subject: [PATCH] [chart/v1-2x-test] Add workers.celery.extraVolumeMounts & workers.kubernetes.extraVolumeMounts (#65059) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add missing extraVolumes deprecation note * Add workers.celery.extraVolumeMounts & workers.kubernetes.extraVolumeMounts * Add newsfragment (cherry picked from commit 88eccf4e9398cebc641c3130a3b531a85e787e71) Co-authored-by: Przemysław Mirowski <17602603+Miretpl@users.noreply.github.com> --- .../pod-template-file.kubernetes-helm-yaml | 12 ++--- chart/newsfragments/65059.significant.rst | 1 + chart/templates/NOTES.txt | 16 +++++++ chart/values.schema.json | 18 +++++++- chart/values.yaml | 20 +++++++++ .../airflow_aux/test_pod_template_file.py | 32 +++++++++---- .../helm_tests/airflow_core/test_worker.py | 45 +++++++++++++------ .../airflow_core/test_worker_sets.py | 22 +++++++-- 8 files changed, 134 insertions(+), 32 deletions(-) create mode 100644 chart/newsfragments/65059.significant.rst diff --git a/chart/files/pod-template-file.kubernetes-helm-yaml b/chart/files/pod-template-file.kubernetes-helm-yaml index e7143e61a5b30..305e7b5e7235e 100644 --- a/chart/files/pod-template-file.kubernetes-helm-yaml +++ b/chart/files/pod-template-file.kubernetes-helm-yaml @@ -86,8 +86,8 @@ spec: {{- if .Values.volumeMounts }} {{- toYaml .Values.volumeMounts | nindent 8 }} {{- end }} - {{- if .Values.workers.extraVolumeMounts }} - {{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 8 }} + {{- if or .Values.workers.extraVolumeMounts .Values.workers.kubernetes.extraVolumeMounts }} + {{- tpl (toYaml (.Values.workers.kubernetes.extraVolumeMounts | default .Values.workers.extraVolumeMounts)) . | nindent 8 }} {{- end }} {{- if semverCompare ">=3.0.0" .Values.airflowVersion }} {{- if or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName }} @@ -145,8 +145,8 @@ spec: {{- if .Values.volumeMounts }} {{- toYaml .Values.volumeMounts | nindent 8 }} {{- end }} - {{- if .Values.workers.extraVolumeMounts }} - {{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 8 }} + {{- if or .Values.workers.extraVolumeMounts .Values.workers.kubernetes.extraVolumeMounts }} + {{- tpl (toYaml (.Values.workers.kubernetes.extraVolumeMounts | default .Values.workers.extraVolumeMounts)) . | nindent 8 }} {{- end }} {{- if .Values.kerberos.enabled }} - name: kerberos-keytab @@ -192,8 +192,8 @@ spec: {{- if .Values.volumeMounts }} {{- toYaml .Values.volumeMounts | nindent 8 }} {{- end }} - {{- if .Values.workers.extraVolumeMounts }} - {{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 8 }} + {{- if or .Values.workers.extraVolumeMounts .Values.workers.kubernetes.extraVolumeMounts }} + {{- tpl (toYaml (.Values.workers.kubernetes.extraVolumeMounts | default .Values.workers.extraVolumeMounts)) . | nindent 8 }} {{- end }} {{- if semverCompare ">=3.0.0" .Values.airflowVersion }} {{- if or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName }} diff --git a/chart/newsfragments/65059.significant.rst b/chart/newsfragments/65059.significant.rst new file mode 100644 index 0000000000000..b45e0d13108de --- /dev/null +++ b/chart/newsfragments/65059.significant.rst @@ -0,0 +1 @@ +``workers.extraVolumeMounts`` field is now deprecated in favor of ``workers.celery.extraVolumeMounts`` and ``workers.kubernetes.extraVolumeMounts``. Please update your configuration accordingly. diff --git a/chart/templates/NOTES.txt b/chart/templates/NOTES.txt index 8142ffc1dcc30..baee62cb88138 100644 --- a/chart/templates/NOTES.txt +++ b/chart/templates/NOTES.txt @@ -717,6 +717,22 @@ DEPRECATION WARNING: {{- end }} +{{- if not (empty .Values.workers.extraVolumes) }} + + DEPRECATION WARNING: + `workers.extraVolumes` has been renamed to `workers.celery.extraVolumes`/`workers.kubernetes.extraVolumes`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.extraVolumeMounts) }} + + DEPRECATION WARNING: + `workers.extraVolumeMounts` has been renamed to `workers.celery.extraVolumeMounts`/`workers.kubernetes.extraVolumeMounts`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + {{- if not (empty .Values.workers.runtimeClassName) }} DEPRECATION WARNING: diff --git a/chart/values.schema.json b/chart/values.schema.json index 28e7ee16e897c..8d152146ca1c7 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -2325,7 +2325,7 @@ } }, "extraVolumeMounts": { - "description": "Additional volume mounts attached to the Airflow Celery workers and pods created with pod-template-file.", + "description": "Additional volume mounts attached to the Airflow Celery workers and pods created with pod-template-file (deprecated, use ``workers.celery.extraVolumeMounts`` and/or ``workers.kubernetes.extraVolumeMounts`` instead).", "type": "array", "default": [], "items": { @@ -3404,6 +3404,14 @@ "$ref": "#/definitions/io.k8s.api.core.v1.Volume" } }, + "extraVolumeMounts": { + "description": "Additional volume mounts attached to the Airflow Celery workers.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount" + } + }, "extraPorts": { "description": "Expose additional ports of Airflow Celery worker container.", "type": "array", @@ -3952,6 +3960,14 @@ "$ref": "#/definitions/io.k8s.api.core.v1.Volume" } }, + "extraVolumeMounts": { + "description": "Additional volume mounts attached to the pods created with pod-template-file.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount" + } + }, "nodeSelector": { "description": "Select certain nodes for pods created with pod-template-file.", "type": "object", diff --git a/chart/values.yaml b/chart/values.yaml index 22d3c44a6fd44..8a3290df5aaec 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1072,6 +1072,10 @@ workers: # Additional volume mounts attached to the Airflow Celery workers # and pods created with pod-template-file + # (deprecated, use + # `workers.celery.extraVolumeMounts` and/or + # `workers.kubernetes.extraVolumeMounts` + # instead) extraVolumeMounts: [] # Mount additional volumes into workers pods. It can be templated like in the following example: # extraVolumeMounts: @@ -1479,6 +1483,14 @@ workers: # defaultMode: 0640 # optional: true + # Additional volume mounts attached to the Airflow Celery workers + extraVolumeMounts: [] + # Mount additional volumes into workers pods. It can be templated like in the following example: + # extraVolumeMounts: + # - name: my-templated-extra-volume + # mountPath: "{{ .Values.my_custom_path }}" + # readOnly: true + # Expose additional ports of Airflow Celery workers. These can be used for additional metric collection. extraPorts: [] @@ -1669,6 +1681,14 @@ workers: # defaultMode: 0640 # optional: true + # Additional volume mounts attached to the pods created with pod-template-file + extraVolumeMounts: [] + # Mount additional volumes into workers pods. It can be templated like in the following example: + # extraVolumeMounts: + # - name: my-templated-extra-volume + # mountPath: "{{ .Values.my_custom_path }}" + # readOnly: true + # Select certain nodes for pods created with pod-template-file nodeSelector: {} diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py b/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py index 281fb76687288..ed57a426ce514 100644 --- a/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py +++ b/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py @@ -1001,23 +1001,39 @@ def test_should_create_valid_volume(self, workers_values): # [2:] -> Skipping logs and config volumes assert jmespath.search("spec.volumes[2:].name", docs[0]) == ["test-volume-airflow"] - def test_should_create_valid_volume_mount(self): - docs = render_chart( - values={ - "workers": { + @pytest.mark.parametrize( + "workers_values", + [ + { + "extraVolumeMounts": [{"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"}], + }, + { + "kubernetes": { "extraVolumeMounts": [ {"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"} ], } }, + { + "extraVolumeMounts": [{"name": "test", "mountPath": "test"}], + "kubernetes": { + "extraVolumeMounts": [ + {"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"} + ], + }, + }, + ], + ) + def test_should_create_valid_volume_mount(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) - assert "test-volume-airflow" in jmespath.search( - "spec.containers[0].volumeMounts[*].name", - docs[0], - ) + volume_mounts = jmespath.search("spec.containers[0].volumeMounts", docs[0]) + assert {"name": "test-volume-airflow", "mountPath": "/opt/test"} in volume_mounts + assert {"name": "test", "mountPath": "test"} not in volume_mounts def test_should_add_env_for_gitsync(self): docs = render_chart( diff --git a/helm-tests/tests/helm_tests/airflow_core/test_worker.py b/helm-tests/tests/helm_tests/airflow_core/test_worker.py index 31fcfef74d3cc..38bba315458d2 100644 --- a/helm-tests/tests/helm_tests/airflow_core/test_worker.py +++ b/helm-tests/tests/helm_tests/airflow_core/test_worker.py @@ -363,27 +363,46 @@ def test_should_add_extra_volume(self, workers_values): # [:-1] -> Skipping config volume assert jmespath.search("spec.template.spec.volumes[:-1].name", docs[0]) == ["test-volume-airflow"] - def test_should_add_extra_volume_mount(self): - docs = render_chart( - values={ - "executor": "CeleryExecutor", - "workers": { + @pytest.mark.parametrize( + "workers_values", + [ + { + "extraVolumeMounts": [{"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"}], + }, + { + "celery": { + "extraVolumeMounts": [ + {"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"} + ], + } + }, + { + "extraVolumeMounts": [{"name": "test", "mountPath": "/opt"}], + "celery": { "extraVolumeMounts": [ {"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"} ], }, }, + ], + ) + def test_should_add_extra_volume_mount(self, workers_values): + docs = render_chart( + values={ + "executor": "CeleryExecutor", + "workers": workers_values, + }, show_only=["templates/workers/worker-deployment.yaml"], ) - assert ( - jmespath.search("spec.template.spec.containers[0].volumeMounts[0].name", docs[0]) - == "test-volume-airflow" - ) - assert ( - jmespath.search("spec.template.spec.initContainers[0].volumeMounts[-1].name", docs[0]) - == "test-volume-airflow" - ) + volume_mounts = jmespath.search("spec.template.spec.containers[0].volumeMounts", docs[0]) + init_volume_mounts = jmespath.search("spec.template.spec.initContainers[0].volumeMounts", docs[0]) + + assert {"name": "test-volume-airflow", "mountPath": "/opt/test"} in init_volume_mounts + assert {"name": "test", "mountPath": "/opt"} not in init_volume_mounts + + assert {"name": "test-volume-airflow", "mountPath": "/opt/test"} in volume_mounts + assert {"name": "test", "mountPath": "/opt"} not in volume_mounts def test_should_add_global_volume_and_global_volume_mount(self): docs = render_chart( diff --git a/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py b/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py index c8a552332a15a..ed2d677fce946 100644 --- a/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py +++ b/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py @@ -2614,6 +2614,20 @@ def test_overwrite_extra_volumes(self, workers_values): ], }, }, + { + "celery": { + "enableDefault": False, + "extraVolumeMounts": [{"name": "test", "mountPath": "/opt"}], + "sets": [ + { + "name": "set1", + "extraVolumeMounts": [ + {"name": "test-volume-mount-{{ .Chart.Name }}", "mountPath": "/opt/test"} + ], + } + ], + }, + }, ], ) def test_overwrite_extra_volume_mounts(self, workers_values): @@ -2624,10 +2638,10 @@ def test_overwrite_extra_volume_mounts(self, workers_values): show_only=["templates/workers/worker-deployment.yaml"], ) - assert jmespath.search("spec.template.spec.containers[0].volumeMounts[0]", docs[0]) == { - "name": "test-volume-mount-airflow", - "mountPath": "/opt/test", - } + volume_mounts = jmespath.search("spec.template.spec.containers[0].volumeMounts", docs[0]) + + assert {"name": "test-volume-mount-airflow", "mountPath": "/opt/test"} in volume_mounts + assert {"name": "test", "mountPath": "/opt"} not in volume_mounts @pytest.mark.parametrize( "workers_values",