[chart/v1-2x-test] Add workers.celery.extraVolumeMounts & workers.kubernetes.extraVolumeMounts (#65059)

chart/files/pod-template-file.kubernetes-helm-yaml

@@ -86,8 +86,8 @@ spec:
         {{- if .Values.volumeMounts }}
           {{- toYaml .Values.volumeMounts | nindent 8 }}
         {{- end }}
-        {{- if .Values.workers.extraVolumeMounts }}
-          {{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 8 }}
+        {{- if or .Values.workers.extraVolumeMounts .Values.workers.kubernetes.extraVolumeMounts }}
+          {{- tpl (toYaml (.Values.workers.kubernetes.extraVolumeMounts | default .Values.workers.extraVolumeMounts)) . | nindent 8 }}
         {{- end }}
         {{- if semverCompare ">=3.0.0" .Values.airflowVersion }}
           {{- if or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName }}
@@ -145,8 +145,8 @@ spec:
         {{- if .Values.volumeMounts }}
           {{- toYaml .Values.volumeMounts | nindent 8 }}
         {{- end }}
-        {{- if .Values.workers.extraVolumeMounts }}
-          {{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 8 }}
+        {{- if or .Values.workers.extraVolumeMounts .Values.workers.kubernetes.extraVolumeMounts }}
+          {{- tpl (toYaml (.Values.workers.kubernetes.extraVolumeMounts | default .Values.workers.extraVolumeMounts)) . | nindent 8 }}
         {{- end }}
         {{- if .Values.kerberos.enabled }}
         - name: kerberos-keytab
@@ -192,8 +192,8 @@ spec:
         {{- if .Values.volumeMounts }}
           {{- toYaml .Values.volumeMounts | nindent 8 }}
         {{- end }}
-        {{- if .Values.workers.extraVolumeMounts }}
-          {{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 8 }}
+        {{- if or .Values.workers.extraVolumeMounts .Values.workers.kubernetes.extraVolumeMounts }}
+          {{- tpl (toYaml (.Values.workers.kubernetes.extraVolumeMounts | default .Values.workers.extraVolumeMounts)) . | nindent 8 }}
         {{- end }}
         {{- if semverCompare ">=3.0.0" .Values.airflowVersion }}
           {{- if or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName }}

chart/newsfragments/65059.significant.rst

@@ -0,0 +1 @@
+``workers.extraVolumeMounts`` field is now deprecated in favor of ``workers.celery.extraVolumeMounts`` and ``workers.kubernetes.extraVolumeMounts``. Please update your configuration accordingly.

chart/templates/NOTES.txt

@@ -717,6 +717,22 @@ DEPRECATION WARNING:
 
 {{- end }}
 
+{{- if not (empty .Values.workers.extraVolumes) }}
+
+ DEPRECATION WARNING:
+    `workers.extraVolumes` has been renamed to `workers.celery.extraVolumes`/`workers.kubernetes.extraVolumes`.
+    Please change your values as support for the old name will be dropped in a future release.
+
+{{- end }}
+
+{{- if not (empty .Values.workers.extraVolumeMounts) }}
+
+ DEPRECATION WARNING:
+    `workers.extraVolumeMounts` has been renamed to `workers.celery.extraVolumeMounts`/`workers.kubernetes.extraVolumeMounts`.
+    Please change your values as support for the old name will be dropped in a future release.
+
+{{- end }}
+
 {{- if not (empty .Values.workers.runtimeClassName) }}
 
  DEPRECATION WARNING:

chart/values.schema.json

@@ -2325,7 +2325,7 @@
                     }
                 },
                 "extraVolumeMounts": {
-                    "description": "Additional volume mounts attached to the Airflow Celery workers and pods created with pod-template-file.",
+                    "description": "Additional volume mounts attached to the Airflow Celery workers and pods created with pod-template-file (deprecated, use ``workers.celery.extraVolumeMounts`` and/or ``workers.kubernetes.extraVolumeMounts`` instead).",
                     "type": "array",
                     "default": [],
                     "items": {
@@ -3404,6 +3404,14 @@
                                 "$ref": "#/definitions/io.k8s.api.core.v1.Volume"
                             }
                         },
+                        "extraVolumeMounts": {
+                            "description": "Additional volume mounts attached to the Airflow Celery workers.",
+                            "type": "array",
+                            "default": [],
+                            "items": {
+                                "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount"
+                            }
+                        },
                         "extraPorts": {
                             "description": "Expose additional ports of Airflow Celery worker container.",
                             "type": "array",
@@ -3952,6 +3960,14 @@
                                 "$ref": "#/definitions/io.k8s.api.core.v1.Volume"
                             }
                         },
+                        "extraVolumeMounts": {
+                            "description": "Additional volume mounts attached to the pods created with pod-template-file.",
+                            "type": "array",
+                            "default": [],
+                            "items": {
+                                "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount"
+                            }
+                        },
                         "nodeSelector": {
                             "description": "Select certain nodes for pods created with pod-template-file.",
                             "type": "object",

chart/values.yaml

@@ -1072,6 +1072,10 @@ workers:
 
   # Additional volume mounts attached to the Airflow Celery workers
   # and pods created with pod-template-file
+  # (deprecated, use
+  #   `workers.celery.extraVolumeMounts` and/or
+  #   `workers.kubernetes.extraVolumeMounts`
+  # instead)
   extraVolumeMounts: []
   # Mount additional volumes into workers pods. It can be templated like in the following example:
   #  extraVolumeMounts:
@@ -1479,6 +1483,14 @@ workers:
     #         defaultMode: 0640
     #         optional: true
 
+    # Additional volume mounts attached to the Airflow Celery workers
+    extraVolumeMounts: []
+    # Mount additional volumes into workers pods. It can be templated like in the following example:
+    #  extraVolumeMounts:
+    #    - name: my-templated-extra-volume
+    #      mountPath: "{{ .Values.my_custom_path }}"
+    #      readOnly: true
+
     # Expose additional ports of Airflow Celery workers. These can be used for additional metric collection.
     extraPorts: []
 
@@ -1669,6 +1681,14 @@ workers:
     #         defaultMode: 0640
     #         optional: true
 
+    # Additional volume mounts attached to the pods created with pod-template-file
+    extraVolumeMounts: []
+    # Mount additional volumes into workers pods. It can be templated like in the following example:
+    #  extraVolumeMounts:
+    #    - name: my-templated-extra-volume
+    #      mountPath: "{{ .Values.my_custom_path }}"
+    #      readOnly: true
+
     # Select certain nodes for pods created with pod-template-file
     nodeSelector: {}
 

helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py

@@ -1001,23 +1001,39 @@ def test_should_create_valid_volume(self, workers_values):
         # [2:] -> Skipping logs and config volumes
         assert jmespath.search("spec.volumes[2:].name", docs[0]) == ["test-volume-airflow"]
 
-    def test_should_create_valid_volume_mount(self):
-        docs = render_chart(
-            values={
-                "workers": {
+    @pytest.mark.parametrize(
+        "workers_values",
+        [
+            {
+                "extraVolumeMounts": [{"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"}],
+            },
+            {
+                "kubernetes": {
                     "extraVolumeMounts": [
                         {"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"}
                     ],
                 }
             },
+            {
+                "extraVolumeMounts": [{"name": "test", "mountPath": "test"}],
+                "kubernetes": {
+                    "extraVolumeMounts": [
+                        {"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"}
+                    ],
+                },
+            },
+        ],
+    )
+    def test_should_create_valid_volume_mount(self, workers_values):
+        docs = render_chart(
+            values={"workers": workers_values},
             show_only=["templates/pod-template-file.yaml"],
             chart_dir=self.temp_chart_dir,
         )
 
-        assert "test-volume-airflow" in jmespath.search(
-            "spec.containers[0].volumeMounts[*].name",
-            docs[0],
-        )
+        volume_mounts = jmespath.search("spec.containers[0].volumeMounts", docs[0])
+        assert {"name": "test-volume-airflow", "mountPath": "/opt/test"} in volume_mounts
+        assert {"name": "test", "mountPath": "test"} not in volume_mounts
 
     def test_should_add_env_for_gitsync(self):
         docs = render_chart(

helm-tests/tests/helm_tests/airflow_core/test_worker.py

@@ -363,27 +363,46 @@ def test_should_add_extra_volume(self, workers_values):
         # [:-1] -> Skipping config volume
         assert jmespath.search("spec.template.spec.volumes[:-1].name", docs[0]) == ["test-volume-airflow"]
 
-    def test_should_add_extra_volume_mount(self):
-        docs = render_chart(
-            values={
-                "executor": "CeleryExecutor",
-                "workers": {
+    @pytest.mark.parametrize(
+        "workers_values",
+        [
+            {
+                "extraVolumeMounts": [{"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"}],
+            },
+            {
+                "celery": {
+                    "extraVolumeMounts": [
+                        {"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"}
+                    ],
+                }
+            },
+            {
+                "extraVolumeMounts": [{"name": "test", "mountPath": "/opt"}],
+                "celery": {
                     "extraVolumeMounts": [
                         {"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"}
                     ],
                 },
             },
+        ],
+    )
+    def test_should_add_extra_volume_mount(self, workers_values):
+        docs = render_chart(
+            values={
+                "executor": "CeleryExecutor",
+                "workers": workers_values,
+            },
             show_only=["templates/workers/worker-deployment.yaml"],
         )
 
-        assert (
-            jmespath.search("spec.template.spec.containers[0].volumeMounts[0].name", docs[0])
-            == "test-volume-airflow"
-        )
-        assert (
-            jmespath.search("spec.template.spec.initContainers[0].volumeMounts[-1].name", docs[0])
-            == "test-volume-airflow"
-        )
+        volume_mounts = jmespath.search("spec.template.spec.containers[0].volumeMounts", docs[0])
+        init_volume_mounts = jmespath.search("spec.template.spec.initContainers[0].volumeMounts", docs[0])
+
+        assert {"name": "test-volume-airflow", "mountPath": "/opt/test"} in init_volume_mounts
+        assert {"name": "test", "mountPath": "/opt"} not in init_volume_mounts
+
+        assert {"name": "test-volume-airflow", "mountPath": "/opt/test"} in volume_mounts
+        assert {"name": "test", "mountPath": "/opt"} not in volume_mounts
 
     def test_should_add_global_volume_and_global_volume_mount(self):
         docs = render_chart(

helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py

@@ -2614,6 +2614,20 @@ def test_overwrite_extra_volumes(self, workers_values):
                     ],
                 },
             },
+            {
+                "celery": {
+                    "enableDefault": False,
+                    "extraVolumeMounts": [{"name": "test", "mountPath": "/opt"}],
+                    "sets": [
+                        {
+                            "name": "set1",
+                            "extraVolumeMounts": [
+                                {"name": "test-volume-mount-{{ .Chart.Name }}", "mountPath": "/opt/test"}
+                            ],
+                        }
+                    ],
+                },
+            },
         ],
     )
     def test_overwrite_extra_volume_mounts(self, workers_values):
@@ -2624,10 +2638,10 @@ def test_overwrite_extra_volume_mounts(self, workers_values):
             show_only=["templates/workers/worker-deployment.yaml"],
         )
 
-        assert jmespath.search("spec.template.spec.containers[0].volumeMounts[0]", docs[0]) == {
-            "name": "test-volume-mount-airflow",
-            "mountPath": "/opt/test",
-        }
+        volume_mounts = jmespath.search("spec.template.spec.containers[0].volumeMounts", docs[0])
+
+        assert {"name": "test-volume-mount-airflow", "mountPath": "/opt/test"} in volume_mounts
+        assert {"name": "test", "mountPath": "/opt"} not in volume_mounts
 
     @pytest.mark.parametrize(
         "workers_values",