Add support for custom emptyDir config

chart/files/pod-template-file.kubernetes-helm-yaml

@@ -117,14 +117,14 @@ spec:
       claimName: {{ template "airflow_dags_volume_claim" . }}
   {{- else if .Values.dags.gitSync.enabled }}
   - name: dags
-    emptyDir: {}
+    emptyDir: {{- toYaml (default (dict) .Values.dags.gitSync.emptyDirConfig) | nindent 6 }}
   {{- end }}
   {{- if .Values.logs.persistence.enabled }}
   - name: logs
     persistentVolumeClaim:
       claimName: {{ template "airflow_logs_volume_claim" . }}
   {{- else }}
-  - emptyDir: {}
+  - emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) | nindent 6 }}
     name: logs
   {{- end }}
   {{- if and  .Values.dags.gitSync.enabled  .Values.dags.gitSync.sshKeySecret }}

chart/templates/dag-processor/dag-processor-deployment.yaml

@@ -238,7 +238,7 @@ spec:
             claimName: {{ template "airflow_dags_volume_claim" . }}
         {{- else if .Values.dags.gitSync.enabled }}
         - name: dags
-          emptyDir: {}
+          emptyDir: {{- toYaml (default (dict) .Values.dags.gitSync.emptyDirConfig) | nindent 12 }}
         {{- end }}
         {{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.sshKeySecret }}
           {{- include "git_sync_ssh_key_volume" . | indent 8 }}
@@ -255,7 +255,7 @@ spec:
             claimName: {{ template "airflow_logs_volume_claim" . }}
         {{- else }}
         - name: logs
-          emptyDir: {}
+          emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) | nindent 12 }}
 {{- end }}
 {{- end }}
 {{- end }}

chart/templates/redis/redis-statefulset.yaml

@@ -106,7 +106,7 @@ spec:
   {{- if not .Values.redis.persistence.enabled }}
       volumes:
         - name: redis-db
-          emptyDir: {}
+          emptyDir: {{- toYaml (default (dict) .Values.redis.emptyDirConfig) | nindent 12 }}
   {{- else }}
   volumeClaimTemplates:
     - metadata:

chart/templates/scheduler/scheduler-deployment.yaml

@@ -298,7 +298,7 @@ spec:
             claimName: {{ template "airflow_dags_volume_claim" . }}
         {{- else if .Values.dags.gitSync.enabled }}
         - name: dags
-          emptyDir: {}
+          emptyDir: {{- toYaml (default (dict) .Values.dags.gitSync.emptyDirConfig) | nindent 12 }}
         {{- if .Values.dags.gitSync.sshKeySecret }}
           {{- include "git_sync_ssh_key_volume" . | indent 8 }}
         {{- end }}
@@ -316,7 +316,7 @@ spec:
             claimName: {{ template "airflow_logs_volume_claim" . }}
   {{- else if not $stateful }}
         - name: logs
-          emptyDir: {}
+          emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) | nindent 12 }}
   {{- else }}
   volumeClaimTemplates:
     - metadata:

chart/templates/triggerer/triggerer-deployment.yaml

@@ -270,7 +270,7 @@ spec:
             claimName: {{ template "airflow_dags_volume_claim" . }}
         {{- else if .Values.dags.gitSync.enabled }}
         - name: dags
-          emptyDir: {}
+          emptyDir: {{- toYaml (default (dict) .Values.dags.gitSync.emptyDirConfig) | nindent 12 }}
         {{- if .Values.dags.gitSync.sshKeySecret }}
           {{- include "git_sync_ssh_key_volume" . | nindent 8 }}
         {{- end }}
@@ -287,7 +287,7 @@ spec:
             claimName: {{ template "airflow_logs_volume_claim" . }}
   {{- else if not $persistence }}
         - name: logs
-          emptyDir: {}
+          emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) | nindent 12 }}
   {{- else }}
   volumeClaimTemplates:
     - metadata:

chart/templates/webserver/webserver-deployment.yaml

@@ -274,7 +274,7 @@ spec:
             claimName: {{ template "airflow_dags_volume_claim" . }}
         {{- else if .Values.dags.gitSync.enabled }}
         - name: dags
-          emptyDir: {}
+          emptyDir: {{- toYaml (default (dict) .Values.dags.gitSync.emptyDirConfig) | nindent 12 }}
         {{- if  .Values.dags.gitSync.sshKeySecret }}
           {{- include "git_sync_ssh_key_volume" . | indent 8 }}
         {{- end }}

chart/templates/workers/worker-deployment.yaml

@@ -373,7 +373,7 @@ spec:
             claimName: {{ template "airflow_dags_volume_claim" . }}
         {{- else if .Values.dags.gitSync.enabled }}
         - name: dags
-          emptyDir: {}
+          emptyDir: {{- toYaml (default (dict) .Values.dags.gitSync.emptyDirConfig) | nindent 12 -}}
         {{- if  .Values.dags.gitSync.sshKeySecret }}
           {{- include "git_sync_ssh_key_volume" . | indent 8 }}
         {{- end }}
@@ -384,7 +384,7 @@ spec:
             claimName: {{ template "airflow_logs_volume_claim" . }}
   {{- else if not $persistence }}
         - name: logs
-          emptyDir: {}
+          emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) | nindent 12 -}}
   {{- else }}
   volumeClaimTemplates:
     - metadata:

chart/values.schema.json

@@ -6164,6 +6164,12 @@
                         }
                     }
                 },
+                "emptyDirConfig": {
+                    "description": "Configuration for redis empty dir volume.",
+                    "type": "object",
+                    "$ref": "#/definitions/io.k8s.api.core.v1.EmptyDirVolumeSource",
+                    "default": null
+                },
                 "resources": {
                     "description": "Resources for the Redis pods",
                     "type": "object",
@@ -7046,6 +7052,12 @@
                                 }
                             ]
                         },
+                        "emptyDirConfig": {
+                            "description": "Configuration for dags empty dir volume.",
+                            "type": "object",
+                            "$ref": "#/definitions/io.k8s.api.core.v1.EmptyDirVolumeSource",
+                            "default": null
+                        },
                         "containerLifecycleHooks": {
                             "description": "Container Lifecycle Hooks definition for the git sync sidecar. If not set, the values from global `containerLifecycleHooks` will be used.",
                             "type": "object",
@@ -7235,6 +7247,12 @@
                             "default": null
                         }
                     }
+                },
+                "emptyDirConfig": {
+                    "description": "Configuration for logs empty dir volume.",
+                    "type": "object",
+                    "$ref": "#/definitions/io.k8s.api.core.v1.EmptyDirVolumeSource",
+                    "default": null
                 }
             }
         }

chart/values.yaml

@@ -2066,6 +2066,11 @@ redis:
     # Annotations to add to redis volumes
     annotations: {}
 
+  # Configuration for empty dir volume (if redis.persistence.enabled == false)
+  # emptyDirConfig:
+  #   sizeLimit: 1Gi
+  #   medium: Memory
+
   resources: {}
   #  limits:
   #   cpu: 100m
@@ -2440,6 +2445,11 @@ dags:
     # - name: ""
     #   value: ""
 
+    # Configuration for empty dir volume
+    # emptyDirConfig:
+    #   sizeLimit: 1Gi
+    #   medium: Memory
+
     resources: {}
     #  limits:
     #   cpu: 100m
@@ -2449,6 +2459,11 @@ dags:
     #   memory: 128Mi
 
 logs:
+  # Configuration for empty dir volume (if logs.persistence.enabled == false)
+  # emptyDirConfig:
+  #   sizeLimit: 1Gi
+  #   medium: Memory
+
   persistence:
     # Enable persistent volume for storing logs
     enabled: false

helm_tests/airflow_aux/test_pod_template_file.py

@@ -267,29 +267,42 @@ def test_should_set_the_dags_volume_claim_correctly_when_using_an_existing_claim
             "spec.volumes", docs[0]
         )
 
-    def test_should_use_empty_dir_for_gitsync_without_persistence(self):
+    @pytest.mark.parametrize(
+        "dags_gitsync_values, expected",
+        [
+            ({"enabled": True}, {"emptyDir": {}}),
+            ({"enabled": True, "emptyDirConfig": {"sizeLimit": "10Gi"}}, {"emptyDir": {"sizeLimit": "10Gi"}}),
+        ],
+    )
+    def test_should_use_empty_dir_for_gitsync_without_persistence(self, dags_gitsync_values, expected):
         docs = render_chart(
-            values={"dags": {"gitSync": {"enabled": True}}},
+            values={"dags": {"gitSync": dags_gitsync_values}},
             show_only=["templates/pod-template-file.yaml"],
             chart_dir=self.temp_chart_dir,
         )
-
-        assert {"name": "dags", "emptyDir": {}} in jmespath.search("spec.volumes", docs[0])
+        assert {"name": "dags", **expected} in jmespath.search("spec.volumes", docs[0])
 
     @pytest.mark.parametrize(
-        "log_persistence_values, expected",
+        "log_values, expected",
         [
-            ({"enabled": False}, {"emptyDir": {}}),
-            ({"enabled": True}, {"persistentVolumeClaim": {"claimName": "release-name-logs"}}),
+            ({"persistence": {"enabled": False}}, {"emptyDir": {}}),
+            (
+                {"persistence": {"enabled": False}, "emptyDirConfig": {"sizeLimit": "10Gi"}},
+                {"emptyDir": {"sizeLimit": "10Gi"}},
+            ),
+            (
+                {"persistence": {"enabled": True}},
+                {"persistentVolumeClaim": {"claimName": "release-name-logs"}},
+            ),
             (
-                {"enabled": True, "existingClaim": "test-claim"},
+                {"persistence": {"enabled": True, "existingClaim": "test-claim"}},
                 {"persistentVolumeClaim": {"claimName": "test-claim"}},
             ),
         ],
     )
-    def test_logs_persistence_changes_volume(self, log_persistence_values, expected):
+    def test_logs_persistence_changes_volume(self, log_values, expected):
         docs = render_chart(
-            values={"logs": {"persistence": log_persistence_values}},
+            values={"logs": log_values},
             show_only=["templates/pod-template-file.yaml"],
             chart_dir=self.temp_chart_dir,
         )

helm_tests/airflow_core/test_dag_processor.py

@@ -368,20 +368,27 @@ def test_livenessprobe_command_depends_on_airflow_version(self, airflow_version,
         )
 
     @pytest.mark.parametrize(
-        "log_persistence_values, expected_volume",
+        "log_values, expected_volume",
         [
-            ({"enabled": False}, {"emptyDir": {}}),
-            ({"enabled": True}, {"persistentVolumeClaim": {"claimName": "release-name-logs"}}),
+            ({"persistence": {"enabled": False}}, {"emptyDir": {}}),
             (
-                {"enabled": True, "existingClaim": "test-claim"},
+                {"persistence": {"enabled": False}, "emptyDirConfig": {"sizeLimit": "10Gi"}},
+                {"emptyDir": {"sizeLimit": "10Gi"}},
+            ),
+            (
+                {"persistence": {"enabled": True}},
+                {"persistentVolumeClaim": {"claimName": "release-name-logs"}},
+            ),
+            (
+                {"persistence": {"enabled": True, "existingClaim": "test-claim"}},
                 {"persistentVolumeClaim": {"claimName": "test-claim"}},
             ),
         ],
     )
-    def test_logs_persistence_changes_volume(self, log_persistence_values, expected_volume):
+    def test_logs_persistence_changes_volume(self, log_values, expected_volume):
         docs = render_chart(
             values={
-                "logs": {"persistence": log_persistence_values},
+                "logs": log_values,
                 "dagProcessor": {"enabled": True},
             },
             show_only=["templates/dag-processor/dag-processor-deployment.yaml"],

helm_tests/airflow_core/test_scheduler.py

@@ -426,19 +426,26 @@ def test_startupprobe_command_depends_on_airflow_version(self, airflow_version,
         )
 
     @pytest.mark.parametrize(
-        "log_persistence_values, expected_volume",
+        "log_values, expected_volume",
         [
-            ({"enabled": False}, {"emptyDir": {}}),
-            ({"enabled": True}, {"persistentVolumeClaim": {"claimName": "release-name-logs"}}),
+            ({"persistence": {"enabled": False}}, {"emptyDir": {}}),
             (
-                {"enabled": True, "existingClaim": "test-claim"},
+                {"persistence": {"enabled": False}, "emptyDirConfig": {"sizeLimit": "10Gi"}},
+                {"emptyDir": {"sizeLimit": "10Gi"}},
+            ),
+            (
+                {"persistence": {"enabled": True}},
+                {"persistentVolumeClaim": {"claimName": "release-name-logs"}},
+            ),
+            (
+                {"persistence": {"enabled": True, "existingClaim": "test-claim"}},
                 {"persistentVolumeClaim": {"claimName": "test-claim"}},
             ),
         ],
     )
-    def test_logs_persistence_changes_volume(self, log_persistence_values, expected_volume):
+    def test_logs_persistence_changes_volume(self, log_values, expected_volume):
         docs = render_chart(
-            values={"logs": {"persistence": log_persistence_values}},
+            values={"logs": log_values},
             show_only=["templates/scheduler/scheduler-deployment.yaml"],
         )
 

helm_tests/airflow_core/test_triggerer.py

@@ -389,21 +389,28 @@ def test_livenessprobe_command_depends_on_airflow_version(self, airflow_version,
         )
 
     @pytest.mark.parametrize(
-        "log_persistence_values, expected_volume",
+        "log_values, expected_volume",
         [
-            ({"enabled": False}, {"emptyDir": {}}),
-            ({"enabled": True}, {"persistentVolumeClaim": {"claimName": "release-name-logs"}}),
+            ({"persistence": {"enabled": False}}, {"emptyDir": {}}),
             (
-                {"enabled": True, "existingClaim": "test-claim"},
+                {"persistence": {"enabled": False}, "emptyDirConfig": {"sizeLimit": "10Gi"}},
+                {"emptyDir": {"sizeLimit": "10Gi"}},
+            ),
+            (
+                {"persistence": {"enabled": True}},
+                {"persistentVolumeClaim": {"claimName": "release-name-logs"}},
+            ),
+            (
+                {"persistence": {"enabled": True, "existingClaim": "test-claim"}},
                 {"persistentVolumeClaim": {"claimName": "test-claim"}},
             ),
         ],
     )
-    def test_logs_persistence_changes_volume(self, log_persistence_values, expected_volume):
+    def test_logs_persistence_changes_volume(self, log_values, expected_volume):
         docs = render_chart(
             values={
                 "triggerer": {"persistence": {"enabled": False}},
-                "logs": {"persistence": log_persistence_values},
+                "logs": log_values,
             },
             show_only=["templates/triggerer/triggerer-deployment.yaml"],
         )

helm_tests/airflow_core/test_worker.py

@@ -456,22 +456,29 @@ def test_disable_livenessprobe(self):
         assert livenessprobe is None
 
     @pytest.mark.parametrize(
-        "log_persistence_values, expected_volume",
+        "log_values, expected_volume",
         [
-            ({"enabled": False}, {"emptyDir": {}}),
-            ({"enabled": True}, {"persistentVolumeClaim": {"claimName": "release-name-logs"}}),
+            ({"persistence": {"enabled": False}}, {"emptyDir": {}}),
             (
-                {"enabled": True, "existingClaim": "test-claim"},
+                {"persistence": {"enabled": False}, "emptyDirConfig": {"sizeLimit": "10Gi"}},
+                {"emptyDir": {"sizeLimit": "10Gi"}},
+            ),
+            (
+                {"persistence": {"enabled": True}},
+                {"persistentVolumeClaim": {"claimName": "release-name-logs"}},
+            ),
+            (
+                {"persistence": {"enabled": True, "existingClaim": "test-claim"}},
                 {"persistentVolumeClaim": {"claimName": "test-claim"}},
             ),
         ],
     )
-    def test_logs_persistence_changes_volume(self, log_persistence_values, expected_volume):
+    def test_logs_persistence_changes_volume(self, log_values, expected_volume):
         docs = render_chart(
             values={
                 "executor": "CeleryExecutor",
                 "workers": {"persistence": {"enabled": False}},
-                "logs": {"persistence": log_persistence_values},
+                "logs": log_values,
             },
             show_only=["templates/workers/worker-deployment.yaml"],
         )

helm_tests/other/test_redis.py

@@ -340,6 +340,23 @@ def test_persistence_volume_annotations(self):
         )
         assert {"foo": "bar"} == jmespath.search("spec.volumeClaimTemplates[0].metadata.annotations", docs[0])
 
+    @pytest.mark.parametrize(
+        "redis_values, expected",
+        [
+            ({"persistence": {"enabled": False}}, {"emptyDir": {}}),
+            (
+                {"persistence": {"enabled": False}, "emptyDirConfig": {"sizeLimit": "10Gi"}},
+                {"emptyDir": {"sizeLimit": "10Gi"}},
+            ),
+        ],
+    )
+    def test_should_use_empty_dir_on_persistence_disabled(self, redis_values, expected):
+        docs = render_chart(
+            values={"redis": redis_values},
+            show_only=["templates/redis/redis-statefulset.yaml"],
+        )
+        assert {"name": "redis-db", **expected} in jmespath.search("spec.template.spec.volumes", docs[0])
+
 
 class TestRedisServiceAccount:
     """Tests redis service account."""