From 5818c0c6ba3c97ad9ad06ca8420490748c533d0e Mon Sep 17 00:00:00 2001 From: Kim Min Woo Date: Sat, 24 Feb 2024 02:34:44 +0900 Subject: [PATCH] Allow templating git-sync repo url --- .../pod-template-file.kubernetes-helm-yaml | 2 +- chart/templates/_helpers.yaml | 4 +-- .../dag-processor-deployment.yaml | 2 +- .../scheduler/scheduler-deployment.yaml | 2 +- .../triggerer/triggerer-deployment.yaml | 2 +- .../webserver/webserver-deployment.yaml | 2 +- .../templates/workers/worker-deployment.yaml | 2 +- chart/values.yaml | 2 +- helm_tests/other/test_git_sync_scheduler.py | 27 +++++++++++++++++++ helm_tests/other/test_git_sync_triggerer.py | 27 +++++++++++++++++++ helm_tests/other/test_git_sync_webserver.py | 27 +++++++++++++++++++ helm_tests/other/test_git_sync_worker.py | 27 +++++++++++++++++++ 12 files changed, 117 insertions(+), 9 deletions(-) diff --git a/chart/files/pod-template-file.kubernetes-helm-yaml b/chart/files/pod-template-file.kubernetes-helm-yaml index 0c35d4ae2ff7d..20cdc302f4316 100644 --- a/chart/files/pod-template-file.kubernetes-helm-yaml +++ b/chart/files/pod-template-file.kubernetes-helm-yaml @@ -48,7 +48,7 @@ spec: {{- if or (and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled)) .Values.workers.extraInitContainers }} initContainers: {{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) }} - {{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 4 }} + {{- include "git_sync_container" (merge . (dict "is_init" "true")) | nindent 4 }} {{- end }} {{- if .Values.workers.extraInitContainers }} {{- toYaml .Values.workers.extraInitContainers | nindent 4 }} diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index 59a28fd9b7df4..d8b8c458c691d 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -252,9 +252,9 @@ If release name contains chart name it will be used as a full name. - name: GIT_SYNC_BRANCH value: {{ .Values.dags.gitSync.branch | quote }} - name: GIT_SYNC_REPO - value: {{ .Values.dags.gitSync.repo | quote }} + value: {{ tpl .Values.dags.gitSync.repo . | quote }} - name: GITSYNC_REPO - value: {{ .Values.dags.gitSync.repo | quote }} + value: {{ tpl .Values.dags.gitSync.repo . | quote }} - name: GIT_SYNC_DEPTH value: {{ .Values.dags.gitSync.depth | quote }} - name: GITSYNC_DEPTH diff --git a/chart/templates/dag-processor/dag-processor-deployment.yaml b/chart/templates/dag-processor/dag-processor-deployment.yaml index 970c94a90322e..04bbbe47da3b3 100644 --- a/chart/templates/dag-processor/dag-processor-deployment.yaml +++ b/chart/templates/dag-processor/dag-processor-deployment.yaml @@ -141,7 +141,7 @@ spec: {{- end }} {{- end }} {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} - {{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }} + {{- include "git_sync_container" (merge . (dict "is_init" "true")) | nindent 8 }} {{- end }} {{- if .Values.dagProcessor.extraInitContainers }} {{- toYaml .Values.dagProcessor.extraInitContainers | nindent 8 }} diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml index 531ea83d1e533..9e4d76c09d449 100644 --- a/chart/templates/scheduler/scheduler-deployment.yaml +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -166,7 +166,7 @@ spec: {{- end }} {{- end }} {{- if and $localOrDagProcessorDisabled .Values.dags.gitSync.enabled }} - {{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }} + {{- include "git_sync_container" (merge . (dict "is_init" "true")) | nindent 8 }} {{- end }} {{- if .Values.scheduler.extraInitContainers }} {{- toYaml .Values.scheduler.extraInitContainers | nindent 8 }} diff --git a/chart/templates/triggerer/triggerer-deployment.yaml b/chart/templates/triggerer/triggerer-deployment.yaml index 618274bf69850..88b421f613efa 100644 --- a/chart/templates/triggerer/triggerer-deployment.yaml +++ b/chart/templates/triggerer/triggerer-deployment.yaml @@ -157,7 +157,7 @@ spec: {{- end }} {{- end }} {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} - {{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }} + {{- include "git_sync_container" (merge . (dict "is_init" "true")) | nindent 8 }} {{- end }} {{- if .Values.triggerer.extraInitContainers }} {{- toYaml .Values.triggerer.extraInitContainers | nindent 8 }} diff --git a/chart/templates/webserver/webserver-deployment.yaml b/chart/templates/webserver/webserver-deployment.yaml index bb6c9d4b53823..2ac5107e34a02 100644 --- a/chart/templates/webserver/webserver-deployment.yaml +++ b/chart/templates/webserver/webserver-deployment.yaml @@ -160,7 +160,7 @@ spec: {{- end }} {{- end }} {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) (semverCompare "<2.0.0" .Values.airflowVersion) }} - {{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }} + {{- include "git_sync_container" (merge . (dict "is_init" "true")) | nindent 8 }} {{- end }} {{- if .Values.webserver.extraInitContainers }} {{- toYaml .Values.webserver.extraInitContainers | nindent 8 }} diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index d2b11b230e69f..cbf1944ab814a 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -217,7 +217,7 @@ spec: {{- end }} {{- end }} {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} - {{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }} + {{- include "git_sync_container" (merge . (dict "is_init" "true")) | nindent 8 }} {{- end }} {{- if .Values.workers.extraInitContainers }} {{- toYaml .Values.workers.extraInitContainers | nindent 8 }} diff --git a/chart/values.yaml b/chart/values.yaml index c16f5088ce679..3bcba4f2271a7 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -2451,7 +2451,7 @@ dags: gitSync: enabled: false - # git repo clone url + # git repo clone url (can be templated) # ssh example: git@github.com:apache/airflow.git # https example: https://github.com/apache/airflow.git repo: https://github.com/apache/airflow.git diff --git a/helm_tests/other/test_git_sync_scheduler.py b/helm_tests/other/test_git_sync_scheduler.py index 7bf7b83ae5572..95b0cd6aa82a8 100644 --- a/helm_tests/other/test_git_sync_scheduler.py +++ b/helm_tests/other/test_git_sync_scheduler.py @@ -364,3 +364,30 @@ def test_resources_are_configurable(self): "spec.template.spec.containers[1].resources.requests.memory", docs[0] ) assert "300m" == jmespath.search("spec.template.spec.containers[1].resources.requests.cpu", docs[0]) + + def test_can_repo_url_be_templated(self): + docs = render_chart( + values={ + "testValues": { + "dict": { + "key": "aa", + }, + }, + "dags": { + "gitSync": { + "enabled": True, + "repo": "https://github.com/{{ .Values.testValues.dict.key }}/{{ .Release.Namespace }}.git", + } + }, + }, + show_only=["templates/scheduler/scheduler-deployment.yaml"], + namespace="bb", + ) + + assert {"name": "GIT_SYNC_REPO", "value": "https://github.com/aa/bb.git"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) + + assert {"name": "GITSYNC_REPO", "value": "https://github.com/aa/bb.git"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) diff --git a/helm_tests/other/test_git_sync_triggerer.py b/helm_tests/other/test_git_sync_triggerer.py index 96f7092666ebe..6d188aed7b92e 100644 --- a/helm_tests/other/test_git_sync_triggerer.py +++ b/helm_tests/other/test_git_sync_triggerer.py @@ -41,3 +41,30 @@ def test_validate_sshkeysecret_not_added_when_persistence_is_enabled(self): show_only=["templates/triggerer/triggerer-deployment.yaml"], ) assert "git-sync-ssh-key" not in jmespath.search("spec.template.spec.volumes[].name", docs[0]) + + def test_can_repo_url_be_templated(self): + docs = render_chart( + values={ + "testValues": { + "dict": { + "key": "aa", + }, + }, + "dags": { + "gitSync": { + "enabled": True, + "repo": "https://github.com/{{ .Values.testValues.dict.key }}/{{ .Release.Namespace }}.git", + } + }, + }, + show_only=["templates/scheduler/scheduler-deployment.yaml"], + namespace="bb", + ) + + assert {"name": "GIT_SYNC_REPO", "value": "https://github.com/aa/bb.git"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) + + assert {"name": "GITSYNC_REPO", "value": "https://github.com/aa/bb.git"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) diff --git a/helm_tests/other/test_git_sync_webserver.py b/helm_tests/other/test_git_sync_webserver.py index 01bf49a65b83b..c6c23f1c767c2 100644 --- a/helm_tests/other/test_git_sync_webserver.py +++ b/helm_tests/other/test_git_sync_webserver.py @@ -172,3 +172,30 @@ def test_validate_sshkeysecret_not_added_when_persistence_is_enabled(self): show_only=["templates/webserver/webserver-deployment.yaml"], ) assert "git-sync-ssh-key" not in jmespath.search("spec.template.spec.volumes[].name", docs[0]) + + def test_can_repo_url_be_templated(self): + docs = render_chart( + values={ + "testValues": { + "dict": { + "key": "aa", + }, + }, + "dags": { + "gitSync": { + "enabled": True, + "repo": "https://github.com/{{ .Values.testValues.dict.key }}/{{ .Release.Namespace }}.git", + } + }, + }, + show_only=["templates/scheduler/scheduler-deployment.yaml"], + namespace="bb", + ) + + assert {"name": "GIT_SYNC_REPO", "value": "https://github.com/aa/bb.git"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) + + assert {"name": "GITSYNC_REPO", "value": "https://github.com/aa/bb.git"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) diff --git a/helm_tests/other/test_git_sync_worker.py b/helm_tests/other/test_git_sync_worker.py index 31589a78f3ba7..168f3093c6978 100644 --- a/helm_tests/other/test_git_sync_worker.py +++ b/helm_tests/other/test_git_sync_worker.py @@ -132,3 +132,30 @@ def test_validate_sshkeysecret_not_added_when_persistence_is_enabled(self): ) assert "git-sync-ssh-key" not in jmespath.search("spec.template.spec.volumes[].name", docs[0]) + + def test_can_repo_url_be_templated(self): + docs = render_chart( + values={ + "testValues": { + "dict": { + "key": "aa", + }, + }, + "dags": { + "gitSync": { + "enabled": True, + "repo": "https://github.com/{{ .Values.testValues.dict.key }}/{{ .Release.Namespace }}.git", + } + }, + }, + show_only=["templates/scheduler/scheduler-deployment.yaml"], + namespace="bb", + ) + + assert {"name": "GIT_SYNC_REPO", "value": "https://github.com/aa/bb.git"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) + + assert {"name": "GITSYNC_REPO", "value": "https://github.com/aa/bb.git"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + )