From 64ca61820ce6c5c236869d6611549aa6297ad006 Mon Sep 17 00:00:00 2001 From: Burak Karakan Date: Wed, 4 May 2022 01:17:54 +0100 Subject: [PATCH 01/10] refactor: move the dags-data volume to a common template Signed-off-by: Burak Karakan --- charts/airflow/templates/_helpers/pods.tpl | 33 +++++++++++++--------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/charts/airflow/templates/_helpers/pods.tpl b/charts/airflow/templates/_helpers/pods.tpl index e17638d9..4686353b 100644 --- a/charts/airflow/templates/_helpers/pods.tpl +++ b/charts/airflow/templates/_helpers/pods.tpl @@ -389,6 +389,25 @@ EXAMPLE USAGE: {{ include "airflow.volumeMounts" (dict "Release" .Release "Value {{- end }} {{- end }} +{{/* +The dag data volume to be used by various pods +EXAMPLE USAGE: {{ include "volumes.dags" (dict "Release" .Release "Values" .Values) }} +*/}} +{{- define "volumes.dags" }} +{{- /* dags */ -}} +{{- if .Values.dags.persistence.enabled }} +- name: dags-data + mountPath: {{ .Values.dags.path }} + subPath: {{ .Values.dags.persistence.subPath }} + {{- if eq .Values.dags.persistence.accessMode "ReadOnlyMany" }} + readOnly: true + {{- end }} +{{- else if .Values.dags.gitSync.enabled }} +- name: dags-data + mountPath: {{ .Values.dags.path }} +{{- end }} +{{- end }} + {{/* The list of `volumes` for web/scheduler/worker/flower Pods EXAMPLE USAGE: {{ include "airflow.volumes" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages "extraVolumes" $extraVolumes) }} @@ -406,19 +425,7 @@ EXAMPLE USAGE: {{ include "airflow.volumes" (dict "Release" .Release "Values" .V defaultMode: 0644 {{- end }} -{{- /* dags */ -}} -{{- if .Values.dags.persistence.enabled }} -- name: dags-data - persistentVolumeClaim: - {{- if .Values.dags.persistence.existingClaim }} - claimName: {{ .Values.dags.persistence.existingClaim }} - {{- else }} - claimName: {{ printf "%s-dags" (include "airflow.fullname" . | trunc 58) }} - {{- end }} -{{- else if .Values.dags.gitSync.enabled }} -- name: dags-data - emptyDir: {} -{{- end }} +{{ include "volumes.dags" (dict "Release" .Release "Values" .Values) }} {{- /* logs */ -}} {{- if .Values.logs.persistence.enabled }} From b2ca992bfe9720f12b6834317c3f8e9b26203eaa Mon Sep 17 00:00:00 2001 From: Burak Karakan Date: Wed, 4 May 2022 01:18:33 +0100 Subject: [PATCH 02/10] feat: add a simple deployment for a git-sync pod Signed-off-by: Burak Karakan --- .../git-sync/git-sync-deployment.yaml | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 charts/airflow/templates/git-sync/git-sync-deployment.yaml diff --git a/charts/airflow/templates/git-sync/git-sync-deployment.yaml b/charts/airflow/templates/git-sync/git-sync-deployment.yaml new file mode 100644 index 00000000..3679f8bc --- /dev/null +++ b/charts/airflow/templates/git-sync/git-sync-deployment.yaml @@ -0,0 +1,85 @@ +{{- if and (.Values.airflow.dags.persistence.enabled .Values.airflow.dags.gitSync.enabled) }} +{{- $podNodeSelector := include "airflow.podNodeSelector" (dict "Release" .Release "Values" .Values "nodeSelector" .Values.airflow.dags.gitSync.nodeSelector) }} +{{- $podAffinity := include "airflow.podAffinity" (dict "Release" .Release "Values" .Values "affinity" .Values.airflow.dags.gitSync.affinity) }} +{{- $podTolerations := include "airflow.podTolerations" (dict "Release" .Release "Values" .Values "tolerations" .Values.airflow.dags.gitSync.tolerations) }} +{{- $podSecurityContext := include "airflow.podSecurityContext" (dict "Release" .Release "Values" .Values "securityContext" .Values.airflow.dags.gitSync.securityContext) }} +{{- $volumes := include "volumes.dags" (dict "Values" .Values ) }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "airflow.fullname" . }}-dags-gitsync + labels: + app: {{ include "airflow.labels.app" . }} + component: dags-git-sync + chart: {{ include "airflow.labels.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if .Values.airflow.dags.gitSync.labels }} + {{- toYaml .Values.airflow.dags.gitSync.labels | nindent 4 }} + {{- end }} + {{- if .Values.airflow.dags.gitSync.annotations }} + annotations: + {{- toYaml .Values.airflow.dags.gitSync.annotations | nindent 4 }} + {{- end }} +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + ## multiple gitSync pods can safely run concurrently + maxSurge: 25% + maxUnavailable: 0 + selector: + matchLabels: + app: {{ include "airflow.labels.app" . }} + component: dags-git-sync + release: {{ .Release.Name }} + template: + metadata: + annotations: + checksum/secret-config-envs: {{ include (print $.Template.BasePath "/config/secret-config-envs.yaml") . | sha256sum }} + checksum/secret-local-settings: {{ include (print $.Template.BasePath "/config/secret-local-settings.yaml") . | sha256sum }} + {{- if .Values.airflow.podAnnotations }} + {{- toYaml .Values.airflow.podAnnotations | nindent 8 }} + {{- end }} + {{- if .Values.airflow.dags.gitSync.podAnnotations }} + {{- toYaml .Values.airflow.dags.gitSync.podAnnotations | nindent 8 }} + {{- end }} + {{- if .Values.airflow.dags.gitSync.safeToEvict }} + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + {{- end }} + labels: + app: {{ include "airflow.labels.app" . }} + component: dags-git-sync + release: {{ .Release.Name }} + {{- if .Values.airflow.dags.gitSync.podLabels }} + {{- toYaml .Values.airflow.dags.gitSync.podLabels | nindent 8 }} + {{- end }} + spec: + restartPolicy: Always + {{- if .Values.airflow.image.pullSecret }} + imagePullSecrets: + - name: {{ .Values.airflow.image.pullSecret }} + {{- end }} + {{- if $podNodeSelector }} + nodeSelector: + {{- $podNodeSelector | nindent 8 }} + {{- end }} + {{- if $podAffinity }} + affinity: + {{- $podAffinity | nindent 8 }} + {{- end }} + {{- if $podTolerations }} + tolerations: + {{- $podTolerations | nindent 8 }} + {{- end }} + {{- if $podSecurityContext }} + securityContext: + {{- $podSecurityContext | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "airflow.serviceAccountName" . }} + containers: + {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values) | indent 8 }} + volumes: + {{- $volumes | indent 8 }} +{{- end }} \ No newline at end of file From af954b2cb776cb65e1704240dcf2b84b64e3fba9 Mon Sep 17 00:00:00 2001 From: Burak Karakan Date: Wed, 4 May 2022 01:19:00 +0100 Subject: [PATCH 03/10] feat: disable the previous validation rule for persistent dags and gitsync Signed-off-by: Burak Karakan --- charts/airflow/templates/_helpers/validate-values.tpl | 3 --- 1 file changed, 3 deletions(-) diff --git a/charts/airflow/templates/_helpers/validate-values.tpl b/charts/airflow/templates/_helpers/validate-values.tpl index 42944a98..ba7bf09b 100644 --- a/charts/airflow/templates/_helpers/validate-values.tpl +++ b/charts/airflow/templates/_helpers/validate-values.tpl @@ -85,9 +85,6 @@ {{/* Checks for `dags.gitSync` */}} {{- if .Values.dags.gitSync.enabled }} - {{- if .Values.dags.persistence.enabled }} - {{ required "If `dags.gitSync.enabled=true`, then `persistence.enabled` must be disabled!" nil }} - {{- end }} {{- if not .Values.dags.gitSync.repo }} {{ required "If `dags.gitSync.enabled=true`, then `dags.gitSync.repo` must be non-empty!" nil }} {{- end }} From df8c4632ba3738c87e1e9314cefef00f8f0ad144 Mon Sep 17 00:00:00 2001 From: Burak Karakan Date: Wed, 4 May 2022 01:22:16 +0100 Subject: [PATCH 04/10] feat: add exclusion for all the gitsync containers if persistence is enabled Signed-off-by: Burak Karakan --- charts/airflow/files/pod_template.kubernetes-helm-yaml | 2 +- charts/airflow/templates/_helpers/pods.tpl | 2 +- charts/airflow/templates/_helpers/validate-values.tpl | 2 +- .../templates/db-migrations/db-migrations-deployment.yaml | 4 ++-- charts/airflow/templates/db-migrations/db-migrations-job.yaml | 2 +- charts/airflow/templates/flower/flower-deployment.yaml | 4 ++-- charts/airflow/templates/scheduler/scheduler-deployment.yaml | 4 ++-- .../airflow/templates/sync/sync-connections-deployment.yaml | 4 ++-- charts/airflow/templates/sync/sync-connections-job.yaml | 2 +- charts/airflow/templates/sync/sync-pools-deployment.yaml | 4 ++-- charts/airflow/templates/sync/sync-pools-job.yaml | 2 +- charts/airflow/templates/sync/sync-users-deployment.yaml | 4 ++-- charts/airflow/templates/sync/sync-users-job.yaml | 2 +- charts/airflow/templates/sync/sync-variables-deployment.yaml | 4 ++-- charts/airflow/templates/sync/sync-variables-job.yaml | 2 +- charts/airflow/templates/triggerer/triggerer-deployment.yaml | 4 ++-- charts/airflow/templates/webserver/webserver-deployment.yaml | 4 ++-- charts/airflow/templates/worker/worker-statefulset.yaml | 4 ++-- 18 files changed, 28 insertions(+), 28 deletions(-) diff --git a/charts/airflow/files/pod_template.kubernetes-helm-yaml b/charts/airflow/files/pod_template.kubernetes-helm-yaml index fff65d0b..2b3fa888 100644 --- a/charts/airflow/files/pod_template.kubernetes-helm-yaml +++ b/charts/airflow/files/pod_template.kubernetes-helm-yaml @@ -48,7 +48,7 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 4 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 4 }} {{- end }} {{- if .Values.airflow.kubernetesPodTemplate.extraInitContainers }} diff --git a/charts/airflow/templates/_helpers/pods.tpl b/charts/airflow/templates/_helpers/pods.tpl index 4686353b..1ad83253 100644 --- a/charts/airflow/templates/_helpers/pods.tpl +++ b/charts/airflow/templates/_helpers/pods.tpl @@ -442,7 +442,7 @@ EXAMPLE USAGE: {{ include "airflow.volumes" (dict "Release" .Release "Values" .V {{- end }} {{- /* git-sync */ -}} -{{- if .Values.dags.gitSync.enabled }} +{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} {{- if .Values.dags.gitSync.sshSecret }} - name: git-secret secret: diff --git a/charts/airflow/templates/_helpers/validate-values.tpl b/charts/airflow/templates/_helpers/validate-values.tpl index ba7bf09b..0ddce053 100644 --- a/charts/airflow/templates/_helpers/validate-values.tpl +++ b/charts/airflow/templates/_helpers/validate-values.tpl @@ -84,7 +84,7 @@ {{- end }} {{/* Checks for `dags.gitSync` */}} -{{- if .Values.dags.gitSync.enabled }} +{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} {{- if not .Values.dags.gitSync.repo }} {{ required "If `dags.gitSync.enabled=true`, then `dags.gitSync.repo` must be non-empty!" nil }} {{- end }} diff --git a/charts/airflow/templates/db-migrations/db-migrations-deployment.yaml b/charts/airflow/templates/db-migrations/db-migrations-deployment.yaml index 1ad60348..48018eaf 100644 --- a/charts/airflow/templates/db-migrations/db-migrations-deployment.yaml +++ b/charts/airflow/templates/db-migrations/db-migrations-deployment.yaml @@ -82,7 +82,7 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} @@ -107,7 +107,7 @@ spec: - name: scripts mountPath: /mnt/scripts readOnly: true - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" . | indent 8 }} {{- end }} diff --git a/charts/airflow/templates/db-migrations/db-migrations-job.yaml b/charts/airflow/templates/db-migrations/db-migrations-job.yaml index 2e729af6..0f646d2a 100644 --- a/charts/airflow/templates/db-migrations/db-migrations-job.yaml +++ b/charts/airflow/templates/db-migrations/db-migrations-job.yaml @@ -76,7 +76,7 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} diff --git a/charts/airflow/templates/flower/flower-deployment.yaml b/charts/airflow/templates/flower/flower-deployment.yaml index 22f5fa11..1ea72d77 100644 --- a/charts/airflow/templates/flower/flower-deployment.yaml +++ b/charts/airflow/templates/flower/flower-deployment.yaml @@ -86,7 +86,7 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} @@ -151,7 +151,7 @@ spec: volumeMounts: {{- $volumeMounts | indent 12 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" . | indent 8 }} {{- end }} diff --git a/charts/airflow/templates/scheduler/scheduler-deployment.yaml b/charts/airflow/templates/scheduler/scheduler-deployment.yaml index c43f3904..5527e870 100644 --- a/charts/airflow/templates/scheduler/scheduler-deployment.yaml +++ b/charts/airflow/templates/scheduler/scheduler-deployment.yaml @@ -94,7 +94,7 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} @@ -196,7 +196,7 @@ spec: readOnly: true {{- end }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} {{- include "airflow.container.git_sync" . | indent 8 }} {{- end }} {{- if .Values.scheduler.logCleanup.enabled }} diff --git a/charts/airflow/templates/sync/sync-connections-deployment.yaml b/charts/airflow/templates/sync/sync-connections-deployment.yaml index 82734305..a90111e8 100644 --- a/charts/airflow/templates/sync/sync-connections-deployment.yaml +++ b/charts/airflow/templates/sync/sync-connections-deployment.yaml @@ -82,7 +82,7 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} @@ -113,7 +113,7 @@ spec: mountPath: "/mnt/templates" readOnly: true {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" . | indent 8 }} {{- end }} diff --git a/charts/airflow/templates/sync/sync-connections-job.yaml b/charts/airflow/templates/sync/sync-connections-job.yaml index c77002ad..7f6ec176 100644 --- a/charts/airflow/templates/sync/sync-connections-job.yaml +++ b/charts/airflow/templates/sync/sync-connections-job.yaml @@ -76,7 +76,7 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} diff --git a/charts/airflow/templates/sync/sync-pools-deployment.yaml b/charts/airflow/templates/sync/sync-pools-deployment.yaml index 1635a987..81aeca61 100644 --- a/charts/airflow/templates/sync/sync-pools-deployment.yaml +++ b/charts/airflow/templates/sync/sync-pools-deployment.yaml @@ -82,7 +82,7 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} @@ -108,7 +108,7 @@ spec: - name: scripts mountPath: /mnt/scripts readOnly: true - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" . | indent 8 }} {{- end }} diff --git a/charts/airflow/templates/sync/sync-pools-job.yaml b/charts/airflow/templates/sync/sync-pools-job.yaml index cda0f7cd..72b7c24d 100644 --- a/charts/airflow/templates/sync/sync-pools-job.yaml +++ b/charts/airflow/templates/sync/sync-pools-job.yaml @@ -76,7 +76,7 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} diff --git a/charts/airflow/templates/sync/sync-users-deployment.yaml b/charts/airflow/templates/sync/sync-users-deployment.yaml index d61d8943..a9d43a9f 100644 --- a/charts/airflow/templates/sync/sync-users-deployment.yaml +++ b/charts/airflow/templates/sync/sync-users-deployment.yaml @@ -82,7 +82,7 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} @@ -113,7 +113,7 @@ spec: mountPath: "/mnt/templates" readOnly: true {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" . | indent 8 }} {{- end }} diff --git a/charts/airflow/templates/sync/sync-users-job.yaml b/charts/airflow/templates/sync/sync-users-job.yaml index d96c8da7..bb3b2153 100644 --- a/charts/airflow/templates/sync/sync-users-job.yaml +++ b/charts/airflow/templates/sync/sync-users-job.yaml @@ -76,7 +76,7 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} diff --git a/charts/airflow/templates/sync/sync-variables-deployment.yaml b/charts/airflow/templates/sync/sync-variables-deployment.yaml index 21926197..4be64441 100644 --- a/charts/airflow/templates/sync/sync-variables-deployment.yaml +++ b/charts/airflow/templates/sync/sync-variables-deployment.yaml @@ -82,7 +82,7 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} @@ -113,7 +113,7 @@ spec: mountPath: "/mnt/templates" readOnly: true {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" . | indent 8 }} {{- end }} diff --git a/charts/airflow/templates/sync/sync-variables-job.yaml b/charts/airflow/templates/sync/sync-variables-job.yaml index bd50349e..919a4582 100644 --- a/charts/airflow/templates/sync/sync-variables-job.yaml +++ b/charts/airflow/templates/sync/sync-variables-job.yaml @@ -76,7 +76,7 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} diff --git a/charts/airflow/templates/triggerer/triggerer-deployment.yaml b/charts/airflow/templates/triggerer/triggerer-deployment.yaml index b079f77e..f8d8a1af 100644 --- a/charts/airflow/templates/triggerer/triggerer-deployment.yaml +++ b/charts/airflow/templates/triggerer/triggerer-deployment.yaml @@ -86,7 +86,7 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} @@ -147,7 +147,7 @@ spec: volumeMounts: {{- $volumeMounts | indent 12 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} {{- include "airflow.container.git_sync" . | indent 8 }} {{- end }} {{- if .Values.airflow.extraContainers }} diff --git a/charts/airflow/templates/webserver/webserver-deployment.yaml b/charts/airflow/templates/webserver/webserver-deployment.yaml index 7af636fb..ac2af3e9 100644 --- a/charts/airflow/templates/webserver/webserver-deployment.yaml +++ b/charts/airflow/templates/webserver/webserver-deployment.yaml @@ -86,7 +86,7 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} @@ -140,7 +140,7 @@ spec: mountPath: /opt/airflow/webserver_config.py subPath: webserver_config.py readOnly: true - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} {{- include "airflow.container.git_sync" . | indent 8 }} {{- end }} {{- if .Values.airflow.extraContainers }} diff --git a/charts/airflow/templates/worker/worker-statefulset.yaml b/charts/airflow/templates/worker/worker-statefulset.yaml index e5147aa1..17735fa0 100644 --- a/charts/airflow/templates/worker/worker-statefulset.yaml +++ b/charts/airflow/templates/worker/worker-statefulset.yaml @@ -91,7 +91,7 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} @@ -168,7 +168,7 @@ spec: volumeMounts: {{- $volumeMounts | indent 12 }} {{- end }} - {{- if .Values.dags.gitSync.enabled }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} {{- include "airflow.container.git_sync" . | indent 8 }} {{- end }} {{- if .Values.workers.logCleanup.enabled }} From 828e3e86bc3a0b3ed6e7d2f2c9fe5c1af89fda72 Mon Sep 17 00:00:00 2001 From: Burak Karakan Date: Thu, 12 May 2022 23:29:12 +0100 Subject: [PATCH 05/10] fix: correct the values reference for dags key Signed-off-by: Burak Karakan --- .../git-sync/git-sync-deployment.yaml | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/charts/airflow/templates/git-sync/git-sync-deployment.yaml b/charts/airflow/templates/git-sync/git-sync-deployment.yaml index 3679f8bc..e7d1e44e 100644 --- a/charts/airflow/templates/git-sync/git-sync-deployment.yaml +++ b/charts/airflow/templates/git-sync/git-sync-deployment.yaml @@ -1,8 +1,8 @@ -{{- if and (.Values.airflow.dags.persistence.enabled .Values.airflow.dags.gitSync.enabled) }} -{{- $podNodeSelector := include "airflow.podNodeSelector" (dict "Release" .Release "Values" .Values "nodeSelector" .Values.airflow.dags.gitSync.nodeSelector) }} -{{- $podAffinity := include "airflow.podAffinity" (dict "Release" .Release "Values" .Values "affinity" .Values.airflow.dags.gitSync.affinity) }} -{{- $podTolerations := include "airflow.podTolerations" (dict "Release" .Release "Values" .Values "tolerations" .Values.airflow.dags.gitSync.tolerations) }} -{{- $podSecurityContext := include "airflow.podSecurityContext" (dict "Release" .Release "Values" .Values "securityContext" .Values.airflow.dags.gitSync.securityContext) }} +{{- if and (.Values.dags.persistence.enabled) (.Values.dags.gitSync.enabled) }} +{{- $podNodeSelector := include "airflow.podNodeSelector" (dict "Release" .Release "Values" .Values "nodeSelector" .Values.dags.gitSync.nodeSelector) }} +{{- $podAffinity := include "airflow.podAffinity" (dict "Release" .Release "Values" .Values "affinity" .Values.dags.gitSync.affinity) }} +{{- $podTolerations := include "airflow.podTolerations" (dict "Release" .Release "Values" .Values "tolerations" .Values.dags.gitSync.tolerations) }} +{{- $podSecurityContext := include "airflow.podSecurityContext" (dict "Release" .Release "Values" .Values "securityContext" .Values.dags.gitSync.securityContext) }} {{- $volumes := include "volumes.dags" (dict "Values" .Values ) }} apiVersion: apps/v1 kind: Deployment @@ -14,12 +14,12 @@ metadata: chart: {{ include "airflow.labels.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} - {{- if .Values.airflow.dags.gitSync.labels }} - {{- toYaml .Values.airflow.dags.gitSync.labels | nindent 4 }} + {{- if .Values.dags.gitSync.labels }} + {{- toYaml .Values.dags.gitSync.labels | nindent 4 }} {{- end }} - {{- if .Values.airflow.dags.gitSync.annotations }} + {{- if .Values.dags.gitSync.annotations }} annotations: - {{- toYaml .Values.airflow.dags.gitSync.annotations | nindent 4 }} + {{- toYaml .Values.dags.gitSync.annotations | nindent 4 }} {{- end }} spec: replicas: 1 @@ -42,18 +42,18 @@ spec: {{- if .Values.airflow.podAnnotations }} {{- toYaml .Values.airflow.podAnnotations | nindent 8 }} {{- end }} - {{- if .Values.airflow.dags.gitSync.podAnnotations }} - {{- toYaml .Values.airflow.dags.gitSync.podAnnotations | nindent 8 }} + {{- if .Values.dags.gitSync.podAnnotations }} + {{- toYaml .Values.dags.gitSync.podAnnotations | nindent 8 }} {{- end }} - {{- if .Values.airflow.dags.gitSync.safeToEvict }} + {{- if .Values.dags.gitSync.safeToEvict }} cluster-autoscaler.kubernetes.io/safe-to-evict: "true" {{- end }} labels: app: {{ include "airflow.labels.app" . }} component: dags-git-sync release: {{ .Release.Name }} - {{- if .Values.airflow.dags.gitSync.podLabels }} - {{- toYaml .Values.airflow.dags.gitSync.podLabels | nindent 8 }} + {{- if .Values.dags.gitSync.podLabels }} + {{- toYaml .Values.dags.gitSync.podLabels | nindent 8 }} {{- end }} spec: restartPolicy: Always From 3512922ac93cd16dbad87ae703f8fad837db780a Mon Sep 17 00:00:00 2001 From: Burak Karakan Date: Fri, 13 May 2022 00:24:48 +0100 Subject: [PATCH 06/10] feat: copy dags from the mounted pvc to the local dag path Signed-off-by: Burak Karakan --- .../files/pod_template.kubernetes-helm-yaml | 6 ++++- charts/airflow/templates/_helpers/pods.tpl | 23 +++++++++++++++++-- .../db-migrations-deployment.yaml | 6 ++++- .../db-migrations/db-migrations-job.yaml | 6 ++++- .../templates/flower/flower-deployment.yaml | 6 ++++- .../scheduler/scheduler-deployment.yaml | 7 +++++- .../sync/sync-connections-deployment.yaml | 6 ++++- .../templates/sync/sync-connections-job.yaml | 6 ++++- .../templates/sync/sync-pools-deployment.yaml | 6 ++++- .../templates/sync/sync-pools-job.yaml | 6 ++++- .../templates/sync/sync-users-deployment.yaml | 6 ++++- .../templates/sync/sync-users-job.yaml | 6 ++++- .../sync/sync-variables-deployment.yaml | 6 ++++- .../templates/sync/sync-variables-job.yaml | 6 ++++- .../triggerer/triggerer-deployment.yaml | 7 +++++- .../webserver/webserver-deployment.yaml | 7 +++++- .../templates/worker/worker-statefulset.yaml | 6 ++++- 17 files changed, 104 insertions(+), 18 deletions(-) diff --git a/charts/airflow/files/pod_template.kubernetes-helm-yaml b/charts/airflow/files/pod_template.kubernetes-helm-yaml index 2b3fa888..b27f765b 100644 --- a/charts/airflow/files/pod_template.kubernetes-helm-yaml +++ b/charts/airflow/files/pod_template.kubernetes-helm-yaml @@ -48,9 +48,13 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 4 }} {{- end }} - {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} + {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 4 }} + {{- else }} {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 4 }} {{- end }} + {{- end }} {{- if .Values.airflow.kubernetesPodTemplate.extraInitContainers }} {{- toYaml .Values.airflow.kubernetesPodTemplate.extraInitContainers | nindent 4 }} {{- end }} diff --git a/charts/airflow/templates/_helpers/pods.tpl b/charts/airflow/templates/_helpers/pods.tpl index 1ad83253..3bd923b9 100644 --- a/charts/airflow/templates/_helpers/pods.tpl +++ b/charts/airflow/templates/_helpers/pods.tpl @@ -254,8 +254,6 @@ EXAMPLE USAGE: {{ include "airflow.container.git_sync" (dict "Release" .Release name: {{ .Values.dags.gitSync.httpSecret }} key: {{ .Values.dags.gitSync.httpSecretPasswordKey }} {{- end }} - {{- /* this has user-defined variables, so must be included BELOW (so the ABOVE `env` take precedence) */ -}} - {{- include "airflow.env" . | indent 4 }} volumeMounts: - name: dags-data mountPath: /dags @@ -273,6 +271,27 @@ EXAMPLE USAGE: {{ include "airflow.container.git_sync" (dict "Release" .Release {{- end }} {{- end }} +{{/* +Define a container which copies the contents of the DAG PVC to the DAG directory in the pod +EXAMPLE USAGE: {{ include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) }} +*/}} +{{- define "airflow.container.dag_pvc_copy" }} +- name: dags-copy-from-pvc + image: busybox:1.35 + imagePullPolicy: Never + securityContext: + runAsUser: {{ .Values.dags.gitSync.image.uid }} + runAsGroup: {{ .Values.dags.gitSync.image.gid }} + volumeMounts: + - name: dags-data + mountPath: /dags + command: + - "cp" + - "-r" + - "/dags" + - {{ .Values.dags.path }} +{{- end }} + {{/* Define a container which regularly deletes airflow logs older than a retention period. EXAMPLE USAGE: {{ include "airflow.container.log_cleanup" (dict "Release" .Release "Values" .Values "resources" $lc_resources "retention_min" $lc_retention_min "interval_sec" $lc_interval_sec) }} diff --git a/charts/airflow/templates/db-migrations/db-migrations-deployment.yaml b/charts/airflow/templates/db-migrations/db-migrations-deployment.yaml index 48018eaf..8aba39e1 100644 --- a/charts/airflow/templates/db-migrations/db-migrations-deployment.yaml +++ b/charts/airflow/templates/db-migrations/db-migrations-deployment.yaml @@ -82,10 +82,14 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} + {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: - name: db-migrations diff --git a/charts/airflow/templates/db-migrations/db-migrations-job.yaml b/charts/airflow/templates/db-migrations/db-migrations-job.yaml index 0f646d2a..81e93814 100644 --- a/charts/airflow/templates/db-migrations/db-migrations-job.yaml +++ b/charts/airflow/templates/db-migrations/db-migrations-job.yaml @@ -76,10 +76,14 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} + {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: - name: db-migrations diff --git a/charts/airflow/templates/flower/flower-deployment.yaml b/charts/airflow/templates/flower/flower-deployment.yaml index 1ea72d77..dbf88653 100644 --- a/charts/airflow/templates/flower/flower-deployment.yaml +++ b/charts/airflow/templates/flower/flower-deployment.yaml @@ -86,10 +86,14 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} + {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: diff --git a/charts/airflow/templates/scheduler/scheduler-deployment.yaml b/charts/airflow/templates/scheduler/scheduler-deployment.yaml index 5527e870..ecdbe656 100644 --- a/charts/airflow/templates/scheduler/scheduler-deployment.yaml +++ b/charts/airflow/templates/scheduler/scheduler-deployment.yaml @@ -94,9 +94,14 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} + {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} + ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- if .Values.scheduler.extraInitContainers }} diff --git a/charts/airflow/templates/sync/sync-connections-deployment.yaml b/charts/airflow/templates/sync/sync-connections-deployment.yaml index a90111e8..57431beb 100644 --- a/charts/airflow/templates/sync/sync-connections-deployment.yaml +++ b/charts/airflow/templates/sync/sync-connections-deployment.yaml @@ -82,10 +82,14 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} + {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: diff --git a/charts/airflow/templates/sync/sync-connections-job.yaml b/charts/airflow/templates/sync/sync-connections-job.yaml index 7f6ec176..1a344d40 100644 --- a/charts/airflow/templates/sync/sync-connections-job.yaml +++ b/charts/airflow/templates/sync/sync-connections-job.yaml @@ -76,10 +76,14 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} + {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: diff --git a/charts/airflow/templates/sync/sync-pools-deployment.yaml b/charts/airflow/templates/sync/sync-pools-deployment.yaml index 81aeca61..a4ac484c 100644 --- a/charts/airflow/templates/sync/sync-pools-deployment.yaml +++ b/charts/airflow/templates/sync/sync-pools-deployment.yaml @@ -82,10 +82,14 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} + {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: diff --git a/charts/airflow/templates/sync/sync-pools-job.yaml b/charts/airflow/templates/sync/sync-pools-job.yaml index 72b7c24d..72e5f978 100644 --- a/charts/airflow/templates/sync/sync-pools-job.yaml +++ b/charts/airflow/templates/sync/sync-pools-job.yaml @@ -76,10 +76,14 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} + {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: diff --git a/charts/airflow/templates/sync/sync-users-deployment.yaml b/charts/airflow/templates/sync/sync-users-deployment.yaml index a9d43a9f..0bce843a 100644 --- a/charts/airflow/templates/sync/sync-users-deployment.yaml +++ b/charts/airflow/templates/sync/sync-users-deployment.yaml @@ -82,10 +82,14 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} + {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: diff --git a/charts/airflow/templates/sync/sync-users-job.yaml b/charts/airflow/templates/sync/sync-users-job.yaml index bb3b2153..896765d7 100644 --- a/charts/airflow/templates/sync/sync-users-job.yaml +++ b/charts/airflow/templates/sync/sync-users-job.yaml @@ -76,10 +76,14 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} + {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: diff --git a/charts/airflow/templates/sync/sync-variables-deployment.yaml b/charts/airflow/templates/sync/sync-variables-deployment.yaml index 4be64441..6d583f68 100644 --- a/charts/airflow/templates/sync/sync-variables-deployment.yaml +++ b/charts/airflow/templates/sync/sync-variables-deployment.yaml @@ -82,10 +82,14 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} + {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: diff --git a/charts/airflow/templates/sync/sync-variables-job.yaml b/charts/airflow/templates/sync/sync-variables-job.yaml index 919a4582..40a58525 100644 --- a/charts/airflow/templates/sync/sync-variables-job.yaml +++ b/charts/airflow/templates/sync/sync-variables-job.yaml @@ -76,10 +76,14 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} + {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: diff --git a/charts/airflow/templates/triggerer/triggerer-deployment.yaml b/charts/airflow/templates/triggerer/triggerer-deployment.yaml index f8d8a1af..b6f2de99 100644 --- a/charts/airflow/templates/triggerer/triggerer-deployment.yaml +++ b/charts/airflow/templates/triggerer/triggerer-deployment.yaml @@ -86,9 +86,14 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} + {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} + ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: diff --git a/charts/airflow/templates/webserver/webserver-deployment.yaml b/charts/airflow/templates/webserver/webserver-deployment.yaml index ac2af3e9..a0d07644 100644 --- a/charts/airflow/templates/webserver/webserver-deployment.yaml +++ b/charts/airflow/templates/webserver/webserver-deployment.yaml @@ -86,9 +86,14 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} + {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} + ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: diff --git a/charts/airflow/templates/worker/worker-statefulset.yaml b/charts/airflow/templates/worker/worker-statefulset.yaml index 17735fa0..c5553a77 100644 --- a/charts/airflow/templates/worker/worker-statefulset.yaml +++ b/charts/airflow/templates/worker/worker-statefulset.yaml @@ -91,9 +91,13 @@ spec: {{- if $extraPipPackages }} {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }} {{- end }} - {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} + {{- if .Values.dags.gitSync.enabled }} + {{- if .Values.dags.persistence.enabled }} + {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }} + {{- else }} {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }} {{- end }} + {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} containers: From e715b042eddb25247fa960b939fac30ed10b84b8 Mon Sep 17 00:00:00 2001 From: Burak Karakan Date: Fri, 13 May 2022 02:05:46 +0100 Subject: [PATCH 07/10] fix: make sure the volumes are correctly mounted for the git-sync deployment Signed-off-by: Burak Karakan --- charts/airflow/templates/_helpers/pods.tpl | 43 ++++---- .../git-sync/git-sync-deployment.yaml | 98 ++++++++++++++++++- 2 files changed, 110 insertions(+), 31 deletions(-) diff --git a/charts/airflow/templates/_helpers/pods.tpl b/charts/airflow/templates/_helpers/pods.tpl index 3bd923b9..36e26ad6 100644 --- a/charts/airflow/templates/_helpers/pods.tpl +++ b/charts/airflow/templates/_helpers/pods.tpl @@ -278,18 +278,15 @@ EXAMPLE USAGE: {{ include "airflow.container.dag_pvc_copy" (dict "Release" .Rele {{- define "airflow.container.dag_pvc_copy" }} - name: dags-copy-from-pvc image: busybox:1.35 - imagePullPolicy: Never + imagePullPolicy: IfNotPresent securityContext: runAsUser: {{ .Values.dags.gitSync.image.uid }} runAsGroup: {{ .Values.dags.gitSync.image.gid }} volumeMounts: - name: dags-data mountPath: /dags - command: - - "cp" - - "-r" - - "/dags" - - {{ .Values.dags.path }} + command: ["/bin/sh","-c"] + args: ["mkdir -p {{ .Values.dags.path }}; cp -r /dags {{ .Values.dags.path }}"] {{- end }} {{/* @@ -408,25 +405,6 @@ EXAMPLE USAGE: {{ include "airflow.volumeMounts" (dict "Release" .Release "Value {{- end }} {{- end }} -{{/* -The dag data volume to be used by various pods -EXAMPLE USAGE: {{ include "volumes.dags" (dict "Release" .Release "Values" .Values) }} -*/}} -{{- define "volumes.dags" }} -{{- /* dags */ -}} -{{- if .Values.dags.persistence.enabled }} -- name: dags-data - mountPath: {{ .Values.dags.path }} - subPath: {{ .Values.dags.persistence.subPath }} - {{- if eq .Values.dags.persistence.accessMode "ReadOnlyMany" }} - readOnly: true - {{- end }} -{{- else if .Values.dags.gitSync.enabled }} -- name: dags-data - mountPath: {{ .Values.dags.path }} -{{- end }} -{{- end }} - {{/* The list of `volumes` for web/scheduler/worker/flower Pods EXAMPLE USAGE: {{ include "airflow.volumes" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages "extraVolumes" $extraVolumes) }} @@ -444,7 +422,20 @@ EXAMPLE USAGE: {{ include "airflow.volumes" (dict "Release" .Release "Values" .V defaultMode: 0644 {{- end }} -{{ include "volumes.dags" (dict "Release" .Release "Values" .Values) }} +{{- /* dags */ -}} +{{- if .Values.dags.persistence.enabled }} +- name: dags-data + persistentVolumeClaim: + {{- if .Values.dags.persistence.existingClaim }} + claimName: {{ .Values.dags.persistence.existingClaim }} + {{- else }} + claimName: {{ printf "%s-dags" (include "airflow.fullname" . | trunc 58) }} + {{- end }} +{{- else if .Values.dags.gitSync.enabled }} +- name: dags-data + emptyDir: {} +{{- end }} + {{- /* logs */ -}} {{- if .Values.logs.persistence.enabled }} diff --git a/charts/airflow/templates/git-sync/git-sync-deployment.yaml b/charts/airflow/templates/git-sync/git-sync-deployment.yaml index e7d1e44e..549b0fbc 100644 --- a/charts/airflow/templates/git-sync/git-sync-deployment.yaml +++ b/charts/airflow/templates/git-sync/git-sync-deployment.yaml @@ -3,7 +3,6 @@ {{- $podAffinity := include "airflow.podAffinity" (dict "Release" .Release "Values" .Values "affinity" .Values.dags.gitSync.affinity) }} {{- $podTolerations := include "airflow.podTolerations" (dict "Release" .Release "Values" .Values "tolerations" .Values.dags.gitSync.tolerations) }} {{- $podSecurityContext := include "airflow.podSecurityContext" (dict "Release" .Release "Values" .Values "securityContext" .Values.dags.gitSync.securityContext) }} -{{- $volumes := include "volumes.dags" (dict "Values" .Values ) }} apiVersion: apps/v1 kind: Deployment metadata: @@ -37,8 +36,6 @@ spec: template: metadata: annotations: - checksum/secret-config-envs: {{ include (print $.Template.BasePath "/config/secret-config-envs.yaml") . | sha256sum }} - checksum/secret-local-settings: {{ include (print $.Template.BasePath "/config/secret-local-settings.yaml") . | sha256sum }} {{- if .Values.airflow.podAnnotations }} {{- toYaml .Values.airflow.podAnnotations | nindent 8 }} {{- end }} @@ -79,7 +76,98 @@ spec: {{- end }} serviceAccountName: {{ include "airflow.serviceAccountName" . }} containers: - {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values) | indent 8 }} + - name: dags-git-sync + image: {{ .Values.dags.gitSync.image.repository }}:{{ .Values.dags.gitSync.image.tag }} + imagePullPolicy: {{ .Values.dags.gitSync.image.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + runAsUser: 0 + resources: + {{- toYaml .Values.dags.gitSync.resources | nindent 12 }} + env: + - name: GIT_SYNC_ROOT + value: "/dags" + - name: GIT_SYNC_DEST + value: "repo" + - name: GIT_SYNC_REPO + value: {{ .Values.dags.gitSync.repo | quote }} + - name: GIT_SYNC_BRANCH + value: {{ .Values.dags.gitSync.branch | quote }} + - name: GIT_SYNC_REV + value: {{ .Values.dags.gitSync.revision | quote }} + - name: GIT_SYNC_DEPTH + value: {{ .Values.dags.gitSync.depth | quote }} + - name: GIT_SYNC_WAIT + value: {{ .Values.dags.gitSync.syncWait | quote }} + - name: GIT_SYNC_TIMEOUT + value: {{ .Values.dags.gitSync.syncTimeout | quote }} + - name: GIT_SYNC_ADD_USER + value: "true" + - name: GIT_SYNC_MAX_SYNC_FAILURES + value: {{ .Values.dags.gitSync.maxFailures | quote }} + {{- if .Values.dags.gitSync.sshSecret }} + - name: GIT_SYNC_SSH + value: "true" + - name: GIT_SSH_KEY_FILE + value: "/etc/git-secret/id_rsa" + {{- end }} + {{- if .Values.dags.gitSync.sshKnownHosts }} + - name: GIT_KNOWN_HOSTS + value: "true" + - name: GIT_SSH_KNOWN_HOSTS_FILE + value: "/etc/git-secret/known_hosts" + {{- else }} + - name: GIT_KNOWN_HOSTS + value: "false" + {{- end }} + {{- if .Values.dags.gitSync.httpSecret }} + - name: GIT_SYNC_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Values.dags.gitSync.httpSecret }} + key: {{ .Values.dags.gitSync.httpSecretUsernameKey }} + - name: GIT_SYNC_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.dags.gitSync.httpSecret }} + key: {{ .Values.dags.gitSync.httpSecretPasswordKey }} + {{- end }} + volumeMounts: + - name: dags-data + mountPath: /dags + {{- if .Values.dags.gitSync.sshSecret }} + - name: git-secret + mountPath: /etc/git-secret/id_rsa + readOnly: true + subPath: {{ .Values.dags.gitSync.sshSecretKey }} + {{- end }} + {{- if .Values.dags.gitSync.sshKnownHosts }} + - name: git-known-hosts + mountPath: /etc/git-secret/known_hosts + readOnly: true + subPath: known_hosts + {{- end }} volumes: - {{- $volumes | indent 8 }} + - name: dags-data + persistentVolumeClaim: + {{- if .Values.dags.persistence.existingClaim }} + claimName: {{ .Values.dags.persistence.existingClaim }} + {{- else }} + claimName: {{ printf "%s-dags" (include "airflow.fullname" . | trunc 58) }} + {{- end }} + + {{- if .Values.dags.gitSync.sshSecret }} + - name: git-secret + secret: + secretName: {{ .Values.dags.gitSync.sshSecret }} + defaultMode: 0400 + {{- end }} + + {{- if .Values.dags.gitSync.sshKnownHosts }} + - name: git-known-hosts + secret: + secretName: {{ include "airflow.fullname" . }}-known-hosts + defaultMode: 0644 + {{- end }} + {{- end }} \ No newline at end of file From dfc9fc881f56580db29aecd0594e121cda0d74e5 Mon Sep 17 00:00:00 2001 From: Burak Karakan Date: Fri, 13 May 2022 19:09:18 +0100 Subject: [PATCH 08/10] fix: implement correct permissions for git-sync deployment Signed-off-by: Burak Karakan --- charts/airflow/templates/git-sync/git-sync-deployment.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/airflow/templates/git-sync/git-sync-deployment.yaml b/charts/airflow/templates/git-sync/git-sync-deployment.yaml index 549b0fbc..9e489707 100644 --- a/charts/airflow/templates/git-sync/git-sync-deployment.yaml +++ b/charts/airflow/templates/git-sync/git-sync-deployment.yaml @@ -80,8 +80,8 @@ spec: image: {{ .Values.dags.gitSync.image.repository }}:{{ .Values.dags.gitSync.image.tag }} imagePullPolicy: {{ .Values.dags.gitSync.image.pullPolicy }} securityContext: - allowPrivilegeEscalation: false - runAsUser: 0 + runAsUser: {{ .Values.dags.gitSync.image.uid }} + runAsGroup: {{ .Values.dags.gitSync.image.gid }} resources: {{- toYaml .Values.dags.gitSync.resources | nindent 12 }} env: @@ -160,7 +160,7 @@ spec: - name: git-secret secret: secretName: {{ .Values.dags.gitSync.sshSecret }} - defaultMode: 0400 + defaultMode: 0644 {{- end }} {{- if .Values.dags.gitSync.sshKnownHosts }} From 4520c46fb7892286b7a3994eac65970127ada2f5 Mon Sep 17 00:00:00 2001 From: Burak Karakan Date: Fri, 13 May 2022 19:49:01 +0100 Subject: [PATCH 09/10] fix: mount correct paths for the copied dags Signed-off-by: Burak Karakan --- charts/airflow/templates/_helpers/pods.tpl | 29 +++++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/charts/airflow/templates/_helpers/pods.tpl b/charts/airflow/templates/_helpers/pods.tpl index 36e26ad6..5387ef96 100644 --- a/charts/airflow/templates/_helpers/pods.tpl +++ b/charts/airflow/templates/_helpers/pods.tpl @@ -280,13 +280,15 @@ EXAMPLE USAGE: {{ include "airflow.container.dag_pvc_copy" (dict "Release" .Rele image: busybox:1.35 imagePullPolicy: IfNotPresent securityContext: - runAsUser: {{ .Values.dags.gitSync.image.uid }} - runAsGroup: {{ .Values.dags.gitSync.image.gid }} + runAsUser: {{ .Values.airflow.image.uid }} + runAsGroup: {{ .Values.airflow.image.gid }} volumeMounts: + - name: original-dag-content + mountPath: /dags/original-dags - name: dags-data - mountPath: /dags + mountPath: /dags/copied-dags command: ["/bin/sh","-c"] - args: ["mkdir -p {{ .Values.dags.path }}; cp -r /dags {{ .Values.dags.path }}"] + args: ["cp -rL /dags/original-dags/repo /dags/copied-dags/"] {{- end }} {{/* @@ -367,12 +369,17 @@ EXAMPLE USAGE: {{ include "airflow.volumeMounts" (dict "Release" .Release "Value {{- /* dags */ -}} {{- if .Values.dags.persistence.enabled }} +{{- if .Values.dags.gitSync.enabled }} +- name: dags-data + mountPath: {{ .Values.dags.path }} +{{- else }} - name: dags-data mountPath: {{ .Values.dags.path }} subPath: {{ .Values.dags.persistence.subPath }} {{- if eq .Values.dags.persistence.accessMode "ReadOnlyMany" }} readOnly: true {{- end }} +{{- end }} {{- else if .Values.dags.gitSync.enabled }} - name: dags-data mountPath: {{ .Values.dags.path }} @@ -424,6 +431,18 @@ EXAMPLE USAGE: {{ include "airflow.volumes" (dict "Release" .Release "Values" .V {{- /* dags */ -}} {{- if .Values.dags.persistence.enabled }} + +{{- if .Values.dags.gitSync.enabled }} +- name: original-dag-content + persistentVolumeClaim: + {{- if .Values.dags.persistence.existingClaim }} + claimName: {{ .Values.dags.persistence.existingClaim }} + {{- else }} + claimName: {{ printf "%s-dags" (include "airflow.fullname" . | trunc 58) }} + {{- end }} +- name: dags-data + emptyDir: {} +{{- else }} - name: dags-data persistentVolumeClaim: {{- if .Values.dags.persistence.existingClaim }} @@ -431,6 +450,8 @@ EXAMPLE USAGE: {{ include "airflow.volumes" (dict "Release" .Release "Values" .V {{- else }} claimName: {{ printf "%s-dags" (include "airflow.fullname" . | trunc 58) }} {{- end }} +{{- end }} + {{- else if .Values.dags.gitSync.enabled }} - name: dags-data emptyDir: {} From a5e8401d3493cde4d33cbdeb3173f799a33cee7c Mon Sep 17 00:00:00 2001 From: Burak Karakan Date: Fri, 13 May 2022 19:49:38 +0100 Subject: [PATCH 10/10] feat: introduce securityContext under the gitSync values Signed-off-by: Burak Karakan --- charts/airflow/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/airflow/values.yaml b/charts/airflow/values.yaml index 569d78ff..0cb4aa2b 100644 --- a/charts/airflow/values.yaml +++ b/charts/airflow/values.yaml @@ -1385,6 +1385,8 @@ dags: ## maxFailures: 0 + securityContext: {} + ################################### ## CONFIG | Kubernetes Ingress ###################################