feat: add support for git-sync with PVC (central sync deployment)

charts/airflow/files/pod_template.kubernetes-helm-yaml

@@ -49,8 +49,12 @@ spec:
     {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 4 }}
     {{- end }}
     {{- if .Values.dags.gitSync.enabled }}
+    {{- if .Values.dags.persistence.enabled  }}
+    {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 4 }}
+    {{- else }}
     {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 4 }}
     {{- end }}
+    {{- end }}
     {{- if .Values.airflow.kubernetesPodTemplate.extraInitContainers }}
     {{- toYaml .Values.airflow.kubernetesPodTemplate.extraInitContainers | nindent 4 }}
     {{- end }}

charts/airflow/templates/_helpers/pods.tpl

@@ -254,8 +254,6 @@ EXAMPLE USAGE: {{ include "airflow.container.git_sync" (dict "Release" .Release
           name: {{ .Values.dags.gitSync.httpSecret }}
           key: {{ .Values.dags.gitSync.httpSecretPasswordKey }}
     {{- end }}
-    {{- /* this has user-defined variables, so must be included BELOW (so the ABOVE `env` take precedence) */ -}}
-    {{- include "airflow.env" . | indent 4 }}
   volumeMounts:
     - name: dags-data
       mountPath: /dags
@@ -273,6 +271,26 @@ EXAMPLE USAGE: {{ include "airflow.container.git_sync" (dict "Release" .Release
     {{- end }}
 {{- end }}
 
+{{/*
+Define a container which copies the contents of the DAG PVC to the DAG directory in the pod
+EXAMPLE USAGE: {{ include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values)  }}
+*/}}
+{{- define "airflow.container.dag_pvc_copy" }}
+- name: dags-copy-from-pvc
+  image: busybox:1.35
+  imagePullPolicy: IfNotPresent
+  securityContext:
+    runAsUser: {{ .Values.airflow.image.uid }}
+    runAsGroup: {{ .Values.airflow.image.gid }}
+  volumeMounts:
+    - name: original-dag-content
+      mountPath: /dags/original-dags
+    - name: dags-data
+      mountPath: /dags/copied-dags
+  command: ["/bin/sh","-c"]
+  args: ["cp -rL /dags/original-dags/repo /dags/copied-dags/"]
+{{- end }}
+
 {{/*
 Define a container which regularly deletes airflow logs older than a retention period.
 EXAMPLE USAGE: {{ include "airflow.container.log_cleanup" (dict "Release" .Release "Values" .Values "resources" $lc_resources "retention_min" $lc_retention_min "interval_sec" $lc_interval_sec) }}
@@ -351,12 +369,17 @@ EXAMPLE USAGE: {{ include "airflow.volumeMounts" (dict "Release" .Release "Value
 
 {{- /* dags */ -}}
 {{- if .Values.dags.persistence.enabled }}
+{{- if .Values.dags.gitSync.enabled }}
+- name: dags-data
+  mountPath: {{ .Values.dags.path }}
+{{- else }}
 - name: dags-data
   mountPath: {{ .Values.dags.path }}
   subPath: {{ .Values.dags.persistence.subPath }}
   {{- if eq .Values.dags.persistence.accessMode "ReadOnlyMany" }}
   readOnly: true
   {{- end }}
+{{- end }}
 {{- else if .Values.dags.gitSync.enabled }}
 - name: dags-data
   mountPath: {{ .Values.dags.path }}
@@ -408,18 +431,33 @@ EXAMPLE USAGE: {{ include "airflow.volumes" (dict "Release" .Release "Values" .V
 
 {{- /* dags */ -}}
 {{- if .Values.dags.persistence.enabled }}
+
+{{- if .Values.dags.gitSync.enabled }}
+- name: original-dag-content
+  persistentVolumeClaim:
+    {{- if .Values.dags.persistence.existingClaim }}
+    claimName: {{ .Values.dags.persistence.existingClaim }}
+    {{- else }}
+    claimName: {{ printf "%s-dags" (include "airflow.fullname" . | trunc 58) }}
+    {{- end }}
+- name: dags-data
+  emptyDir: {}
+{{- else }}
 - name: dags-data
   persistentVolumeClaim:
     {{- if .Values.dags.persistence.existingClaim }}
     claimName: {{ .Values.dags.persistence.existingClaim }}
     {{- else }}
     claimName: {{ printf "%s-dags" (include "airflow.fullname" . | trunc 58) }}
     {{- end }}
+{{- end }}
+
 {{- else if .Values.dags.gitSync.enabled }}
 - name: dags-data
   emptyDir: {}
 {{- end }}
 
+
 {{- /* logs */ -}}
 {{- if .Values.logs.persistence.enabled }}
 - name: logs-data
@@ -435,7 +473,7 @@ EXAMPLE USAGE: {{ include "airflow.volumes" (dict "Release" .Release "Values" .V
 {{- end }}
 
 {{- /* git-sync */ -}}
-{{- if .Values.dags.gitSync.enabled }}
+{{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
 {{- if .Values.dags.gitSync.sshSecret }}
 - name: git-secret
   secret:

charts/airflow/templates/_helpers/validate-values.tpl

@@ -84,10 +84,7 @@
 {{- end }}
 
 {{/* Checks for `dags.gitSync` */}}
-{{- if .Values.dags.gitSync.enabled }}
-  {{- if .Values.dags.persistence.enabled }}
-  {{ required "If `dags.gitSync.enabled=true`, then `persistence.enabled` must be disabled!" nil }}
-  {{- end }}
+{{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
   {{- if not .Values.dags.gitSync.repo }}
   {{ required "If `dags.gitSync.enabled=true`, then `dags.gitSync.repo` must be non-empty!" nil }}
   {{- end }}

charts/airflow/templates/db-migrations/db-migrations-deployment.yaml

@@ -83,9 +83,13 @@ spec:
         {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
         {{- end }}
         {{- if .Values.dags.gitSync.enabled }}
+        {{- if .Values.dags.persistence.enabled  }}
+        {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }}
+        {{- else }}
         ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
         {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
         {{- end }}
+        {{- end }}
         {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
       containers:
         - name: db-migrations
@@ -107,7 +111,7 @@ spec:
             - name: scripts
               mountPath: /mnt/scripts
               readOnly: true
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
         {{- include "airflow.container.git_sync" . | indent 8 }}
         {{- end }}

charts/airflow/templates/db-migrations/db-migrations-job.yaml

@@ -77,9 +77,13 @@ spec:
         {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
         {{- end }}
         {{- if .Values.dags.gitSync.enabled }}
+        {{- if .Values.dags.persistence.enabled  }}
+        {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }}
+        {{- else }}
         ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
         {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
         {{- end }}
+        {{- end }}
         {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
       containers:
         - name: db-migrations

charts/airflow/templates/flower/flower-deployment.yaml

@@ -87,9 +87,13 @@ spec:
         {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
         {{- end }}
         {{- if .Values.dags.gitSync.enabled }}
+        {{- if .Values.dags.persistence.enabled  }}
+        {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }}
+        {{- else }}
         ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
         {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
         {{- end }}
+        {{- end }}
         {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
         {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
       containers:
@@ -151,7 +155,7 @@ spec:
           volumeMounts:
             {{- $volumeMounts | indent 12 }}
           {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
         {{- include "airflow.container.git_sync" . | indent 8 }}
         {{- end }}

charts/airflow/templates/git-sync/git-sync-deployment.yaml

@@ -0,0 +1,173 @@
+{{- if and (.Values.dags.persistence.enabled) (.Values.dags.gitSync.enabled) }}
+{{- $podNodeSelector := include "airflow.podNodeSelector" (dict "Release" .Release "Values" .Values "nodeSelector" .Values.dags.gitSync.nodeSelector) }}
+{{- $podAffinity := include "airflow.podAffinity" (dict "Release" .Release "Values" .Values "affinity" .Values.dags.gitSync.affinity) }}
+{{- $podTolerations := include "airflow.podTolerations" (dict "Release" .Release "Values" .Values "tolerations" .Values.dags.gitSync.tolerations) }}
+{{- $podSecurityContext := include "airflow.podSecurityContext" (dict "Release" .Release "Values" .Values "securityContext" .Values.dags.gitSync.securityContext) }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "airflow.fullname" . }}-dags-gitsync
+  labels:
+    app: {{ include "airflow.labels.app" . }}
+    component: dags-git-sync
+    chart: {{ include "airflow.labels.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+    {{- if .Values.dags.gitSync.labels }}
+    {{- toYaml .Values.dags.gitSync.labels | nindent 4 }}
+    {{- end }}
+  {{- if .Values.dags.gitSync.annotations }}
+  annotations:
+    {{- toYaml .Values.dags.gitSync.annotations | nindent 4 }}
+  {{- end }}
+spec:
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      ## multiple gitSync pods can safely run concurrently
+      maxSurge: 25%
+      maxUnavailable: 0
+  selector:
+    matchLabels:
+      app: {{ include "airflow.labels.app" . }}
+      component: dags-git-sync
+      release: {{ .Release.Name }}
+  template:
+    metadata:
+      annotations:
+        {{- if .Values.airflow.podAnnotations }}
+        {{- toYaml .Values.airflow.podAnnotations | nindent 8 }}
+        {{- end }}
+        {{- if .Values.dags.gitSync.podAnnotations }}
+        {{- toYaml .Values.dags.gitSync.podAnnotations | nindent 8 }}
+        {{- end }}
+        {{- if .Values.dags.gitSync.safeToEvict }}
+        cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
+        {{- end }}
+      labels:
+        app: {{ include "airflow.labels.app" . }}
+        component: dags-git-sync
+        release: {{ .Release.Name }}
+        {{- if .Values.dags.gitSync.podLabels }}
+        {{- toYaml .Values.dags.gitSync.podLabels | nindent 8 }}
+        {{- end }}
+    spec:
+      restartPolicy: Always
+      {{- if .Values.airflow.image.pullSecret }}
+      imagePullSecrets:
+        - name: {{ .Values.airflow.image.pullSecret }}
+      {{- end }}
+      {{- if $podNodeSelector }}
+      nodeSelector:
+        {{- $podNodeSelector | nindent 8 }}
+      {{- end }}
+      {{- if $podAffinity }}
+      affinity:
+        {{- $podAffinity | nindent 8 }}
+      {{- end }}
+      {{- if $podTolerations }}
+      tolerations:
+        {{- $podTolerations | nindent 8 }}
+      {{- end }}
+      {{- if $podSecurityContext }}
+      securityContext:
+        {{- $podSecurityContext | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "airflow.serviceAccountName" . }}
+      containers:
+        - name: dags-git-sync
+          image: {{ .Values.dags.gitSync.image.repository }}:{{ .Values.dags.gitSync.image.tag }}
+          imagePullPolicy: {{ .Values.dags.gitSync.image.pullPolicy }}
+          securityContext:
+            runAsUser: {{ .Values.dags.gitSync.image.uid }}
+            runAsGroup: {{ .Values.dags.gitSync.image.gid }}
+          resources:
+            {{- toYaml .Values.dags.gitSync.resources | nindent 12 }}
+          env:
+            - name: GIT_SYNC_ROOT
+              value: "/dags"
+            - name: GIT_SYNC_DEST
+              value: "repo"
+            - name: GIT_SYNC_REPO
+              value: {{ .Values.dags.gitSync.repo | quote }}
+            - name: GIT_SYNC_BRANCH
+              value: {{ .Values.dags.gitSync.branch | quote }}
+            - name: GIT_SYNC_REV
+              value: {{ .Values.dags.gitSync.revision | quote }}
+            - name: GIT_SYNC_DEPTH
+              value: {{ .Values.dags.gitSync.depth | quote }}
+            - name: GIT_SYNC_WAIT
+              value: {{ .Values.dags.gitSync.syncWait | quote }}
+            - name: GIT_SYNC_TIMEOUT
+              value: {{ .Values.dags.gitSync.syncTimeout | quote }}
+            - name: GIT_SYNC_ADD_USER
+              value: "true"
+            - name: GIT_SYNC_MAX_SYNC_FAILURES
+              value: {{ .Values.dags.gitSync.maxFailures | quote }}
+            {{- if .Values.dags.gitSync.sshSecret }}
+            - name: GIT_SYNC_SSH
+              value: "true"
+            - name: GIT_SSH_KEY_FILE
+              value: "/etc/git-secret/id_rsa"
+            {{- end }}
+            {{- if .Values.dags.gitSync.sshKnownHosts }}
+            - name: GIT_KNOWN_HOSTS
+              value: "true"
+            - name: GIT_SSH_KNOWN_HOSTS_FILE
+              value: "/etc/git-secret/known_hosts"
+            {{- else }}
+            - name: GIT_KNOWN_HOSTS
+              value: "false"
+            {{- end }}
+            {{- if .Values.dags.gitSync.httpSecret }}
+            - name: GIT_SYNC_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.dags.gitSync.httpSecret }}
+                  key: {{ .Values.dags.gitSync.httpSecretUsernameKey }}
+            - name: GIT_SYNC_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.dags.gitSync.httpSecret }}
+                  key: {{ .Values.dags.gitSync.httpSecretPasswordKey }}
+            {{- end }}
+          volumeMounts:
+            - name: dags-data
+              mountPath: /dags
+            {{- if .Values.dags.gitSync.sshSecret }}
+            - name: git-secret
+              mountPath: /etc/git-secret/id_rsa
+              readOnly: true
+              subPath: {{ .Values.dags.gitSync.sshSecretKey }}
+            {{- end }}
+            {{- if .Values.dags.gitSync.sshKnownHosts }}
+            - name: git-known-hosts
+              mountPath: /etc/git-secret/known_hosts
+              readOnly: true
+              subPath: known_hosts
+            {{- end }}
+      volumes:
+        - name: dags-data
+          persistentVolumeClaim:
+            {{- if .Values.dags.persistence.existingClaim }}
+            claimName: {{ .Values.dags.persistence.existingClaim }}
+            {{- else }}
+            claimName: {{ printf "%s-dags" (include "airflow.fullname" . | trunc 58) }}
+            {{- end }}
+
+        {{- if .Values.dags.gitSync.sshSecret }}
+        - name: git-secret
+          secret:
+            secretName: {{ .Values.dags.gitSync.sshSecret }}
+            defaultMode: 0644
+        {{- end }}
+
+        {{- if .Values.dags.gitSync.sshKnownHosts }}
+        - name: git-known-hosts
+          secret:
+            secretName: {{ include "airflow.fullname" . }}-known-hosts
+            defaultMode: 0644
+        {{- end }}
+
+{{- end }}

charts/airflow/templates/scheduler/scheduler-deployment.yaml

@@ -95,8 +95,13 @@ spec:
         {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
         {{- end }}
         {{- if .Values.dags.gitSync.enabled }}
+        {{- if .Values.dags.persistence.enabled  }}
+        {{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }}
+        {{- else }}
+        ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
         {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
         {{- end }}
+        {{- end }}
         {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
         {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
         {{- if .Values.scheduler.extraInitContainers }}
@@ -196,7 +201,7 @@ spec:
               readOnly: true
             {{- end }}
           {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         {{- include "airflow.container.git_sync" . | indent 8 }}
         {{- end }}
         {{- if .Values.scheduler.logCleanup.enabled }}