diff --git a/charts/airflow/templates/_helpers/pods.tpl b/charts/airflow/templates/_helpers/pods.tpl index a60cc86f..1d6641d3 100644 --- a/charts/airflow/templates/_helpers/pods.tpl +++ b/charts/airflow/templates/_helpers/pods.tpl @@ -324,11 +324,27 @@ EXAMPLE USAGE: {{ include "airflow.container.s3_sync" (dict "Release" .Release " - "-c" {{- if $.sync_one_time }} - | - aws s3 cp --recursive s3://{{ .Values.dags.s3Sync.bucket }}/{{ .Values.dags.s3Sync.s3Path }} {{ include "airflow.dags.path" . }} + BASE_DIR={{ .Values.dags.path | trimSuffix "/" }} + TEMP_DIR=$(mktemp -u $BASE_DIR/tmp.XXXXXX) + SYM_LINK="{{ include "airflow.dags.path" . | trimSuffix "/" }}" + + aws s3 cp --recursive s3://{{ .Values.dags.s3Sync.bucket }}/{{ .Values.dags.s3Sync.s3Path }} $TEMP_DIR + ln -s $TEMP_DIR $SYM_LINK {{- else }} - | + BASE_DIR="{{ .Values.dags.path | trimSuffix "/" }}" + SYM_LINK="{{ include "airflow.dags.path" . | trimSuffix "/" }}" while true; do - aws s3 sync --delete s3://{{ .Values.dags.s3Sync.bucket }}/{{ .Values.dags.s3Sync.s3Path }} {{ include "airflow.dags.path" . }} + TEMP_DIR=$(mktemp -u $BASE_DIR/tmp.XXXXXX) + CURR_DIR=$(readlink -f "$SYM_LINK") + + cp -a $CURR_DIR $TEMP_DIR + aws s3 sync --delete s3://{{ .Values.dags.s3Sync.bucket }}/{{ .Values.dags.s3Sync.s3Path }} $TEMP_DIR + if [ $? -eq 0 ]; then + rm -f $SYM_LINK + ln -s $TEMP_DIR $SYM_LINK + rm -rf $CURR_DIR + fi sleep {{ $.Values.dags.s3Sync.syncWait }} done {{- end }} diff --git a/charts/airflow/values.yaml b/charts/airflow/values.yaml index 1bf7976c..5e5129eb 100644 --- a/charts/airflow/values.yaml +++ b/charts/airflow/values.yaml @@ -1464,7 +1464,7 @@ dags: ## configs for a sync from s3 object storage ## s3Sync: - ## if the git-sync sidecar container is enabled + ## if the s3-sync sidecar container is enabled ## enabled: false