[stable/airflow] release 7.10.0

stable/airflow/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 description: Airflow is a platform to programmatically author, schedule and monitor workflows
 name: airflow
-version: 7.9.0
+version: 7.10.0
 appVersion: 1.10.12
 icon: https://airflow.apache.org/_images/pin_large.png
 home: https://airflow.apache.org/

stable/airflow/README.md

@@ -4,11 +4,22 @@
 
 ## Installation
 
-To install the Airflow Helm Chart:
+(Helm 2) To install the Airflow Helm Chart:
 ```bash
 helm install stable/airflow \
-  --version "X.X.X" \
   --name "airflow" \
+  --version "X.X.X" \
+  --namespace "airflow" \
+  --values ./custom-values.yaml
+```
+
+(Helm 3) To install the Airflow Helm Chart:
+```bash
+helm repo add stable https://kubernetes-charts.storage.googleapis.com
+helm repo update
+
+helm install "airflow" stable/airflow \
+  --version "X.X.X" \
   --namespace "airflow" \
   --values ./custom-values.yaml
 ```
@@ -23,14 +34,15 @@ To uninstall the Airflow Helm Chart:
 helm delete "airflow"
 ```
 
-To run bash commands in the Airflow Scheduler Pod:
+To run bash commands in the Airflow Webserver Pod:
 ```bash
-# use this to run commands like: `airflow create_user`
+# create an interactive bash session in the Webserver Pod
+# use this bash session for commands like: `airflow create_user`
 kubectl exec \
   -it \
   --namespace airflow \
-  --container airflow-scheduler \
-  Deployment/airflow-scheduler \
+  --container airflow-web \
+  Deployment/airflow-web \
   /bin/bash
 ```
 
@@ -39,6 +51,7 @@ kubectl exec \
 > NOTE: for chart version numbers, see [Chart.yaml](Chart.yaml) or [helm hub](https://hub.helm.sh/charts/stable/airflow).
 
 For steps you must take when upgrading this chart, please review:
+* [v7.9.X → v7.10.0](UPGRADE.md#v79x--v7100)
 * [v7.8.X → v7.9.0](UPGRADE.md#v78x--v790)
 * [v7.7.X → v7.8.0](UPGRADE.md#v77x--v780)
 * [v7.6.X → v7.7.0](UPGRADE.md#v76x--v770)

stable/airflow/UPGRADE.md

@@ -1,5 +1,14 @@
 # Upgrading Steps
 
+## `v7.9.X` → `v7.10.0`
+
+__The following IMPROVEMENTS have been made:__
+* We now make use of the `_CMD` variables for `AIRFLOW__CORE__SQL_ALCHEMY_CONN_CMD`, `AIRFLOW__CELERY__RESULT_BACKEND_CMD`, and `AIRFLOW__CELERY__BROKER_URL_CMD`:
+  * This fixes the Scheduler liveness probe implemented in `7.8.0`
+  * This fixes using `kubectl exec` to run commands like `airflow create_user`
+* Configs passed with `airflow.config` are now passed as-defined in your `values.yaml`:
+  * This fixes an issue where people had to escape `"` characters in JSON strings
+
 ## `v7.8.X` → `v7.9.0`
 
 __The following IMPROVEMENTS have been made:__

stable/airflow/templates/_helpers.tpl

@@ -67,39 +67,46 @@ Used to discover the master Service and Secret name created by the sub-chart.
 {{- end -}}
 {{- end -}}
 
-
 {{/*
-Construct the AIRFLOW__CORE__SQL_ALCHEMY_CONN connection string.
+Bash command which echos the DB connection string in SQLAlchemy format.
+NOTE:
+ - used by `AIRFLOW__CORE__SQL_ALCHEMY_CONN_CMD`
+ - the `DATABASE_PASSWORD_CMD` sub-command is set in `configmap-env`
 */}}
-{{- define "airflow.connection.alchemy" -}}
+{{- define "DATABASE_SQLALCHEMY_CMD" -}}
 {{- if .Values.postgresql.enabled -}}
-postgresql+psycopg2://${DATABASE_USER}:$( echo ${DATABASE_PASSWORD} | python3 -c "import urllib.parse; encoded_pass = urllib.parse.quote(input()); print(encoded_pass)" )@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB}
+echo -n "postgresql+psycopg2://${DATABASE_USER}:$(eval $DATABASE_PASSWORD_CMD)@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB}"
 {{- else if and (not .Values.postgresql.enabled) (eq "postgres" .Values.externalDatabase.type) -}}
-postgresql+psycopg2://${DATABASE_USER}:$( echo ${DATABASE_PASSWORD} | python3 -c "import urllib.parse; encoded_pass = urllib.parse.quote(input()); print(encoded_pass)" )@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB}${DATABASE_PROPERTIES}
+echo -n "postgresql+psycopg2://${DATABASE_USER}:$(eval $DATABASE_PASSWORD_CMD)@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB}${DATABASE_PROPERTIES}"
 {{- else if and (not .Values.postgresql.enabled) (eq "mysql" .Values.externalDatabase.type) -}}
-mysql+mysqldb://${DATABASE_USER}:$( echo ${DATABASE_PASSWORD} | python3 -c "import urllib.parse; encoded_pass = urllib.parse.quote(input()); print(encoded_pass)" )@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB}${DATABASE_PROPERTIES}
+echo -n "mysql+mysqldb://${DATABASE_USER}:$(eval $DATABASE_PASSWORD_CMD)@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB}${DATABASE_PROPERTIES}"
 {{- end -}}
 {{- end -}}
 
 {{/*
-Construct the AIRFLOW__CELERY__RESULT_BACKEND connection string.
+Bash command which echos the DB connection string in Celery result_backend format.
+NOTE:
+ - used by `AIRFLOW__CELERY__RESULT_BACKEND_CMD`
+ - the `DATABASE_PASSWORD_CMD` sub-command is set in `configmap-env`
 */}}
-{{- define "airflow.connection.celery.backend" -}}
+{{- define "DATABASE_CELERY_CMD" -}}
 {{- if .Values.postgresql.enabled -}}
-db+postgresql://${DATABASE_USER}:$( echo ${DATABASE_PASSWORD} | python3 -c "import urllib.parse; encoded_pass = urllib.parse.quote(input()); print(encoded_pass)" )@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB}
+echo -n "db+postgresql://${DATABASE_USER}:$(eval $DATABASE_PASSWORD_CMD)@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB}"
 {{- else if and (not .Values.postgresql.enabled) (eq "postgres" .Values.externalDatabase.type) -}}
-db+postgresql://${DATABASE_USER}:$( echo ${DATABASE_PASSWORD} | python3 -c "import urllib.parse; encoded_pass = urllib.parse.quote(input()); print(encoded_pass)" )@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB}${DATABASE_PROPERTIES}
+echo -n "db+postgresql://${DATABASE_USER}:$(eval $DATABASE_PASSWORD_CMD)@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB}${DATABASE_PROPERTIES}"
 {{- else if and (not .Values.postgresql.enabled) (eq "mysql" .Values.externalDatabase.type) -}}
-db+mysql://${DATABASE_USER}:$( echo ${DATABASE_PASSWORD} | python3 -c "import urllib.parse; encoded_pass = urllib.parse.quote(input()); print(encoded_pass)" )@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB}${DATABASE_PROPERTIES}
+echo -n "db+mysql://${DATABASE_USER}:$(eval $DATABASE_PASSWORD_CMD)@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB}${DATABASE_PROPERTIES}"
 {{- end -}}
 {{- end -}}
 
 {{/*
-Construct the AIRFLOW__CELERY__BROKER_URL connection string.
-Note that a redis URL with no password excludes the "@" prefix.
+Bash command which echos the Redis connection string.
+NOTE:
+ - used by `AIRFLOW__CELERY__BROKER_URL_CMD`
+ - the `REDIS_PASSWORD_CMD` sub-command is set in `configmap-env`
 */}}
-{{- define "airflow.connection.celery.broker" -}}
-redis://$( echo ${REDIS_PASSWORD} | python3 -c "import urllib.parse; encoded_pass = urllib.parse.quote(input()); print(f\":{encoded_pass}@\") if len(encoded_pass) > 0 else None" )${REDIS_HOST}:${REDIS_PORT}/${REDIS_DBNUM}
+{{- define "REDIS_CONNECTION_CMD" -}}
+echo -n "redis://$(eval $REDIS_PASSWORD_CMD)${REDIS_HOST}:${REDIS_PORT}/${REDIS_DBNUM}"
 {{- end -}}
 
 {{/*
@@ -139,7 +146,8 @@ When applicable, we use the secrets created by the postgres/redis charts (which
 {{- /* --------------------------- */ -}}
 {{- /* ---------- REDIS ---------- */ -}}
 {{- /* --------------------------- */ -}}
-{{- if and (.Values.redis.enabled) (eq .Values.airflow.executor "CeleryExecutor") }}
+{{- if eq .Values.airflow.executor "CeleryExecutor" }}
+{{- if .Values.redis.enabled }}
 {{- if .Values.redis.existingSecret }}
 - name: REDIS_PASSWORD
   valueFrom:
@@ -153,7 +161,7 @@ When applicable, we use the secrets created by the postgres/redis charts (which
       name: {{ include "airflow.redis.fullname" . }}
       key: redis-password
 {{- end }}
-{{- else if (eq .Values.airflow.executor "CeleryExecutor") }}
+{{- else }}
 {{- if .Values.externalRedis.passwordSecret }}
 - name: REDIS_PASSWORD
   valueFrom:
@@ -165,6 +173,7 @@ When applicable, we use the secrets created by the postgres/redis charts (which
   value: ""
 {{- end }}
 {{- end }}
+{{- end }}
 {{- /* ---------------------------- */ -}}
 {{- /* ---------- FLOWER ---------- */ -}}
 {{- /* ---------------------------- */ -}}

stable/airflow/templates/config/configmap-env.yaml

@@ -11,7 +11,9 @@ data:
   ## Force UTC timezone
   TZ: Etc/UTC
 
+  ## ----------------
   ## Database
+  ## ----------------
   {{- if .Values.postgresql.enabled }}
   DATABASE_HOST: "{{ include "airflow.postgresql.fullname" . }}"
   DATABASE_PORT: "5432"
@@ -22,11 +24,24 @@ data:
   DATABASE_PORT: "{{ .Values.externalDatabase.port }}"
   DATABASE_USER: "{{ .Values.externalDatabase.user }}"
   DATABASE_DB: "{{ .Values.externalDatabase.database }}"
-  DATABASE_PROPERTIES: "{{.Values.externalDatabase.properties}}"
+  DATABASE_PROPERTIES: |
+    {{ .Values.externalDatabase.properties }}
   {{- end }}
+  # bash command which echos the URL encoded value of $DATABASE_PASSWORD
+  DATABASE_PASSWORD_CMD: |
+    echo ${DATABASE_PASSWORD} | python3 -c "import urllib.parse; encoded_pass = urllib.parse.quote(input()); print(encoded_pass)"
+  # bash command which echos the DB connection string in SQLAlchemy format
+  DATABASE_SQLALCHEMY_CMD: |
+    {{ include "DATABASE_SQLALCHEMY_CMD" . }}
+  # bash command which echos the DB connection string in Celery result_backend format
+  DATABASE_CELERY_CMD: |
+    {{ include "DATABASE_CELERY_CMD" . }}
 
-  {{- if (eq .Values.airflow.executor "CeleryExecutor") }}
+
+  {{- if eq .Values.airflow.executor "CeleryExecutor" }}
+  ## ----------------
   ## Redis
+  ## ----------------
   {{- if .Values.redis.enabled }}
   REDIS_HOST: "{{ include "airflow.redis.fullname" . }}-master"
   REDIS_PORT: "6379"
@@ -36,51 +51,70 @@ data:
   REDIS_PORT: "{{ .Values.externalRedis.port }}"
   REDIS_DBNUM: "{{ .Values.externalRedis.databaseNumber }}"
   {{- end }}
+  # a bash command which echos the URL encoded value of $REDIS_PASSWORD
+  # NOTE: if $REDIS_PASSWORD is non-empty, prints `:${REDIS_PASSWORD}@`, else ``
+  REDIS_PASSWORD_CMD: |
+    echo ${REDIS_PASSWORD} | python3 -c "import urllib.parse; encoded_pass = urllib.parse.quote(input()); print(f\":{encoded_pass}@\") if len(encoded_pass) > 0 else None"
+  # a bash command which echos the Redis connection string
+  REDIS_CONNECTION_CMD: |
+    {{ include "REDIS_CONNECTION_CMD" . }}
+  {{- end }}
+
+  ## ----------------
+  ## Airflow
+  ## ----------------
+  AIRFLOW__CORE__BASE_LOG_FOLDER: "{{ .Values.logs.path }}"
+  AIRFLOW__CORE__DAGS_FOLDER: "{{ .Values.dags.path }}"
+  AIRFLOW__CORE__DAG_PROCESSOR_MANAGER_LOG_LOCATION: "{{ .Values.logs.path }}/dag_processor_manager/dag_processor_manager.log"
+  AIRFLOW__CORE__DONOT_PICKLE: "{{ .Values.dags.doNotPickle }}"
+  AIRFLOW__CORE__ENABLE_XCOM_PICKLING: "false" # for forward compatibility with 2.0
+  AIRFLOW__CORE__EXECUTOR: "{{ .Values.airflow.executor }}"
+  {{- if .Values.airflow.fernetKey }}
+  AIRFLOW__CORE__FERNET_KEY: "{{ .Values.airflow.fernetKey }}"
+  {{- end }}
+  AIRFLOW__CORE__SQL_ALCHEMY_CONN_CMD: |
+    bash -c 'eval "$DATABASE_SQLALCHEMY_CMD"'
+  {{- if .Values.web.serializeDAGs }}
+  AIRFLOW__CORE__STORE_SERIALIZED_DAGS: "{{ .Values.web.serializeDAGs }}"
+  {{- end }}
+  AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY: "{{ .Values.logs.path }}/scheduler"
+  AIRFLOW__WEBSERVER__BASE_URL: "{{ .Values.web.baseUrl }}"
+  AIRFLOW__WEBSERVER__WEB_SERVER_PORT: "8080"
+
+  {{- if (eq .Values.airflow.executor "CeleryExecutor") }}
+  ## ----------------
+  ## Airflow - CeleryExecutor
+  ## ----------------
+  AIRFLOW__CELERY__BROKER_URL_CMD: |
+    bash -c 'eval "$REDIS_CONNECTION_CMD"'
   {{- if .Values.flower.enabled }}
-  ## Airflow (Flower)
-  AIRFLOW__CELERY__FLOWER_URL_PREFIX: "{{ .Values.flower.urlPrefix }}"
   AIRFLOW__CELERY__FLOWER_PORT: "5555"
+  AIRFLOW__CELERY__FLOWER_URL_PREFIX: "{{ .Values.flower.urlPrefix }}"
   {{- end }}
-  ## Airflow (Worker)
+  AIRFLOW__CELERY__RESULT_BACKEND_CMD: |
+    bash -c 'eval "$DATABASE_CELERY_CMD"'
   AIRFLOW__CELERY__WORKER_CONCURRENCY: "{{ .Values.workers.celery.instances }}"
   AIRFLOW__CELERY__WORKER_LOG_SERVER_PORT: "8793"
   {{- end }}
 
   {{- if (eq .Values.airflow.executor "KubernetesExecutor") }}
-  ## Airflow (Kubernetes)
+  ## ----------------
+  ## Airflow - KubernetesExecutor
+  ## ----------------
+  {{- if (not .Values.airflow.config.AIRFLOW__KUBERNETES__ENV_FROM_CONFIGMAP_REF) }}
+  AIRFLOW__KUBERNETES__ENV_FROM_CONFIGMAP_REF: "{{ include "airflow.fullname" . }}-env"
+  {{- end }}
   {{- if (not .Values.airflow.config.AIRFLOW__KUBERNETES__NAMESPACE) }}
   AIRFLOW__KUBERNETES__NAMESPACE: "{{ .Release.Namespace }}"
   {{- end }}
   {{- if (not .Values.airflow.config.AIRFLOW__KUBERNETES__WORKER_SERVICE_ACCOUNT_NAME) }}
   AIRFLOW__KUBERNETES__WORKER_SERVICE_ACCOUNT_NAME: "{{ include "airflow.serviceAccountName" . }}"
   {{- end }}
-  {{- if (not .Values.airflow.config.AIRFLOW__KUBERNETES__ENV_FROM_CONFIGMAP_REF) }}
-  AIRFLOW__KUBERNETES__ENV_FROM_CONFIGMAP_REF: "{{ include "airflow.fullname" . }}-env"
-  {{- end }}
-  {{- end }}
-
-  ## Airflow (Logs)
-  AIRFLOW__CORE__BASE_LOG_FOLDER: "{{ .Values.logs.path }}"
-  AIRFLOW__CORE__DAG_PROCESSOR_MANAGER_LOG_LOCATION: "{{ .Values.logs.path }}/dag_processor_manager/dag_processor_manager.log"
-  AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY: "{{ .Values.logs.path }}/scheduler"
-
-  ## Airflow
-  AIRFLOW__CORE__DAGS_FOLDER: "{{ .Values.dags.path }}"
-  AIRFLOW__CORE__DONOT_PICKLE: "{{ .Values.dags.doNotPickle }}"
-  AIRFLOW__CORE__ENABLE_XCOM_PICKLING: "false" # for forward compatibility with 2.0
-  AIRFLOW__CORE__EXECUTOR: "{{ .Values.airflow.executor }}"
-  AIRFLOW__WEBSERVER__BASE_URL: "{{ .Values.web.baseUrl }}"
-  AIRFLOW__WEBSERVER__WEB_SERVER_PORT: "8080"
-  {{- if .Values.airflow.fernetKey }}
-  AIRFLOW__CORE__FERNET_KEY: "{{ .Values.airflow.fernetKey }}"
-  {{- end }}
-  {{- if .Values.web.serializeDAGs }}
-  AIRFLOW__CORE__STORE_SERIALIZED_DAGS: "{{ .Values.web.serializeDAGs }}"
   {{- end }}
 
   {{- if .Values.airflow.config }}
-  ## User Configs
-  {{- range $setting, $option := .Values.airflow.config }}
-  {{ $setting }}: "{{ $option }}"
-  {{- end }}
+  ## ----------------
+  ## Airflow - User Configs
+  ## ----------------
+  {{- toYaml .Values.airflow.config | nindent 2 }}
   {{- end }}

stable/airflow/templates/config/configmap-scripts.yaml

@@ -25,8 +25,8 @@ data:
     #!/bin/bash -e
     echo "*** starting graceful worker shutdown"
 
-    # source the required environment variables
-    source /home/airflow/airflow_env.sh
+    # set the required environment variables
+    export AIRFLOW__CELERY__BROKER_URL=$(eval $AIRFLOW__CELERY__BROKER_URL_CMD)
 
     # prevent the worker accepting new tasks
     echo "*** preventing worker accepting new tasks"
@@ -40,10 +40,7 @@ data:
     done
   preinit-db.sh: |
     #!/bin/bash
-    # export the required environment variables
-    export AIRFLOW__CORE__SQL_ALCHEMY_CONN="{{ include "airflow.connection.alchemy" . }}"
-
-    echo "*** Waiting 10s for postgres"
+    echo "*** Waiting 10s for database"
     sleep 10
 
     COUNT=0

stable/airflow/templates/config/configmap-variables-pools.yaml

@@ -2,12 +2,12 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-    name: {{ include "airflow.fullname" . }}-variables-pools
-    labels:
-        app: {{ include "airflow.labels.app" . }}
-        chart: {{ include "airflow.labels.chart" . }}
-        release: {{ .Release.Name }}
-        heritage: {{ .Release.Service }}
+  name: {{ include "airflow.fullname" . }}-variables-pools
+  labels:
+    app: {{ include "airflow.labels.app" . }}
+    chart: {{ include "airflow.labels.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
 data:
   {{- if .Values.scheduler.variables }}
   variables.json: |

stable/airflow/templates/flower/flower-deployment.yaml

@@ -107,13 +107,7 @@ spec:
                && sleep {{ .Values.flower.initialStartupDelay }} \
               {{- end }}
                && mkdir -p /home/airflow/.local/bin \
-               && echo 'export PATH="/home/airflow/.local/bin:$PATH"' >> /home/airflow/airflow_env.sh \
-               && echo 'export AIRFLOW__CORE__SQL_ALCHEMY_CONN="{{ include "airflow.connection.alchemy" . }}"' >> /home/airflow/airflow_env.sh \
-               && echo 'export AIRFLOW__CELERY__RESULT_BACKEND="{{ include "airflow.connection.celery.backend" . }}"' >> /home/airflow/airflow_env.sh \
-               && echo 'export AIRFLOW__CELERY__BROKER_URL="{{ include "airflow.connection.celery.broker" . }}"' >> /home/airflow/airflow_env.sh \
-               && echo 'source /home/airflow/airflow_env.sh' >> /home/airflow/.profile \
-               && echo 'source /home/airflow/airflow_env.sh' >> /home/airflow/.bashrc \
-               && source /home/airflow/airflow_env.sh \
+               && export PATH="/home/airflow/.local/bin:$PATH" \
                && echo "*** running flower..." \
                && exec airflow flower
           livenessProbe:

stable/airflow/templates/flower/flower-ingress.yaml

@@ -3,16 +3,16 @@ apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: {{ include "airflow.fullname" . }}-flower
+  {{- if .Values.ingress.flower.annotations }}
+  annotations:
+    {{- toYaml .Values.ingress.flower.annotations | nindent 4 }}
+  {{- end }}
   labels:
     app: {{ include "airflow.labels.app" . }}
     component: flower
-    chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
-    release: "{{ .Release.Name }}"
-    heritage: "{{ .Release.Service }}"
-  annotations:
-    {{- range $key, $value := .Values.ingress.flower.annotations }}
-    {{ $key }}: {{ $value | quote }}
-    {{- end }}
+    chart: {{ include "airflow.labels.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
 spec:
   {{- if .Values.ingress.flower.tls.enabled }}
   tls: