Helm Docker image sources are now included in the Airlfow codebase (#…

…9650)

We can now build all the images from Airlfow sources in
a reproducible fashion and our users can use the helm chart
based on the images build from official images + code in
Airflow Codebase.

We also have consistent versioning scheme based on
calver version of releasing the images coupled with
the version of the original package.

Part of #9401

(cherry picked from commit c6e6d6d)

chart/dockerfiles/README.md

@@ -0,0 +1,28 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+
+Those are images that are needed for the Helm Chart.
+
+In each of the images you can find "build_and_push.sh" script that builds and pushes the image.
+
+You need to be a PMC with direct push access to "apache/airflow" DockerHub registry
+to be able to push to the Airflow DockerHub registry.
+
+You can set the DOCKERHUB_USER variable to push to your own DockerHub user if you want
+ to test the image or build your own image.

chart/dockerfiles/pgbouncer-exporter/README.md

@@ -0,0 +1,21 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+
+The dockerfile and build script for pgbouncer-exporter can be found
+in https://github.com/apache/airflow-pgbouncer-exporter

chart/dockerfiles/pgbouncer/Dockerfile

@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+ARG ALPINE_VERSION="3.12"
+
+FROM alpine:${ALPINE_VERSION} AS builder
+
+ARG PGBOUNCER_VERSION
+ARG AIRFLOW_PGBOUNCER_VERSION
+
+ARG PGBOUNCER_SHA256="a0c13d10148f557e36ff7ed31793abb7a49e1f8b09aa2d4695d1c28fa101fee7"
+
+RUN apk --no-cache add make pkgconfig build-base libtool wget gcc g++ libevent-dev libressl-dev c-ares-dev ca-certificates
+RUN wget https://github.com/pgbouncer/pgbouncer/releases/download/pgbouncer_$(echo ${PGBOUNCER_VERSION} | sed s/\\./_/g)/pgbouncer-${PGBOUNCER_VERSION}.tar.gz
+RUN echo "${PGBOUNCER_SHA256}  pgbouncer-${PGBOUNCER_VERSION}.tar.gz" | sha256sum -c -
+RUN tar -xzvf pgbouncer-$PGBOUNCER_VERSION.tar.gz
+WORKDIR pgbouncer-$PGBOUNCER_VERSION
+RUN ./configure --prefix=/usr --disable-debug && make && make install
+RUN mkdir /etc/pgbouncer
+RUN cp ./etc/pgbouncer.ini /etc/pgbouncer/
+RUN touch /etc/pgbouncer/userlist.txt
+RUN sed -i -e "s|logfile = |#logfile = |"  \
+           -e "s|pidfile = |#pidfile = |"  \
+           -e "s|listen_addr = .*|listen_addr = 0.0.0.0|" \
+           -e "s|auth_type = .*|auth_type = md5|" \
+           /etc/pgbouncer/pgbouncer.ini
+
+FROM alpine:${ALPINE_VERSION}
+
+ARG PGBOUNCER_VERSION
+ARG AIRFLOW_PGBOUNCER_VERSION
+ARG COMMIT_SHA
+
+LABEL org.apache.airflow.component="pgbouncer"
+LABEL org.apache.airflow.pgbouncer.version="${PGBOUNCER_VERSION}"
+LABEL org.apache.airflow.airflow_pgbouncer.version="${AIRFLOW_PGBOUNCER_VERSION}"
+LABEL org.apache.airflow.commit_sha="${COMMIT_SHA}"
+
+MAINTAINER "Apache Airflow Community <dev@airflow.apache.org>"
+
+RUN apk --no-cache add libevent libressl c-ares
+
+COPY --from=builder /etc/pgbouncer /etc/pgbouncer
+COPY --from=builder /usr/bin/pgbouncer /usr/bin/pgbouncer
+
+# Healthcheck
+HEALTHCHECK --interval=10s --timeout=3s CMD stat /tmp/.s.PGSQL.*
+
+EXPOSE 6432
+
+# pgbouncer can't run as root, so let's drop to 'nobody'
+ENTRYPOINT ["/usr/bin/pgbouncer", "-u", "nobody", "/etc/pgbouncer/pgbouncer.ini" ]

chart/dockerfiles/pgbouncer/build_and_push.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+set -euo pipefail
+DOCKERHUB_USER=${DOCKERHUB_USER:="apache"}
+DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow"}
+PGBOUNCER_VERSION="1.14.0"
+AIRFLOW_PGBOUNCER_VERSION="2020.07.10"
+COMMIT_SHA=$(git rev-parse HEAD)
+
+cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1
+
+TAG="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:airflow-pgbouncer-${AIRFLOW_PGBOUNCER_VERSION}-${PGBOUNCER_VERSION}"
+
+docker build . \
+    --pull \
+    --build-arg "PGBOUNCER_VERSION=${PGBOUNCER_VERSION}" \
+    --build-arg "AIRFLOW_PGBOUNCER_VERSION=${AIRFLOW_PGBOUNCER_VERSION}"\
+    --build-arg "COMMIT_SHA=${COMMIT_SHA}" \
+    --tag "${TAG}"
+
+docker push "${TAG}"

chart/dockerfiles/statsd-exporter/Dockerfile

@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+ARG STATSD_VERSION="missing_version"
+
+FROM prom/statsd-exporter:${STATSD_VERSION}
+
+ARG STATSD_VERSION
+ARG AIRFLOW_STATSD_EXPORTER_VERSION
+ARG COMMIT_SHA
+
+LABEL org.apache.airflow.component="statsd-exporter"
+LABEL org.apache.airflow.stasd.version="${STATSD_VERSION}"
+LABEL org.apache.airflow.airflow_stasd_exporter.version="${AIRFLOW_STATSD_EXPORTER_VERSION}"
+LABEL org.apache.airflow.commit_sha="${COMMIT_SHA}"
+
+MAINTAINER "Apache Airflow Community <dev@airflow.apache.org>"
+
+COPY mappings.yml /etc/statsd-exporter/mappings.yml

chart/dockerfiles/statsd-exporter/build_and_push.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+set -euo pipefail
+DOCKERHUB_USER=${DOCKERHUB_USER:="apache"}
+DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow"}
+STATSD_VERSION="v0.17.0"
+AIRFLOW_STATSD_EXPORTER_VERSION="2020.07.10"
+COMMIT_SHA=$(git rev-parse HEAD)
+
+cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1
+
+TAG="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:airflow-statsd-exporter-${AIRFLOW_STATSD_EXPORTER_VERSION}-${STATSD_VERSION}"
+
+docker build . \
+    --pull \
+    --build-arg "STATSD_VERSION=${STATSD_VERSION}" \
+    --build-arg "AIRFLOW_STATSD_EXPORTER_VERSION=${AIRFLOW_STATSD_EXPORTER_VERSION}" \
+    --build-arg "COMMIT_SHA=${COMMIT_SHA}" \
+    --tag "${TAG}"
+
+docker push "${TAG}"

chart/dockerfiles/statsd-exporter/mappings.yml

@@ -0,0 +1,87 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+mappings:
+  # Map dot separated stats to labels
+  - match: airflow.dagrun.dependency-check.*.*
+    name: "airflow_dagrun_dependency_check"
+    labels:
+      dag_id: "$1"
+
+  - match: airflow.operator_successes_(.*)
+    match_type: regex
+    name: "airflow_operator_successes"
+    labels:
+      operator: "$1"
+
+  - match: airflow.operator_failures_(.*)
+    match_type: regex
+    name: "airflow_operator_failures"
+    labels:
+      operator: "$1"
+
+  - match: airflow.scheduler_heartbeat
+    match_type: regex
+    name: "airflow_scheduler_heartbeat"
+    labels:
+      type: counter
+
+  - match: airflow.dag.*.*.duration
+    name: "airflow_task_duration"
+    labels:
+      dag_id: "$1"
+      task_id: "$2"
+
+  - match: airflow.dagrun.duration.success.*
+    name: "airflow_dagrun_duration"
+    labels:
+      dag_id: "$1"
+
+  - match: airflow.dagrun.duration.failed.*
+    name: "airflow_dagrun_failed"
+    labels:
+      dag_id: "$1"
+
+  - match: airflow.dagrun.schedule_delay.*
+    name: "airflow_dagrun_schedule_delay"
+    labels:
+      dag_id: "$1"
+
+  - match: airflow.dag_processing.last_runtime.*
+    name: "airflow_dag_processing_last_runtime"
+    labels:
+      dag_file: "$1"
+
+  - match: airflow.dag_processing.last_run.seconds_ago.*
+    name: "airflow_dag_processing_last_run_seconds_ago"
+    labels:
+      dag_file: "$1"
+
+  - match: airflow.pool.open_slots.*
+    name: "airflow_pool_open_slots"
+    labels:
+      pool: "$1"
+
+  - match: airflow.pool.used_slots.*
+    name: "airflow_pool_used_slots"
+    labels:
+      pool: "$1"
+
+  - match: airflow.pool.starving_tasks.*
+    name: "airflow_pool_starving_tasks"
+    labels:
+      pool: "$1"

chart/values.yaml

@@ -75,20 +75,20 @@ images:
     tag: ~
     pullPolicy: IfNotPresent
   statsd:
-    repository: astronomerinc/ap-statsd-exporter
-    tag: 0.11.0
+    repository: apache/airflow
+    tag: airflow-statsd-exporter-2020.07.10-v0.17.0
     pullPolicy: IfNotPresent
   redis:
     repository: redis
     tag: 6-buster
     pullPolicy: IfNotPresent
   pgbouncer:
-    repository: astronomerinc/ap-pgbouncer
-    tag: 1.8.1
+    repository: apache/airflow
+    tag: airflow-pgbouncer-2020.07.10-1.14.0
     pullPolicy: IfNotPresent
   pgbouncerExporter:
-    repository: astronomerinc/ap-pgbouncer-exporter
-    tag: 0.5.0-1
+    repository: apache/airflow
+    tag: airflow-pgbouncer-exporter-2020.07.10-0.5.0
     pullPolicy: IfNotPresent
 
 # Environment variables for all airflow containers

scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh

@@ -17,7 +17,6 @@
 # under the License.
 # shellcheck source=scripts/ci/libraries/_script_init.sh
 . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-
 set -euo pipefail
 
 export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:="3.6"}