Builds prod images on DockerHub from packages (#12908)

This build combines building both CI and PROD image in one
script execution on DockerHub per python version.

First the CI image is build and secondly, the image is used
to build all the packages from sources and then those
packages are used to build the PROD image.

Resulting image will be package image built from latest sources.

Closes: #12261
(cherry picked from commit f9e9ad2)

scripts/ci/images/ci_build_dockerhub.sh

@@ -16,11 +16,14 @@
 # specific language governing permissions and limitations
 # under the License.
 
+# shellcheck disable=SC2030,SC2031
+
 # This is hook build used by DockerHub. We are also using it
 # on CI to potentially rebuild (and refresh layers that
 # are not cached) Docker images that are used to run CI jobs
 export FORCE_ANSWER_TO_QUESTIONS="yes"
 export VERBOSE_COMMANDS="true"
+export VERBOSE="true"
 
 : "${DOCKER_REPO:?"ERROR: Please specify DOCKER_REPO variable following the pattern HOST/DOCKERHUB_USER/DOCKERHUB_REPO"}"
 
@@ -35,48 +38,57 @@ echo "DOCKERHUB_USER=${DOCKERHUB_USER}"
 echo "DOCKERHUB_REPO=${DOCKERHUB_REPO}"
 echo
 
-: "${DOCKER_TAG:?"ERROR: Please specify DOCKER_TAG variable following the pattern BRANCH-pythonX.Y[-ci]"}"
+: "${DOCKER_TAG:?"ERROR: Please specify DOCKER_TAG variable following the pattern BRANCH-pythonX.Y"}"
 
 echo "DOCKER_TAG=${DOCKER_TAG}"
 
-[[ ${DOCKER_TAG:=} =~ ${DEFAULT_BRANCH}-python([0-9.]*)(.*) ]] && export PYTHON_MAJOR_MINOR_VERSION=${BASH_REMATCH[1]}
+[[ ${DOCKER_TAG:=} =~ ${DEFAULT_BRANCH}-python([0-9.]*) ]] && export PYTHON_MAJOR_MINOR_VERSION=${BASH_REMATCH[1]}
 
-: "${PYTHON_MAJOR_MINOR_VERSION:?"The tag '${DOCKER_TAG}' should follow the pattern .*-pythonX.Y[-ci]"}"
+: "${PYTHON_MAJOR_MINOR_VERSION:?"The tag '${DOCKER_TAG}' should follow the pattern .*-pythonX.Y"}"
 
 echo "Detected PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION}"
 echo
 
-FORCE_AIRFLOW_PROD_BASE_TAG="${DOCKER_TAG}"
-export FORCE_AIRFLOW_PROD_BASE_TAG
-
-readonly FORCE_AIRFLOW_PROD_BASE_TAG
-
-if [[ "${FORCE_AIRFLOW_PROD_BASE_TAG}" =~ [0-9].* ]]; then
-    # Disable cache if we are building a tagged version
-    export DOCKER_CACHE="disabled"
-fi
-
-# shellcheck source=scripts/ci/libraries/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+(
+    export INSTALL_FROM_PYPI="true"
+    export INSTALL_FROM_DOCKER_CONTEXT_FILES="false"
+    export INSTALL_PROVIDERS_FROM_SOURCES="true"
+    export AIRFLOW_PRE_CACHED_PIP_PACKAGES="true"
+    export DOCKER_CACHE="pulled"
+    # shellcheck source=scripts/ci/libraries/_script_init.sh
+    . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
 
-if [[ ${DOCKER_TAG} == *python*-ci ]]; then
     echo
-    echo "Building CI image"
+    echo "Building and pushing CI image for ${PYTHON_MAJOR_MINOR_VERSION} in a sub-process"
     echo
     rm -rf "${BUILD_CACHE_DIR}"
     build_images::prepare_ci_build
     build_images::rebuild_ci_image_if_needed
-    push_pull_remove_images::push_ci_images
-elif [[ ${DOCKER_TAG} == *python* ]]; then
+    if [[ ! "${DOCKER_TAG}" =~ ^[0-9].* ]]; then
+        # Do not push if we are building a tagged version
+        push_pull_remove_images::push_ci_images
+    fi
+)
+
+(
+    export INSTALL_FROM_PYPI="false"
+    export INSTALL_FROM_DOCKER_CONTEXT_FILES="true"
+    export INSTALL_PROVIDERS_FROM_SOURCES="false"
+    export AIRFLOW_PRE_CACHED_PIP_PACKAGES="false"
+    export DOCKER_CACHE="pulled"
+
+    if [[ "${DOCKER_TAG}" =~ ^[0-9].* ]]; then
+        # Disable cache and set name of the tag as image name if we are building a tagged version
+        export DOCKER_CACHE="disabled"
+        export FORCE_AIRFLOW_PROD_BASE_TAG="${DOCKER_TAG}"
+    fi
+    # shellcheck source=scripts/ci/libraries/_script_init.sh
+    . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
     echo
-    echo "Building prod image"
+    echo "Building and pushing PROD image for ${PYTHON_MAJOR_MINOR_VERSION} in a sub-process"
     echo
     rm -rf "${BUILD_CACHE_DIR}"
     build_images::prepare_prod_build
-    build_images::build_prod_images
+    build_images::build_prod_images_from_packages
     push_pull_remove_images::push_prod_images
-else
-    echo
-    echo "Skipping the build in Dockerhub. The tag is not good: ${DOCKER_TAG}"
-    echo
-fi
+)

scripts/ci/images/ci_prepare_prod_image_on_ci.sh

@@ -15,6 +15,15 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+
+export INSTALL_FROM_PYPI="false"
+export INSTALL_FROM_DOCKER_CONTEXT_FILES="true"
+export INSTALL_PROVIDERS_FROM_SOURCES="false"
+export AIRFLOW_PRE_CACHED_PIP_PACKAGES="false"
+export DOCKER_CACHE="local"
+export VERBOSE="true"
+
+
 # shellcheck source=scripts/ci/libraries/_script_init.sh
 . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
 
@@ -23,9 +32,6 @@
 function build_prod_images_on_ci() {
     build_images::prepare_prod_build
 
-    rm -rf "${BUILD_CACHE_DIR}"
-    mkdir -pv "${BUILD_CACHE_DIR}"
-
     if [[ ${USE_GITHUB_REGISTRY} == "true" && ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} == "true" ]]; then
 
         # Tries to wait for the image indefinitely
@@ -34,10 +40,8 @@ function build_prod_images_on_ci() {
         build_images::wait_for_image_tag "${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}" \
             ":${GITHUB_REGISTRY_PULL_IMAGE_TAG}" "${AIRFLOW_PROD_IMAGE}"
 
-        build_images::wait_for_image_tag "${GITHUB_REGISTRY_AIRFLOW_PROD_BUILD_IMAGE}" \
-            ":${GITHUB_REGISTRY_PULL_IMAGE_TAG}" "${AIRFLOW_PROD_BUILD_IMAGE}"
     else
-        build_images::build_prod_images
+        build_images::build_prod_images_from_packages
     fi
 
 

scripts/ci/libraries/_build_images.sh

@@ -856,6 +856,39 @@ function build_images::determine_docker_cache_strategy() {
 }
 
 
+function build_images::build_prod_images_from_packages() {
+    # Cleanup dist and docker-context-files folders
+    mkdir -pv "${AIRFLOW_SOURCES}/dist"
+    mkdir -pv "${AIRFLOW_SOURCES}/docker-context-files"
+    rm -f "${AIRFLOW_SOURCES}/dist/"*.{whl,tar.gz}
+    rm -f "${AIRFLOW_SOURCES}/docker-context-files/"*.{whl,tar.gz}
+
+    pip_download_command="pip download -d /dist '.[${INSTALLED_EXTRAS}]' --constraint 'https://raw.githubusercontent.com/apache/airflow/${DEFAULT_CONSTRAINTS_BRANCH}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt'"
+
+    # Download all dependencies needed
+    docker run --rm --entrypoint /bin/bash \
+        "${EXTRA_DOCKER_FLAGS[@]}" \
+        "${AIRFLOW_CI_IMAGE}" -c "${pip_download_command}"
+
+    # Remove all downloaded apache airflow packages
+    rm -f "${AIRFLOW_SOURCES}/dist/"apache_airflow*.whl
+    rm -f "${AIRFLOW_SOURCES}/dist/"apache-airflow*.tar.gz
+
+    # Remove all downloaded apache airflow packages
+    mv -f "${AIRFLOW_SOURCES}/dist/"* "${AIRFLOW_SOURCES}/docker-context-files/"
+
+    # Build apache airflow packages
+    build_airflow_packages::build_airflow_packages
+
+    # Remove generated tar.gz packages
+    rm -f "${AIRFLOW_SOURCES}/dist/"apache-airflow*.tar.gz
+
+    # move the packages to docker-context-files folder
+    mkdir -pv "${AIRFLOW_SOURCES}/docker-context-files"
+    mv "${AIRFLOW_SOURCES}/dist/"* "${AIRFLOW_SOURCES}/docker-context-files/"
+    build_images::build_prod_images
+}
+
 # Useful information for people who stumble upon a pip check failure
 function build_images::inform_about_pip_check() {
         echo """

scripts/ci/libraries/_initialization.sh

@@ -144,6 +144,15 @@ function initialization::initialize_base_variables() {
     # If no Airflow Home defined - fallback to ${HOME}/airflow
     AIRFLOW_HOME_DIR=${AIRFLOW_HOME:=${HOME}/airflow}
     export AIRFLOW_HOME_DIR
+
+    export INSTALLED_EXTRAS="async,amazon,celery,kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,imap,google,azure,mysql,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv"
+    readonly INSTALLED_EXTRAS
+
+    PIP_VERSION="20.2.4"
+    export PIP_VERSION
+
+    WHEEL_VERSION="0.35.1"
+    export WHEEL_VERSION
 }
 
 # Determine current branch