From f5eb614aa83d5a053f0444d57a5306cda8a14a88 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 31 Jul 2019 08:35:45 +0200 Subject: [PATCH] [AIRFLOW-5079] Checklicence test uses own, much smaller image --- Dockerfile-checklicence | 73 ++++++++++++++++++++++++++++++++++ hooks/build | 26 ++++++++++++ scripts/ci/_utils.sh | 21 ++++++++++ scripts/ci/ci_check_license.sh | 9 ++--- 4 files changed, 123 insertions(+), 6 deletions(-) create mode 100644 Dockerfile-checklicence diff --git a/Dockerfile-checklicence b/Dockerfile-checklicence new file mode 100644 index 0000000000000..7cb9b0e9dde0a --- /dev/null +++ b/Dockerfile-checklicence @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# WARNING: THIS DOCKERFILE IS NOT INTENDED FOR PRODUCTION USE OR DEPLOYMENT. +# +FROM debian:stretch-slim + +SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] + +# Make sure noninteractie debian install is used and language variables set +ENV DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \ + LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 + +ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ + +# Note missing man directories on debian-stretch +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 +# Install OpenJDK +RUN mkdir -pv /usr/share/man/man1 \ + && mkdir -pv /usr/share/man/man7 \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + sudo \ + curl \ + openjdk-8-jdk \ + && apt-get autoremove -yqq --purge \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +ARG RAT_VERSION="0.12" + +ENV RAT_VERSION="${RAT_VERSION}" \ + RAT_JAR="/tmp/apache-rat-${RAT_VERSION}.jar" \ + RAT_URL="http://repo1.maven.org/maven2/org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar" + +RUN echo "Downloading RAT from ${RAT_URL} to ${RAT_JAR}" \ + && curl -sL ${RAT_URL} > ${RAT_JAR} + +ARG AIRFLOW_USER=airflow +ENV AIRFLOW_USER=${AIRFLOW_USER} + +ARG HOME=/home/airflow +ENV HOME=${HOME} + +ARG AIRFLOW_HOME=${HOME}/airflow +ENV AIRFLOW_HOME=${AIRFLOW_HOME} + +ARG AIRFLOW_SOURCES=/opt/airflow +ENV AIRFLOW_SOURCES=${AIRFLOW_SOURCES} + +RUN adduser airflow \ + && echo "airflow ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/airflow \ + && chmod 0440 /etc/sudoers.d/airflow + +COPY --chown=airflow:airflow scripts ${AIRFLOW_SOURCES}/scripts + +USER airflow + +WORKDIR ${AIRFLOW_SOURCES} + +ENTRYPOINT ["/opt/airflow/scripts/ci/in_container/run_check_licence.sh"] diff --git a/hooks/build b/hooks/build index b9384053d6eed..00acd6fedca53 100755 --- a/hooks/build +++ b/hooks/build @@ -290,6 +290,10 @@ echo "Skip slim image: ${AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE}" export AIRFLOW_CONTAINER_SKIP_CI_IMAGE=${AIRFLOW_CONTAINER_SKIP_CI_IMAGE:="false"} echo "Skip CI image: ${AIRFLOW_CONTAINER_SKIP_CI_IMAGE}" +# Skips downloading and building the checklicence image - one used for checking the licences +export AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE=${AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE:="false"} +echo "Skip CHECKLICENCE image: ${AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE}" + # Skips pulling the airflow images - this will use cache but will build it all from scratch export AIRFLOW_CONTAINER_SKIP_PULLING_AIRFLOW_IMAGES=${AIRFLOW_CONTAINER_SKIP_PULLING_AIRFLOW_IMAGES:="false"} echo "Skip pulling Airflow images: ${AIRFLOW_CONTAINER_SKIP_PULLING_AIRFLOW_IMAGES}" @@ -318,12 +322,15 @@ if [[ "${AIRFLOW_RELEASE_BUILD}" == "true" ]]; then export AIRFLOW_CI_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_VERSION}-ci" export AIRFLOW_CI_IMAGE_LATEST="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:latest-python${PYTHON_VERSION}-${TAG_PREFIX}-ci" export AIRFLOW_CI_IMAGE_LATEST_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:latest-${TAG_PREFIX}-ci" + + export AIRFLOW_CHECKLICENCE_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:checklicence" else export AIRFLOW_SLIM_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${TAG_PREFIX}-python${PYTHON_VERSION}-ci-slim" export AIRFLOW_SLIM_CI_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${TAG_PREFIX}-ci-slim" export AIRFLOW_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${TAG_PREFIX}-python${PYTHON_VERSION}-ci" export AIRFLOW_CI_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${TAG_PREFIX}-ci" + export AIRFLOW_CHECKLICENCE_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:checklicence" fi # In the future we can enable buildkit. # It's experimental now and cache does not work out of the box with buildkit in Docker 18.09.2, buildkit 0.3.3 @@ -346,6 +353,7 @@ if [[ "${AIRFLOW_CONTAINER_CLEANUP_IMAGES}" == "true" ]]; then docker rmi "${PYTHON_BASE_IMAGE}" || true docker rmi "${AIRFLOW_SLIM_CI_IMAGE}" || true docker rmi "${AIRFLOW_CI_IMAGE}" || true + docker rmi "${AIRFLOW_CHECKLICENCE_IMAGE}" || true echo echo "###################################################################" echo "NOTE!! Removed Airflow images for Python version ${PYTHON_VERSION}." @@ -383,6 +391,11 @@ if [[ "${AIRFLOW_CONTAINER_USE_PULLED_IMAGES_CACHE}" == "true" ]]; then else IMAGES_TO_PULL="${IMAGES_TO_PULL} ${AIRFLOW_SLIM_CI_IMAGE}" fi + if [[ "${AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE}" == "true" ]]; then + echo "Skip downloading the CHECKLICENCE Airflow image" + else + IMAGES_TO_PULL="${IMAGES_TO_PULL} ${AIRFLOW_CHECKLICENCE_IMAGE}" + fi DOCKER_CACHE_DIRECTIVE_CI=() DOCKER_CACHE_DIRECTIVE_CI_SLIM=() @@ -428,6 +441,8 @@ if [[ "${AIRFLOW_CONTAINER_USE_PULLED_IMAGES_CACHE}" == "true" ]]; then DOCKER_CACHE_DIRECTIVE_CI+=("--cache-from" "${IMAGE}") elif [[ "${IMAGE}" == "${AIRFLOW_SLIM_CI_IMAGE}" ]]; then DOCKER_CACHE_DIRECTIVE_CI_SLIM+=("--cache-from" "${IMAGE}") + elif [[ "${IMAGE}" == "${AIRFLOW_CHECKLICENCE_IMAGE}" ]]; then + DOCKER_CACHE_DIRECTIVE_CHECKLICENCE+=("--cache-from" "${IMAGE}") else echo echo "Don't know how to set cache directive for ${IMAGE}. Exiting" @@ -584,6 +599,17 @@ else fi fi +if [[ "${AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE}" == "true" ]]; then + echo "Skip building the CHECKLICENCE image" +else + start_step "Build Airflow CHECKLICENCE image" + docker build . -f Dockerfile-checklicence -t ${AIRFLOW_CHECKLICENCE_IMAGE} + add_image_to_push "${AIRFLOW_CHECKLICENCE_IMAGE}" + + save_to_file AIRFLOW_CHECKLICENCE_IMAGE +fi + + start_step "Pushing images" if [[ "${AIRFLOW_CONTAINER_PUSH_IMAGES}" != "false" ]]; then diff --git a/scripts/ci/_utils.sh b/scripts/ci/_utils.sh index 98652eadce0bd..ca8768e15ad36 100644 --- a/scripts/ci/_utils.sh +++ b/scripts/ci/_utils.sh @@ -298,6 +298,7 @@ function force_python_3_5() { function rebuild_image_if_needed_for_static_checks() { export AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE="false" export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="true" + export AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE="true" export AIRFLOW_CONTAINER_PUSH_IMAGES="false" export AIRFLOW_CONTAINER_BUILD_NPM="false" # Skip NPM builds to make them faster ! @@ -359,6 +360,7 @@ EOF function rebuild_image_if_needed_for_tests() { export AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE="true" + export AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE="true" export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="false" PYTHON_VERSION=${PYTHON_VERSION:=$(python -c \ 'import sys; print("%s.%s" % (sys.version_info.major, sys.version_info.minor))')} @@ -417,6 +419,25 @@ EOF export AIRFLOW_CI_IMAGE } +function rebuild_image_for_checklicence() { + export AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE="true" + export AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE="false" + export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="true" + export AIRFLOW_CONTAINER_PUSH_IMAGES="false" + + export THE_IMAGE="CHECKLICENCE" + echo + echo "Rebuilding image" + echo + # shellcheck source=../../hooks/build + ./hooks/build | tee -a "${OUTPUT_LOG}" + update_all_md5_files + echo + echo "Image rebuilt" + echo + AIRFLOW_CHECKLICENCE_IMAGE=$(cat "${BUILD_CACHE_DIR}/.AIRFLOW_CHECKLICENCE_IMAGE") + export AIRFLOW_CHECKLICENCE_IMAGE +} # # Starts the script/ If VERBOSE variable is set to true, it enables verbose output of commands executed # Also prints some useful diagnostics information at start of the script diff --git a/scripts/ci/ci_check_license.sh b/scripts/ci/ci_check_license.sh index 68c2a4cf3009f..501cc1e697b6c 100755 --- a/scripts/ci/ci_check_license.sh +++ b/scripts/ci/ci_check_license.sh @@ -29,15 +29,12 @@ force_python_3_5 script_start -rebuild_image_if_needed_for_tests +rebuild_image_for_checklicence docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" -t \ - --entrypoint /opt/airflow/scripts/ci/in_container/run_check_licence.sh \ - --env PYTHONDONTWRITEBYTECODE="true" \ - --env AIRFLOW_CI_VERBOSE=${VERBOSE} \ - --env AIRFLOW_CI_VERBOSE=${VERBOSE} \ + --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ - "${AIRFLOW_CI_IMAGE}" \ + "${AIRFLOW_CHECKLICENCE_IMAGE}" script_end