[AIRFLOW-5829] Get rid of the checklicence image.

.travis.yml

@@ -32,17 +32,13 @@ stages:
   - test
 jobs:
   include:
-    - name: "Static checks (no pylint, no license check)"
-      stage: pre-test
-      script: ./scripts/ci/ci_run_all_static_tests_except_pylint_licence.sh
-    - name: "Check license compliance for Apache"
+    - name: "Static checks"
+      python: "3.5"
       stage: pre-test
-      script: ./scripts/ci/ci_check_license.sh
+      script: ./scripts/ci/ci_run_all_static_tests.sh
       env: AIRFLOW_MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS="true"
-    - name: "Pylint checks"
-      stage: pre-test
-      script: ./scripts/ci/ci_run_all_static_tests_pylint.sh
     - name: "Build documentation"
+      python: "3.5"
       stage: pre-test
       script: ./scripts/ci/ci_docs.sh
     - name: "Tests postgres kubernetes python 3.6 (persistent)"

BREEZE.rst

@@ -84,11 +84,6 @@ There are three images that we are currently managing:
   Its tag follows the pattern of ``<BRANCH>-python<PYTHON_VERSION>-ci``
   (for example, ``apache/airflow:master-python3.6-ci``). The image is built using the
   `<Dockerfile>`_ Dockerfile.
-* **Checklicense image** that is used during license check with the Apache RAT tool. It does not
-  require any of the dependencies that the two CI images need so it is built using a different Dockerfile
-  `<Dockerfile-checklicence>`_ and only contains Java + Apache RAT tool. The image is
-  labelled with ``checklicence``, for example: ``apache/airflow:checklicence``. No versioning is used for
-  the Checklicence image.
 
 Before you run tests, enter the environment or run local static checks, the necessary local images should be
 pulled and built from Docker Hub. This happens automatically for the test environment but you need to

Dockerfile

@@ -144,11 +144,11 @@ RUN mkdir -pv /usr/share/man/man1 \
     && rm -rf /var/lib/apt/lists/*
 
 ENV HADOOP_DISTRO="cdh" HADOOP_MAJOR="5" HADOOP_DISTRO_VERSION="5.11.0" HADOOP_VERSION="2.6.0" \
-    HADOOP_HOME="/tmp/hadoop-cdh"
-ENV HIVE_VERSION="1.1.0" HIVE_HOME="/tmp/hive"
+    HADOOP_HOME="/opt/hadoop-cdh"
+ENV HIVE_VERSION="1.1.0" HIVE_HOME="/opt/hive"
 ENV HADOOP_URL="https://archive.cloudera.com/${HADOOP_DISTRO}${HADOOP_MAJOR}/${HADOOP_DISTRO}/${HADOOP_MAJOR}/"
 ENV MINICLUSTER_BASE="https://github.com/bolkedebruin/minicluster/releases/download/" \
-    MINICLUSTER_HOME="/tmp/minicluster" \
+    MINICLUSTER_HOME="/opt/minicluster" \
     MINICLUSTER_VER="1.1"
 
 RUN mkdir -pv "${HADOOP_HOME}" \
@@ -176,10 +176,10 @@ ENV MINICLUSTER_URL="${MINICLUSTER_BASE}${MINICLUSTER_VER}/minicluster-${MINICLU
     MINICLUSTER_TMP_FILE="/tmp/minicluster.zip"
 
 RUN curl -sL "${MINICLUSTER_URL}" > "${MINICLUSTER_TMP_FILE}" \
-    && unzip "${MINICLUSTER_TMP_FILE}" -d "/tmp" \
+    && unzip "${MINICLUSTER_TMP_FILE}" -d "/opt" \
     && rm "${MINICLUSTER_TMP_FILE}"
 
-ENV PATH "${PATH}:/tmp/hive/bin"
+ENV PATH "${PATH}:/opt/hive/bin"
 
 RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \
     && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian stretch stable" \
@@ -203,6 +203,20 @@ RUN curl -Lo kind \
    && chmod +x kind \
    && mv kind /usr/local/bin/kind
 
+ARG RAT_VERSION="0.13"
+
+ENV RAT_VERSION="${RAT_VERSION}" \
+    RAT_JAR="/opt/apache-rat-${RAT_VERSION}.jar" \
+    RAT_URL="https://repo1.maven.org/maven2/org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar"
+ENV RAT_JAR_MD5="${RAT_JAR}.md5" \
+    RAT_URL_MD5="${RAT_URL}.md5"
+
+RUN echo "Downloading RAT from ${RAT_URL} to ${RAT_JAR}" \
+    && curl -sL "${RAT_URL}" > "${RAT_JAR}" \
+    && curl -sL "${RAT_URL_MD5}" > "${RAT_JAR_MD5}" \
+    && jar -tf "${RAT_JAR}" \
+    && md5sum -c <<<"$(cat "${RAT_JAR_MD5}") ${RAT_JAR}"
+
 ARG AIRFLOW_USER=airflow
 ENV AIRFLOW_USER=${AIRFLOW_USER}
 

breeze

@@ -932,7 +932,6 @@ prepare_command_file "${BUILD_CACHE_DIR}/${LAST_DC_TEST_FILE}" "${DC_RUN_COMMAND
 prepare_command_file "${BUILD_CACHE_DIR}/${LAST_DC_FILE}" '"' "false"
 
 rebuild_ci_image_if_needed
-rebuild_checklicence_image_if_needed
 
 export AIRFLOW_CONTAINER_DOCKER_IMAGE=\
 ${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CONTAINER_BRANCH_NAME}-python${PYTHON_VERSION}-ci

common/_files_for_rebuild_check.sh

@@ -21,7 +21,6 @@ FILES_FOR_REBUILD_CHECK=(
  "setup.py"
  "setup.cfg"
  "Dockerfile"
- "Dockerfile-checklicence"
  ".dockerignore"
  "airflow/version.py"
  "airflow/www/package.json"

hooks/build

@@ -115,7 +115,7 @@ function build_python_image {
         --build-arg AIRFLOW_BRANCH="${AIRFLOW_CONTAINER_BRANCH_NAME}" \
         --build-arg AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD="${AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD}" \
         --build-arg HOME="${HOME}" \
-        "${DOCKER_CACHE_DIRECTIVE[@]}" \
+        "${DOCKER_CACHE_DIRECTIVE_CI[@]}" \
         -t "${MY_IMAGE_TAG}" \
         --target "${TARGET_IMAGE}" \
         .
@@ -202,10 +202,6 @@ export AIRFLOW_CONTAINER_SKIP_LATEST_PYTHON_PULL=${AIRFLOW_CONTAINER_SKIP_LATEST
 export AIRFLOW_CONTAINER_SKIP_CI_IMAGE=${AIRFLOW_CONTAINER_SKIP_CI_IMAGE:="false"}
 echo "Skip CI image: ${AIRFLOW_CONTAINER_SKIP_CI_IMAGE}"
 
-# Skips downloading and building the checklicence image - one used for checking the licences
-export AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE=${AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE:="false"}
-echo "Skip CHECKLICENCE image: ${AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE}"
-
 # Fixes permissions for git-checked out files. This is needed to have consistent build cache across
 # Dockerhub, TravisCI and locally checked out code
 export AIRFLOW_FIX_PERMISSIONS=${AIRFLOW_FIX_PERMISSIONS:="all"}
@@ -217,20 +213,14 @@ echo "The build optimised for CI: ${AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD}"
 # Base python image for the build
 export PYTHON_BASE_IMAGE=python:${PYTHON_VERSION}-slim-stretch
 
-# Base image for checklicence image
-export CHECKLICENCE_BASE_IMAGE=debian:stretch-slim
-
 if [[ "${AIRFLOW_RELEASE_BUILD}" == "true" ]]; then
     export AIRFLOW_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_VERSION}-python${PYTHON_VERSION}-ci"
     export AIRFLOW_CI_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_VERSION}-ci"
     export AIRFLOW_CI_IMAGE_LATEST="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:latest-python${PYTHON_VERSION}-${TAG_PREFIX}-ci"
     export AIRFLOW_CI_IMAGE_LATEST_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:latest-${TAG_PREFIX}-ci"
-
-    export AIRFLOW_CHECKLICENCE_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:checklicence"
 else
     export AIRFLOW_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${TAG_PREFIX}-python${PYTHON_VERSION}-ci"
     export AIRFLOW_CI_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${TAG_PREFIX}-ci"
-    export AIRFLOW_CHECKLICENCE_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:checklicence"
 fi
 # In the future we can enable buildkit.
 # It's experimental now and cache does not work out of the box with buildkit in Docker 18.09.2, buildkit 0.3.3
@@ -251,9 +241,7 @@ if [[ "${AIRFLOW_CONTAINER_CLEANUP_IMAGES}" == "true" ]]; then
     echo
     start_step "Removing images"
     docker rmi "${PYTHON_BASE_IMAGE}" || true
-    docker rmi "${CHECKLICENCE_BASE_IMAGE}" || true
     docker rmi "${AIRFLOW_CI_IMAGE}" || true
-    docker rmi "${AIRFLOW_CHECKLICENCE_IMAGE}" || true
     echo
     echo "###################################################################"
     echo "NOTE!! Removed Airflow images for Python version ${PYTHON_VERSION}."
@@ -268,7 +256,6 @@ fi
 start_step "Populating cache"
 
 DOCKER_CACHE_DIRECTIVE_CI=()
-DOCKER_CACHE_DIRECTIVE_CHECKLICENCE=()
 
 if [[ "${AIRFLOW_CONTAINER_USE_PULLED_IMAGES_CACHE}" == "true" ]]; then
     echo
@@ -286,9 +273,6 @@ if [[ "${AIRFLOW_CONTAINER_USE_PULLED_IMAGES_CACHE}" == "true" ]]; then
             echo "Force pull base python image."
             echo
             docker pull "${PYTHON_BASE_IMAGE}"
-            echo "Force pull base checklicence image."
-            echo
-            docker pull "${CHECKLICENCE_BASE_IMAGE}"
             echo
             set +x
         fi
@@ -298,10 +282,6 @@ if [[ "${AIRFLOW_CONTAINER_USE_PULLED_IMAGES_CACHE}" == "true" ]]; then
         echo "Downloading the CI Airflow image"
         IMAGES_TO_PULL="${IMAGES_TO_PULL} ${AIRFLOW_CI_IMAGE}"
     fi
-    if [[ "${AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE}" != "true" ]]; then
-        echo "Downloading the CHECKLICENCE Airflow image"
-        IMAGES_TO_PULL="${IMAGES_TO_PULL} ${AIRFLOW_CHECKLICENCE_IMAGE}"
-    fi
 
     DOCKER_CACHE_DIRECTIVE_CI=()
     if [[ ${IMAGES_TO_PULL} == "" ]]; then
@@ -343,8 +323,6 @@ if [[ "${AIRFLOW_CONTAINER_USE_PULLED_IMAGES_CACHE}" == "true" ]]; then
             fi
             if [[ "${IMAGE}" == "${AIRFLOW_CI_IMAGE}" ]]; then
                 DOCKER_CACHE_DIRECTIVE_CI+=("--cache-from" "${IMAGE}")
-            elif [[ "${IMAGE}" == "${AIRFLOW_CHECKLICENCE_IMAGE}" ]]; then
-                DOCKER_CACHE_DIRECTIVE_CHECKLICENCE+=("--cache-from" "${IMAGE}")
             else
                 echo
                 echo "Don't know how to set cache directive for ${IMAGE}. Exiting"
@@ -359,33 +337,28 @@ start_step "Setting cache options"
 
 if [[ "${AIRFLOW_CONTAINER_USE_NO_CACHE}" == "true" ]]; then
     DOCKER_CACHE_DIRECTIVE_CI+=("--no-cache")
-    DOCKER_CACHE_DIRECTIVE_CHECKLICENCE+=("--no-cache")
     echo
     echo "Skip cache for builds. Everything will be rebuilt from scratch."
     echo
     echo "Cache directives used: "
     echo "CI build: ${DOCKER_CACHE_DIRECTIVE_CI[*]}"
-    echo "CI checklicence build: ${DOCKER_CACHE_DIRECTIVE_CHECKLICENCE[*]}"
     echo
 elif [[ "${AIRFLOW_CONTAINER_USE_PULLED_IMAGES_CACHE}" == "true" ]]; then
     echo
     echo "This build uses Docker cache from pulled images"
     echo "Cache directives used: "
     set +u
     echo "CI build: ${DOCKER_CACHE_DIRECTIVE_CI[*]}"
-    echo "CI checklicence build: ${DOCKER_CACHE_DIRECTIVE_CHECKLICENCE[*]}"
     set -u
     echo
 else
     DOCKER_CACHE_DIRECTIVE_CI=()
-    DOCKER_CACHE_DIRECTIVE_CHECKLICENCE=()
     echo
     echo "Use default cache from locally built images."
     echo
     echo "Cache directives used: "
     set +u
     echo "CI build: ${DOCKER_CACHE_DIRECTIVE_CI[*]}"
-    echo "CI checklicence build: ${DOCKER_CACHE_DIRECTIVE_CHECKLICENCE[*]}"
     set -u
     echo
 fi
@@ -466,18 +439,6 @@ else
     fi
 fi
 
-if [[ "${AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE}" == "true" ]]; then
-    echo "Skip building the CHECKLICENCE image"
-else
-    start_step "Build Airflow CHECKLICENCE image"
-    set +u
-    docker build . -f Dockerfile-checklicence \
-        "${DOCKER_CACHE_DIRECTIVE_CHECKLICENCE[@]}" -t "${AIRFLOW_CHECKLICENCE_IMAGE}"
-    set -u
-    add_image_to_push "${AIRFLOW_CHECKLICENCE_IMAGE}"
-fi
-
-
 start_step "Pushing images"
 
 if [[ "${AIRFLOW_CONTAINER_PUSH_IMAGES}" != "false" ]]; then