[AIRFLOW-5842] Swtch to Debian buster image as a base (#7647)

(cherry picked from commit cad20c2)

.travis.yml

@@ -69,6 +69,7 @@ jobs:
         BACKEND=postgres
         PYTHON_VERSION=2.7
         ENABLE_KIND_CLUSTER=true
+        RUNTIME=kubernetes
         KUBERNETES_MODE=persistent_mode
         KUBERNETES_VERSION=v1.15.3
       stage: test
@@ -77,6 +78,7 @@ jobs:
         BACKEND=postgres
         PYTHON_VERSION=2.7
         ENABLE_KIND_CLUSTER=true
+        RUNTIME=kubernetes
         KUBERNETES_MODE=git_mode
         KUBERNETES_VERSION=v1.15.3
       stage: test

Dockerfile

@@ -15,12 +15,12 @@
 #
 # WARNING: THIS DOCKERFILE IS NOT INTENDED FOR PRODUCTION USE OR DEPLOYMENT.
 #
-ARG PYTHON_BASE_IMAGE="python:3.6-slim-stretch"
+ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
 FROM ${PYTHON_BASE_IMAGE} as main
 
 SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
 
-ARG PYTHON_BASE_IMAGE="python:3.6-slim-stretch"
+ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
 ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE}
 
 ARG AIRFLOW_VERSION="2.0.0.dev0"
@@ -61,6 +61,7 @@ RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \
            freetds-bin \
            freetds-dev \
            git \
+           graphviz \
            gosu \
            libffi-dev \
            libkrb5-dev \
@@ -80,14 +81,6 @@ RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-# Install graphviz - needed to build docs with diagrams
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends \
-           graphviz \
-    && apt-get autoremove -yqq --purge \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-
 # Install MySQL client from Oracle repositories (Debian installs mariadb)
 RUN KEY="A4A9406876FCBD3C456770C88C718D3B5072E1F5" \
     && GNUPGHOME="$(mktemp -d)" \
@@ -116,15 +109,16 @@ RUN adduser airflow \
     && echo "airflow ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/airflow \
     && chmod 0440 /etc/sudoers.d/airflow
 
-ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
-
-# Note missing man directories on debian-stretch
+# Note missing man directories on debian-buster
 # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199
 RUN mkdir -pv /usr/share/man/man1 \
     && mkdir -pv /usr/share/man/man7 \
+    && echo "deb http://ftp.us.debian.org/debian sid main" \
+        > /etc/apt/sources.list.d/openjdk.list \
     && apt-get update \
     && apt-get install --no-install-recommends -y \
       gnupg \
+      openjdk-8-jdk \
       apt-transport-https \
       bash-completion \
       ca-certificates \
@@ -134,7 +128,6 @@ RUN mkdir -pv /usr/share/man/man1 \
       less \
       lsb-release \
       net-tools \
-      openjdk-8-jdk \
       openssh-client \
       openssh-server \
       postgresql-client \
@@ -147,6 +140,7 @@ RUN mkdir -pv /usr/share/man/man1 \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
+ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
 
 # Install Hadoop and Hive
 # It is done in one step to share variables.

breeze

@@ -436,7 +436,7 @@ function prepare_command_files() {
     CI_ENTRYPOINT_FILE="/opt/airflow/scripts/ci/in_container/entrypoint_ci.sh"
 
     # Base python image for the build
-    export PYTHON_BASE_IMAGE=python:${PYTHON_VERSION}-slim-stretch
+    export PYTHON_BASE_IMAGE=python:${PYTHON_VERSION}-slim-buster
     export AIRFLOW_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-python${PYTHON_VERSION}-ci"
     export BUILT_IMAGE_FLAG_FILE="${BUILD_CACHE_DIR}/${BRANCH_NAME}/.built_${PYTHON_VERSION}"
 

scripts/ci/_utils.sh

@@ -1039,7 +1039,7 @@ function set_common_image_variables {
     export AIRFLOW_CI_SAVED_IMAGE_DIR="${BUILD_CACHE_DIR}/${DEFAULT_BRANCH}-python${PYTHON_VERSION}-ci-image"
     export AIRFLOW_CI_IMAGE_ID_FILE="${BUILD_CACHE_DIR}/${DEFAULT_BRANCH}-python${PYTHON_VERSION}-ci-image.sha256"
     export AIRFLOW_CI_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${DEFAULT_BRANCH}-ci"
-    export PYTHON_BASE_IMAGE="python:${PYTHON_VERSION}-slim-stretch"
+    export PYTHON_BASE_IMAGE="python:${PYTHON_VERSION}-slim-buster"
     export BUILT_IMAGE_FLAG_FILE="${BUILD_CACHE_DIR}/${BRANCH_NAME}/.built_${PYTHON_VERSION}"
 
 }

tests/operators/test_virtualenv_operator.py

@@ -27,6 +27,8 @@
 
 from subprocess import CalledProcessError
 
+import pytest
+
 from airflow import DAG
 from airflow.operators.python_operator import PythonVirtualenvOperator
 from airflow.utils import timezone
@@ -131,6 +133,8 @@ def f():
             return True
         self._run_as_operator(f, python_version='2.7', requirements=['dill'])
 
+    @pytest.mark.skipif(sys.version_info.major < 3,
+                        reason="Virtualenv for python 3 cannot work with python 2 only host")
     def test_python_3(self):
         def f():
             import sys
@@ -166,6 +170,8 @@ def f(a):
             return a
         self._run_as_operator(f, system_site_packages=False, use_dill=False, op_args=[4])
 
+    @pytest.mark.skipif(sys.version_info.major < 3,
+                        reason="Virtualenv for python 3 cannot work with python 2 only host")
     def test_string_args(self):
         def f():
             global virtualenv_string_args