[AIRFLOW-1314] Cleanup the config

Closes #2414 from bloomberg:airflow-kubernetes-executor

.travis.yml

@@ -59,7 +59,7 @@ env:
     - TOX_ENV=py35-backend_postgres
     - TOX_ENV=flake8
     - TOX_ENV=py27-backend_postgres KUBERNETES_VERSION=v1.8.0
-    - TOX_ENV=py27-backend_postgres KUBERNETES_VERSION=v1.9.0
+    - TOX_ENV=py35-backend_postgres KUBERNETES_VERSION=v1.9.0
 matrix:
   exclude:
     - python: "3.5"
@@ -78,11 +78,11 @@ matrix:
       env: TOX_ENV=flake8
     - python: "3.5"
       env: TOX_ENV=py27-backend_postgres KUBERNETES_VERSION=v1.8.0
-    - python: "3.5"
-      env: TOX_ENV=py27-backend_postgres KUBERNETES_VERSION=v1.9.0
+    - python: "2.7"
+      env: TOX_ENV=py35-backend_postgres KUBERNETES_VERSION=v1.9.0
   allow_failures:
     - env: TOX_ENV=py27-backend_postgres KUBERNETES_VERSION=v1.8.0
-    - env: TOX_ENV=py27-backend_postgres KUBERNETES_VERSION=v1.9.0
+    - env: TOX_ENV=py35-backend_postgres KUBERNETES_VERSION=v1.9.0
 cache:
   directories:
     - $HOME/.wheelhouse/

airflow/bin/cli.py

@@ -21,8 +21,6 @@
 from __future__ import print_function
 import logging
 
-import reprlib
-
 import os
 import socket
 import subprocess
@@ -266,12 +264,8 @@ def _tabulate(pools):
     else:
         log.info(_tabulate(pools=pools))
 
-<<<<<<< HEAD:airflow/bin/cli/cli.py
-@cli_utils.action_logging
-def variables(args):
-=======
->>>>>>> [AIRFLOW-1314] Rebasing against master:airflow/bin/cli.py
 
+@cli_utils.action_logging
 def variables(args):
     if args.get:
         try:

scripts/ci/kubernetes/docker/requirements.txt

@@ -1,35 +1,103 @@
-alembic
-bleach
-configparser
-croniter
-dill
-flask
-flask-admin
-flask-caching
-flask-login
-flask-swagger
-flask-wtf
-funcsigs
-future
-gitpython
-gunicorn
-iso8601
-jinja2
-lxml
-markdown
-pandas
-pendulum
-psutil
-pygments
-python-daemon
-python-dateutil
-python-nvd3
-requests
-setproctitle
-sqlalchemy
-sqlalchemy-utc
-tabulate
-thrift
-tzlocal
-werkzeug
-zope.deprecation
+alembic==0.8.10
+asn1crypto==0.24.0
+aws-xray-sdk==0.95
+Babel==2.5.3
+bleach==2.1.2
+boto==2.48.0
+boto3==1.7.4
+botocore==1.10.4
+cachetools==2.0.1
+certifi==2018.4.16
+cffi==1.11.5
+chardet==3.0.4
+click==6.7
+colorama==0.3.9
+configparser==3.5.0
+cookies==2.2.1
+croniter==0.3.20
+cryptography==2.2.2
+defusedxml==0.5.0
+dill==0.2.7.1
+docker==3.2.1
+docker-pycreds==0.2.2
+docutils==0.14
+Flask==0.12.2
+Flask-Admin==1.4.1
+Flask-AppBuilder==1.10.0
+Flask-Babel==0.11.1
+Flask-Caching==1.3.3
+Flask-Login==0.2.11
+Flask-OpenID==1.2.5
+Flask-SQLAlchemy==2.1
+flask-swagger==0.2.13
+Flask-WTF==0.14.2
+freezegun==0.3.10
+funcsigs==1.0.0
+future==0.16.0
+gitdb2==2.0.3
+GitPython==2.1.9
+google-auth==1.4.1
+gunicorn==19.7.1
+html5lib==1.0.1
+idna==2.6
+ipaddress==1.0.22
+iso8601==0.1.12
+itsdangerous==0.24
+Jinja2==2.8.1
+jmespath==0.9.3
+jsondiff==1.1.1
+jsonpickle==0.9.6
+kubernetes==6.0.0
+lockfile==0.12.2
+lxml==3.8.0
+Mako==1.0.7
+Markdown==2.6.11
+MarkupSafe==1.0
+mock==2.0.0
+moto==1.3.3
+nose==1.3.7
+numpy==1.14.2
+oauthlib==2.0.7
+ordereddict==1.1
+pandas==0.22.0
+parameterized==0.6.1
+pbr==4.0.2
+pendulum==1.4.4
+psutil==4.4.2
+pyaml==17.12.1
+pyasn1==0.4.2
+pyasn1-modules==0.2.1
+pycparser==2.18
+Pygments==2.2.0
+python-daemon==2.1.2
+python-dateutil==2.6.1
+python-editor==1.0.3
+python-nvd3==0.15.0
+python-slugify==1.2.5
+python3-openid==3.1.0
+pytz==2018.4
+pytzdata==2018.4
+PyYAML==3.12
+requests==2.18.4
+requests-oauthlib==0.8.0
+responses==0.9.0
+rsa==3.4.2
+s3transfer==0.1.13
+setproctitle==1.1.10
+six==1.11.0
+smmap2==2.0.3
+SQLAlchemy==1.1.18
+SQLAlchemy-Utc==0.10.0
+tabulate==0.7.7
+thrift==0.11.0
+tzlocal==1.5.1
+unicodecsv==0.14.1
+Unidecode==1.0.22
+urllib3==1.22
+webencodings==0.5.1
+websocket-client==0.47.0
+Werkzeug==0.14.1
+wrapt==1.10.11
+WTForms==2.1
+xmltodict==0.11.0
+zope.deprecation==4.3.0

scripts/ci/kubernetes/kube/airflow.yaml.template

@@ -194,8 +194,38 @@ data:
     sql_alchemy_conn = $SQL_ALCHEMY_CONN
 
     [scheduler]
-    dag_dir_list_interval = 60
+    dag_dir_list_interval = 300
     child_process_log_directory = /root/airflow/logs/scheduler
+    # Task instances listen for external kill signal (when you clear tasks
+    # from the CLI or the UI), this defines the frequency at which they should
+    # listen (in seconds).
+    job_heartbeat_sec = 5
+    max_threads = 16
+
+    # The scheduler constantly tries to trigger new tasks (look at the
+    # scheduler section in the docs for more information). This defines
+    # how often the scheduler should run (in seconds).
+    scheduler_heartbeat_sec = 5
+
+    # after how much time should the scheduler terminate in seconds
+    # -1 indicates to run continuously (see also num_runs)
+    run_duration = -1
+
+    # after how much time a new DAGs should be picked up from the filesystem
+    min_file_process_interval = 0
+
+    statsd_on = False
+    statsd_host = localhost
+    statsd_port = 8125
+    statsd_prefix = airflow
+
+    # How many seconds to wait between file-parsing loops to prevent the logs from being spammed.
+    min_file_parsing_loop_time = 1
+
+    print_stats_interval = 30
+    scheduler_zombie_task_threshold = 300
+    max_tis_per_query = 0
+    authenticate = False
 
     [kubernetes]
     airflow_configmap = airflow-configmap

scripts/ci/kubernetes/minikube/start_minikube.sh

@@ -15,66 +15,92 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# Guard against a kubernetes cluster already being up
-kubectl get pods &> /dev/null
-if [ $? -eq 0 ]; then
-  echo "kubectl get pods returned 0 exit code, exiting early"
-  exit 0
-fi
-#
+# This script was based on one made by @kimoonkim for kubernetes-hdfs
 
-curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.26.0/minikube-linux-amd64 && chmod +x minikube
-curl -Lo kubectl  https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl && chmod +x kubectl
+#!/usr/bin/env bash
 
-sudo mkdir -p /usr/local/bin
-sudo mv minikube /usr/local/bin/minikube
-sudo mv kubectl /usr/local/bin/kubectl
+_MY_SCRIPT="${BASH_SOURCE[0]}"
+_MY_DIR=$(cd "$(dirname "$_MY_SCRIPT")" && pwd)
+# Avoids 1.7.x because of https://github.com/kubernetes/minikube/issues/2240
+_KUBERNETES_VERSION="${KUBERNETES_VERSION}"
+
+echo "setting up kubernetes ${_KUBERNETES_VERSION}"
+
+_MINIKUBE_VERSION="v0.25.2"
+_HELM_VERSION=v2.8.1
+_VM_DRIVER=none
+USE_MINIKUBE_DRIVER_NONE=true
+
+_UNAME_OUT=$(uname -s)
+case "${_UNAME_OUT}" in
+    Linux*)     _MY_OS=linux;;
+    Darwin*)    _MY_OS=darwin;;
+    *)          _MY_OS="UNKNOWN:${unameOut}"
+esac
+echo "Local OS is ${_MY_OS}"
 
 export MINIKUBE_WANTUPDATENOTIFICATION=false
 export MINIKUBE_WANTREPORTERRORPROMPT=false
-export MINIKUBE_HOME=$HOME
 export CHANGE_MINIKUBE_NONE_USER=true
-mkdir $HOME/.kube || true
-touch $HOME/.kube/config
 
-export KUBECONFIG=$HOME/.kube/config
+cd $_MY_DIR
 
-start_minikube(){
-  sudo -E minikube start --vm-driver=none --kubernetes-version="${KUBERNETES_VERSION}"
+rm -rf tmp
+mkdir -p bin tmp
 
-  # this for loop waits until kubectl can access the api server that minikube has created
-  for i in {1..90} # timeout 3 minutes
-  do
-    echo "------- Running kubectl get pods -------"
-    STDERR=$(kubectl get pods  2>&1 >/dev/null)
-    if [ $? -eq 0 ]; then
-      echo $STDERR
+sudo mkdir -p /usr/local/bin
 
-      # We do not need dynamic hostpath provisioning, so disable the default storageclass
-      sudo -E minikube addons disable default-storageclass && kubectl delete storageclasses --all
+if [[ ! -x /usr/local/bin/kubectl ]]; then
+  echo Downloading kubectl, which is a requirement for using minikube.
+  curl -Lo bin/kubectl  \
+    https://storage.googleapis.com/kubernetes-release/release/${_KUBERNETES_VERSION}/bin/${_MY_OS}/amd64/kubectl
+  chmod +x bin/kubectl
+fi
+if [[ ! -x /usr/local/bin/minikube ]]; then
+  echo Downloading minikube.
+  curl -Lo bin/minikube  \
+    https://storage.googleapis.com/minikube/releases/${_MINIKUBE_VERSION}/minikube-${_MY_OS}-amd64
+  chmod +x bin/minikube
+fi
+
+sudo mv bin/minikube /usr/local/bin/minikube
+sudo mv bin/kubectl /usr/local/bin/kubectl
 
-      # We need to give permission to watch pods to the airflow scheduler.
-      # The easiest way to do that is by giving admin access to the default serviceaccount (NOT SAFE!)
-      kubectl create clusterrolebinding add-on-cluster-admin   --clusterrole=cluster-admin   --serviceaccount=default:default
-      exit 0
+export PATH="${_MY_DIR}/bin:$PATH"
+
+if [[ "${USE_MINIKUBE_DRIVER_NONE:-}" = "true" ]]; then
+  # Run minikube with none driver.
+  # See https://blog.travis-ci.com/2017-10-26-running-kubernetes-on-travis-ci-with-minikube
+  _VM_DRIVER="--vm-driver=none"
+  if [[ ! -x /usr/local/bin/nsenter ]]; then
+    # From https://engineering.bitnami.com/articles/implementing-kubernetes-integration-tests-in-travis.html
+    # Travis ubuntu trusty env doesn't have nsenter, needed for --vm-driver=none
+    which nsenter >/dev/null && return 0
+    echo "INFO: Building 'nsenter' ..."
+cat <<-EOF | docker run -i --rm -v "$(pwd):/build" ubuntu:14.04 >& nsenter.build.log
+        apt-get update
+        apt-get install -qy git bison build-essential autopoint libtool automake autoconf gettext pkg-config
+        git clone --depth 1 git://git.kernel.org/pub/scm/utils/util-linux/util-linux.git /tmp/util-linux
+        cd /tmp/util-linux
+        ./autogen.sh
+        ./configure --without-python --disable-all-programs --enable-nsenter
+        make nsenter
+        cp -pfv nsenter /build
+EOF
+    if [ ! -f ./nsenter ]; then
+        echo "ERROR: nsenter build failed, log:"
+        cat nsenter.build.log
+        return 1
     fi
-    echo $STDERR
-    sleep 2
-  done
-}
-
-cleanup_minikube(){
-  sudo -E minikube stop
-  sudo -E minikube delete
-  docker stop $(docker ps -a -q) || true
-  docker rm $(docker ps -a -q) || true
-  sleep 1
-}
-
-start_minikube
-echo "Minikube cluster creation timedout. Attempting to restart the minikube cluster."
-cleanup_minikube
-start_minikube
-echo "Minikube cluster creation timedout a second time. Failing."
-
-exit 1
+    echo "INFO: nsenter build OK"
+    sudo mv ./nsenter /usr/local/bin
+  fi
+fi
+
+echo "your path is ${PATH}"
+
+_MINIKUBE="sudo PATH=$PATH minikube"
+
+$_MINIKUBE config set bootstrapper localkube
+$_MINIKUBE start --kubernetes-version=${_KUBERNETES_VERSION}  --vm-driver=none
+$_MINIKUBE update-context