Dropped support for Python 2 (#1954)

Signed-off-by: Travis Addair <taddair@uber.com>

.buildkite/gen-pipeline.sh

@@ -8,14 +8,11 @@ repository=823773083436.dkr.ecr.us-east-1.amazonaws.com/buildkite
 
 # list of all the tests
 tests=( \
-       test-cpu-openmpi-py2_7-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_4_1-pyspark2_3_2 \
        test-cpu-openmpi-py3_6-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_4_1-pyspark2_3_2 \
-       test-cpu-gloo-py2_7-tf1_15_0-keras2_3_1-torch1_4_0-mxnet1_5_0-pyspark2_4_0 \
        test-cpu-gloo-py3_6-tf1_15_0-keras2_3_1-torch1_4_0-mxnet1_5_0-pyspark2_4_0 \
        test-cpu-gloo-py3_7-tf2_2_0-keras2_3_1-torch1_5_0-mxnet1_5_0-pyspark2_4_0 \
        test-cpu-gloo-py3_8-tf2_2_0-keras2_3_1-torch1_5_0-mxnet1_5_0-pyspark2_4_0 \
        test-cpu-openmpi-py3_6-tf1_14_0-keras2_2_4-torch1_2_0-mxnet1_4_1-pyspark2_4_0 \
-       test-cpu-openmpi-py2_7-tf2_0_0-keras2_3_1-torch1_3_0-mxnet1_5_0-pyspark2_4_0 \
        test-cpu-openmpi-py3_6-tf2_0_0-keras2_3_1-torch1_3_0-mxnet1_5_0-pyspark2_4_0 \
        test-cpu-openmpi-py3_6-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0 \
        test-cpu-mpich-py3_6-tf1_15_0-keras2_3_1-torch1_3_0-mxnet1_5_0-pyspark2_4_0 \
@@ -103,11 +100,6 @@ run_mpi_pytest() {
     exclude_keras="| sed 's/[a-z_]*tensorflow2[a-z_.]*//g'"
   fi
 
-  local exclude_elastic=""
-  if [[ ${test} == *"py2_"* ]]; then
-    exclude_elastic="| sed 's/test_elastic[a-z_.]*//g'"
-  fi
-
   local excluded_tests="| sed 's/test_interactiverun.py//g' | sed 's/test_spark_keras.py//g' | sed 's/test_spark_torch.py//g'"
 
   # Spark and Run test does not need to be executed with horovodrun, but we still run it below.
@@ -123,7 +115,7 @@ run_mpi_pytest() {
   # pytests have 4x GPU use cases and require a separate queue
   run_test "${test}" "${queue}" \
     ":pytest: Run PyTests (${test})" \
-    "bash -c \"${oneccl_env} cd /horovod/test && (echo test_*.py ${exclude_keras} ${exclude_elastic} ${excluded_tests} ${exclude_standalone_test} | xargs -n 1 \\\$(cat /mpirun_command) pytest -v --capture=no) && pytest --forked -v --capture=no ${standalone_tests}\""
+    "bash -c \"${oneccl_env} cd /horovod/test && (echo test_*.py ${exclude_keras} ${excluded_tests} ${exclude_standalone_test} | xargs -n 1 \\\$(cat /mpirun_command) pytest -v --capture=no) && pytest --forked -v --capture=no ${standalone_tests}\""
 }
 
 run_mpi_integration() {
@@ -216,11 +208,6 @@ run_gloo_pytest() {
     exclude_keras="| sed 's/[a-z_]*tensorflow2[a-z_.]*//g'"
   fi
 
-  local exclude_elastic=""
-  if [[ ${test} == *"py2_"* ]]; then
-    exclude_elastic="| sed 's/test_elastic[a-z_.]*//g'"
-  fi
-
   # These are tested as integration style tests.
   local excluded_tests="| sed 's/test_interactiverun.py//g' | sed 's/test_spark_keras.py//g' | sed 's/test_spark_torch.py//g'"
 
@@ -236,7 +223,7 @@ run_gloo_pytest() {
 
   run_test "${test}" "${queue}" \
     ":pytest: Run PyTests (${test})" \
-    "bash -c \"cd /horovod/test && (echo test_*.py ${exclude_keras} ${exclude_elastic} ${excluded_tests} ${exclude_standalone_test} | xargs -n 1 horovodrun -np 2 -H localhost:2 --gloo pytest -v --capture=no) && pytest --forked -v --capture=no ${standalone_tests}\""
+    "bash -c \"cd /horovod/test && (echo test_*.py ${exclude_keras} ${excluded_tests} ${exclude_standalone_test} | xargs -n 1 horovodrun -np 2 -H localhost:2 --gloo pytest -v --capture=no) && pytest --forked -v --capture=no ${standalone_tests}\""
 }
 
 run_gloo_integration() {
@@ -298,7 +285,7 @@ run_spark_integration() {
   # Horovod Spark Estimator tests
   if [[ ${test} != *"tf1_1_0"* && ${test} != *"tf1_6_0"* && ${test} != *"torch0_"* && ${test} != *"mpich"* && ${test} != *"oneccl"* ]]; then
     if [[ ${test} != *"tf2"* && ${test} != *"tfhead"* ]]; then
-      if [[ ! (  ${test} == *"py2"* && ${test} == *"gloo"* && ${test} != *"openmpi-gloo"* ) ]]; then
+      if [[ ${test} != *"gloo"* || ${test} == *"openmpi-gloo"* ]]; then
         run_test "${test}" "${queue}" \
           ":spark: Spark Keras Rossmann Run (${test})" \
           "bash -c \"OMP_NUM_THREADS=1 python /horovod/examples/keras_spark_rossmann_run.py --num-proc 2 --data-dir file:///data --epochs 3 --sample-rate 0.01\""
@@ -319,7 +306,7 @@ run_spark_integration() {
       fi
     fi
 
-    if [[ ! (  ${test} == *"py2"* && ${test} == *"gloo"* && ${test} != *"openmpi-gloo"* ) ]]; then
+    if [[ ${test} != *"gloo"* || ${test} == *"openmpi-gloo"* ]]; then
       run_test "${test}" "${queue}" \
         ":spark: Spark Torch MNIST (${test})" \
         "bash -c \"OMP_NUM_THREADS=1 python /horovod/examples/pytorch_spark_mnist.py --num-proc 2 --work-dir /work --data-dir /data --epochs 3\""

CHANGELOG.md

@@ -16,4 +16,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Removed
 
+- Dropped support for Python 2. ([#1954](https://github.com/horovod/horovod/pull/1954))
+
 ### Fixed

Dockerfile.cpu

@@ -5,8 +5,8 @@ ENV PYTORCH_VERSION=1.4.0
 ENV TORCHVISION_VERSION=0.5.0
 ENV MXNET_VERSION=1.6.0
 
-# Python 2.7 or 3.6 is supported by Ubuntu Bionic out of the box
-ARG python=2.7
+# Python 3.6 is supported by Ubuntu Bionic out of the box
+ARG python=3.6
 ENV PYTHON_VERSION=${python}
 
 # Set default shell to /bin/bash
@@ -25,13 +25,11 @@ RUN apt-get update && apt-get install -y --allow-downgrades --allow-change-held-
         libpng-dev \
         python${PYTHON_VERSION} \
         python${PYTHON_VERSION}-dev \
+        python${PYTHON_VERSION}-distutils \
         librdmacm1 \
         libibverbs1 \
         ibverbs-providers
 
-RUN if [[ "${PYTHON_VERSION}" == "3.6" ]]; then \
-        apt-get install -y python${PYTHON_VERSION}-distutils; \
-    fi
 RUN ln -s /usr/bin/python${PYTHON_VERSION} /usr/bin/python
 
 RUN curl -O https://bootstrap.pypa.io/get-pip.py && \

Dockerfile.gpu

@@ -8,8 +8,8 @@ ENV CUDNN_VERSION=7.6.5.32-1+cuda10.1
 ENV NCCL_VERSION=2.4.8-1+cuda10.1
 ENV MXNET_VERSION=1.6.0
 
-# Python 2.7 or 3.6 is supported by Ubuntu Bionic out of the box
-ARG python=2.7
+# Python 3.6 is supported by Ubuntu Bionic out of the box
+ARG python=3.6
 ENV PYTHON_VERSION=${python}
 
 # Set default shell to /bin/bash
@@ -31,13 +31,11 @@ RUN apt-get update && apt-get install -y --allow-downgrades --allow-change-held-
         libpng-dev \
         python${PYTHON_VERSION} \
         python${PYTHON_VERSION}-dev \
+        python${PYTHON_VERSION}-distutils \
         librdmacm1 \
         libibverbs1 \
         ibverbs-providers
 
-RUN if [[ "${PYTHON_VERSION}" == "3.6" ]]; then \
-        apt-get install -y python${PYTHON_VERSION}-distutils; \
-    fi
 RUN ln -s /usr/bin/python${PYTHON_VERSION} /usr/bin/python
 
 RUN curl -O https://bootstrap.pypa.io/get-pip.py && \

Dockerfile.test.cpu

@@ -5,7 +5,7 @@ FROM ubuntu:${UBUNTU_VERSION}
 # the first usage only applies to the FROM tag.
 ARG UBUNTU_VERSION=16.04
 ARG MPI_KIND=OpenMPI
-ARG PYTHON_VERSION=2.7
+ARG PYTHON_VERSION=3.6
 ARG TENSORFLOW_PACKAGE=tensorflow==1.14.0
 ARG KERAS_PACKAGE=keras==2.2.4
 ARG PYTORCH_PACKAGE=torch==1.2.0+cpu
@@ -32,20 +32,14 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test
 RUN apt-get update -qq && apt-get install -y --no-install-recommends g++-7
 
 # Install Python.
-RUN apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev
-RUN if [[ "${PYTHON_VERSION}" == "3."* ]]; then \
-        apt-get install -y python${PYTHON_VERSION}-distutils; \
-    fi
+RUN apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-distutils
 RUN ln -s -f /usr/bin/python${PYTHON_VERSION} /usr/bin/python
 RUN wget https://bootstrap.pypa.io/get-pip.py && python get-pip.py && rm get-pip.py
 RUN pip install -U --force pip 'setuptools<46.0.0' requests pytest mock pytest-forked
 
 # Install PySpark.
 RUN apt-get update -qq && apt install -y openjdk-8-jdk-headless
 RUN pip install ${PYSPARK_PACKAGE}
-RUN if [[ "${PYTHON_VERSION}" == "2."* ]]; then \
-    pip install 'pyarrow<0.16.0'; \
-fi
 
 # Install MPI.
 RUN if [[ ${MPI_KIND} == "OpenMPI" ]]; then \
@@ -121,7 +115,7 @@ RUN if [[ ${PYTORCH_PACKAGE} == "torch-nightly" ]]; then \
 RUN pip install "Pillow<7.0" --no-deps
 
 # Install MXNet.
-RUN if [[ ${MXNET_PACKAGE} == "mxnet-nightly" && ${PYTHON_VERSION} != "2.7" ]]; then \
+RUN if [[ ${MXNET_PACKAGE} == "mxnet-nightly" ]]; then \
         pip install --pre mxnet-mkl -f https://dist.mxnet.io/python/all; \
     else \
         pip install ${MXNET_PACKAGE} ; \

Dockerfile.test.gpu

@@ -7,7 +7,7 @@ ARG CUDA_DOCKER_VERSION=10.0-devel-ubuntu16.04
 ARG CUDNN_VERSION=7.6.0.64-1+cuda10.0
 ARG NCCL_VERSION_OVERRIDE=2.4.7-1+cuda10.0
 ARG MPI_KIND=OpenMPI
-ARG PYTHON_VERSION=2.7
+ARG PYTHON_VERSION=3.6
 ARG TENSORFLOW_PACKAGE=tensorflow-gpu==1.14.0
 ARG KERAS_PACKAGE=keras==2.2.4
 ARG PYTORCH_PACKAGE=torch==1.2.0
@@ -39,10 +39,7 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test
 RUN apt-get update -qq && apt-get install -y --no-install-recommends g++-7
 
 # Install Python.
-RUN apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev
-RUN if [[ "${PYTHON_VERSION}" == "3."* ]]; then \
-        apt-get install -y python${PYTHON_VERSION}-distutils; \
-    fi
+RUN apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-distutils
 RUN ln -s -f /usr/bin/python${PYTHON_VERSION} /usr/bin/python
 RUN wget https://bootstrap.pypa.io/get-pip.py && python get-pip.py && rm get-pip.py
 RUN pip install -U --force pip 'setuptools<46.0.0' requests pytest mock pytest-forked
@@ -95,7 +92,7 @@ RUN if [[ ${PYTORCH_PACKAGE} == "torch-nightly" ]]; then \
 RUN pip install "Pillow<7.0" --no-deps
 
 # Install MXNet.
-RUN if [[ ${MXNET_PACKAGE} == "mxnet-nightly" && ${PYTHON_VERSION} != "2.7" ]]; then \
+RUN if [[ ${MXNET_PACKAGE} == "mxnet-nightly" ]]; then \
         pip install --pre mxnet-cu101mkl -f https://dist.mxnet.io/python/all; \
     else \
         pip install ${MXNET_PACKAGE} ; \

build-docker-images.sh

@@ -28,7 +28,7 @@ function build_one()
 docker rmi $(cat Dockerfile.gpu | grep FROM | awk '{print $2}') || true
 docker rmi $(cat Dockerfile.cpu | grep FROM | awk '{print $2}') || true
 
-# build for py2 and py3, cpu and gpu
+# build for cpu and gpu
 build_one 3.6 gpu
 build_one 3.6 cpu
 

docker-compose.test.yml

@@ -6,18 +6,6 @@ services:
       dockerfile: Dockerfile.test.cpu
     privileged: true
     shm_size: 8gb
-  test-cpu-openmpi-py2_7-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_4_1-pyspark2_3_2:
-    extends: test-cpu-base
-    build:
-      args:
-        MPI_KIND: OpenMPI
-        PYTHON_VERSION: 2.7
-        TENSORFLOW_PACKAGE: tensorflow==1.6.0
-        KERAS_PACKAGE: keras==2.1.2
-        PYTORCH_PACKAGE: torch==0.4.1
-        TORCHVISION_PACKAGE: torchvision==0.2.2.post3
-        MXNET_PACKAGE: mxnet==1.4.1
-        PYSPARK_PACKAGE: pyspark==2.3.2
   test-cpu-openmpi-py3_6-tf1_6_0-keras2_1_2-torch0_4_1-mxnet1_4_1-pyspark2_3_2:
     extends: test-cpu-base
     build:
@@ -31,18 +19,6 @@ services:
         TORCHVISION_PACKAGE: torchvision==0.2.2.post3
         MXNET_PACKAGE: mxnet==1.4.1
         PYSPARK_PACKAGE: pyspark==2.3.2
-  test-cpu-gloo-py2_7-tf1_15_0-keras2_3_1-torch1_4_0-mxnet1_5_0-pyspark2_4_0:
-    extends: test-cpu-base
-    build:
-      args:
-        MPI_KIND: None
-        PYTHON_VERSION: 2.7
-        TENSORFLOW_PACKAGE: tensorflow-cpu==1.15.0
-        KERAS_PACKAGE: keras==2.3.1
-        PYTORCH_PACKAGE: torch==1.4.0+cpu
-        TORCHVISION_PACKAGE: torchvision==0.5.0+cpu
-        MXNET_PACKAGE: mxnet==1.5.0
-        PYSPARK_PACKAGE: pyspark==2.4.0
   test-cpu-gloo-py3_6-tf1_15_0-keras2_3_1-torch1_4_0-mxnet1_5_0-pyspark2_4_0:
     extends: test-cpu-base
     build:
@@ -95,18 +71,6 @@ services:
         TORCHVISION_PACKAGE: torchvision==0.4.1+cpu
         MXNET_PACKAGE: mxnet==1.4.1
         PYSPARK_PACKAGE: pyspark==2.4.0
-  test-cpu-openmpi-py2_7-tf2_0_0-keras2_3_1-torch1_3_0-mxnet1_5_0-pyspark2_4_0:
-    extends: test-cpu-base
-    build:
-      args:
-        MPI_KIND: OpenMPI
-        PYTHON_VERSION: 2.7
-        TENSORFLOW_PACKAGE: tensorflow==2.0.0
-        KERAS_PACKAGE: keras==2.3.1
-        PYTORCH_PACKAGE: torch==1.3.0+cpu
-        TORCHVISION_PACKAGE: torchvision==0.4.1+cpu
-        MXNET_PACKAGE: mxnet==1.5.0
-        PYSPARK_PACKAGE: pyspark==2.4.0
   test-cpu-openmpi-py3_6-tf2_0_0-keras2_3_1-torch1_3_0-mxnet1_5_0-pyspark2_4_0:
     extends: test-cpu-base
     build:

docs/elastic.rst

@@ -20,7 +20,6 @@ When to use elastic training
 Requirements
 ~~~~~~~~~~~~
 
-- Python >= 3.6
 - TensorFlow >= 1.15 or PyTorch >= 1.0
 - Horovod >= 0.20.0 with Gloo support (install Horovod using ``HOROVOD_WITH_GLOO=1`` to ensure it is installed)
 - A way to discover available hosts at runtime

docs/install.rst

@@ -3,6 +3,17 @@
 Horovod Installation Guide
 ==========================
 
+Requirements
+------------
+
+- Python >= 3.6
+- MPI or CMake
+- TensorFlow, PyTorch, or MXNet
+
+For best performance on GPU:
+
+- `NCCL 2 <https://developer.nvidia.com/nccl>`__
+
 Frameworks
 ----------
 

examples/elastic/pytorch_mnist_elastic.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 import argparse
 import torch.nn as nn
 import torch.nn.functional as F

examples/elastic/pytorch_synthetic_benchmark_elastic.py

@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import argparse
 import torch.backends.cudnn as cudnn
 import torch.nn.functional as F

examples/elastic/tensorflow2_synthetic_benchmark_elastic.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-from __future__ import absolute_import, division, print_function
-
 import argparse
 import os
 import numpy as np

examples/elastic/tensorflow_keras_mnist_elastic.py

@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import tensorflow as tf
 from tensorflow import keras
 from tensorflow.keras.models import Sequential

examples/keras_imagenet_resnet50.py

@@ -11,8 +11,6 @@
 # increase in the top-1 validation error compared to the single-crop top-1 validation error from
 # https://github.com/KaimingHe/deep-residual-networks.
 #
-from __future__ import print_function
-
 import argparse
 import keras
 from keras import backend as K

examples/keras_mnist.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 import keras
 from keras.datasets import mnist
 from keras.models import Sequential

examples/keras_mnist_advanced.py

@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import argparse
 
 import keras

examples/keras_spark_mnist.py

@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import argparse
 import os
 import subprocess

examples/pytorch_imagenet_resnet50.py

@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import torch
 import argparse
 import torch.backends.cudnn as cudnn

examples/pytorch_mnist.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 import argparse
 import torch.multiprocessing as mp
 import torch.nn as nn

examples/pytorch_spark_mnist.py

@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import argparse
 import os
 import subprocess

examples/pytorch_synthetic_benchmark.py

@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import argparse
 import torch.backends.cudnn as cudnn
 import torch.nn.functional as F

examples/tensorflow2_synthetic_benchmark.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-from __future__ import absolute_import, division, print_function
-
 import argparse
 import os
 import numpy as np

examples/tensorflow_keras_mnist.py

@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import math
 
 import tensorflow as tf

examples/tensorflow_mnist_estimator.py

@@ -14,10 +14,6 @@
 #  limitations under the License.
 """Convolutional Neural Network Estimator for MNIST, built with tf.layers."""
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
 import os
 import errno
 import numpy as np