Stop testing Python 2 on CI (#15990)

apache · Feb 3, 2020 · 7caffa6 · 7caffa6
1 parent 65aab9e
commit 7caffa6
Show file tree

Hide file tree

Showing 37 changed files with 40 additions and 366 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -931,7 +931,6 @@ add_custom_target(mxnet_lint COMMAND ${CMAKE_COMMAND} -DMSVC=${MSVC} -DPYTHON_EX
 
 if(BUILD_CYTHON_MODULES)
   include(cmake/BuildCythonModules.cmake)
-  add_cython_modules(2) # Build cython module for python2 if python2 is found
   add_cython_modules(3) # Build cython module for python3 if python3 is found
   if((NOT ${PYTHON2_FOUND}) AND (NOT ${PYTHON3_FOUND}))
     message(FATAL_ERROR "No python interpreter found to build cython modules")

diff --git a/Makefile b/Makefile
@@ -455,7 +455,7 @@ ifeq ($(USE_DIST_KVSTORE), 1)
 endif
 
 .PHONY: clean all extra-packages test lint clean_all rcpplint rcppexport roxygen\
-	cython2 cython3 cython cyclean
+	cython3 cython cyclean
 
 all: lib/libmxnet.a lib/libmxnet.so $(BIN) extra-packages extension_libs
 
@@ -683,9 +683,6 @@ subgraph_lib:
 cython:
 	cd python; $(PYTHON) setup.py build_ext --inplace --with-cython
 
-cython2:
-	cd python; python2 setup.py build_ext --inplace --with-cython
-
 cython3:
 	cd python; python3 setup.py build_ext --inplace --with-cython
 

diff --git a/ci/docker/install/centos7_python.sh b/ci/docker/install/centos7_python.sh
@@ -28,8 +28,5 @@ yum -y install python36u
 
 # Install PIP
 curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py"
-python2.7 get-pip.py
 python3.6 get-pip.py
-
-pip2 install nose pylint numpy nose-timer requests h5py scipy==1.0.1
-pip3 install nose pylint numpy nose-timer requests h5py scipy==1.0.1
+pip3 install nose pylint numpy nose-timer requests h5py scipy==1.2.3
diff --git a/ci/docker/install/requirements b/ci/docker/install/requirements
@@ -31,3 +31,4 @@ pylint==2.3.1; python_version >= '3.0'
 requests<2.19.0,>=2.18.4
 scipy==1.2.1
 six==1.11.0
+setuptools
diff --git a/ci/docker/install/tensorrt.sh b/ci/docker/install/tensorrt.sh
@@ -18,7 +18,6 @@
 # under the License.
 
 # Install gluoncv since we're testing Gluon models as well
-pip2 install gluoncv==0.2.0
 pip3 install gluoncv==0.2.0
 
 # Install Protobuf

diff --git a/ci/docker/install/ubuntu_arm.sh b/ci/docker/install/ubuntu_arm.sh
@@ -21,4 +21,8 @@ set -ex
 
 apt update || true
 apt install -y \
-    unzip
+    unzip \
+    python3 \
+    python3-pip
+
+pip3 install setuptools
diff --git a/ci/docker/install/ubuntu_caffe.sh b/ci/docker/install/ubuntu_caffe.sh
@@ -55,4 +55,4 @@ ln -s /usr/lib/x86_64-linux-gnu/libhdf5_serial_hl.so.10.0.2 /usr/lib/x86_64-linu
 make all pycaffe -j$(nproc)
 
 cd python
-for req in $(cat requirements.txt); do pip2 install $req; done
+for req in $(cat requirements.txt); do pip3 install $req; done
diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_core.sh
@@ -21,6 +21,8 @@
 # the whole docker cache for the image
 
 set -ex
+# FIXME(larroy) enable in a different PR
+#perl -pi -e 's/archive.ubuntu.com/us-west-2.ec2.archive.ubuntu.com/' /etc/apt/sources.list
 apt-get update || true
 
 # Avoid interactive package installers such as tzdata.

diff --git a/ci/docker/install/ubuntu_nightly_tests.sh b/ci/docker/install/ubuntu_nightly_tests.sh
@@ -32,5 +32,4 @@ apt-get -y install time
 apt-get install -y subversion maven -y #>/dev/null
 
 # Packages needed for the Straight Dope Nightly tests.
-pip2 install pandas scikit-image prompt_toolkit
 pip3 install pandas scikit-image prompt_toolkit
diff --git a/ci/docker/install/ubuntu_onnx.sh b/ci/docker/install/ubuntu_onnx.sh
@@ -31,5 +31,4 @@ apt-get update || true
 apt-get install -y libprotobuf-dev protobuf-compiler
 
 echo "Installing pytest, pytest-cov, protobuf, Pillow, ONNX and tabulate ..."
-pip2 install pytest==3.6.3 pytest-cov==2.5.1 protobuf==3.5.2 onnx==1.3.0 Pillow==5.0.0 tabulate==0.7.5
 pip3 install pytest==3.6.3 pytest-cov==2.5.1 protobuf==3.5.2 onnx==1.3.0 Pillow==5.0.0 tabulate==0.7.5
diff --git a/ci/docker/install/ubuntu_publish.sh b/ci/docker/install/ubuntu_publish.sh
@@ -61,9 +61,7 @@ apt-get install -y python python-pip python3 python3-pip
 
 # the version of the pip shipped with ubuntu may be too lower, install a recent version here
 # Restrict pip version to <19 due to use of Python 3.4 on Ubuntu 14.04
-python2 -m pip install --upgrade 'pip<19'
 python3 -m pip install --upgrade 'pip<19'
 
 # Restrict numpy version to <1.18 due to use of Python 3.4 on Ubuntu 14.04
-python2 -m pip install --upgrade --ignore-installed nose cpplint==1.3.0 'numpy>1.16.0,<1.17' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
-python3 -m pip install --upgrade --ignore-installed nose cpplint==1.3.0 pylint==2.3.1 'numpy>1.16.0,<1.18' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
+python3 -m pip install --upgrade --ignore-installed nose cpplint==1.3.0 pylint==2.3.1 'numpy>1.16.0,<1.18' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
diff --git a/ci/docker/install/ubuntu_python.sh b/ci/docker/install/ubuntu_python.sh
@@ -28,7 +28,4 @@ apt-get install -y python-dev python3-dev virtualenv wget
 # the version of the pip shipped with ubuntu may be too lower, install a recent version here
 wget -nv https://bootstrap.pypa.io/get-pip.py
 python3 get-pip.py
-python2 get-pip.py
-
 pip3 install -r /work/requirements
-pip2 install -r /work/requirements
diff --git a/ci/docker/install/ubuntu_tutorials.sh b/ci/docker/install/ubuntu_tutorials.sh
@@ -25,5 +25,4 @@ apt-get update || true
 apt-get install graphviz python-opencv
 
 # sckit-learn past version 0.20 does not support python version 2 and 3.4
-pip2 install jupyter matplotlib Pillow opencv-python "scikit-learn<0.21.0" graphviz==0.8.4 tqdm mxboard scipy gluoncv
 pip3 install jupyter matplotlib Pillow opencv-python scikit-learn graphviz==0.8.4 tqdm mxboard scipy gluoncv
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
@@ -124,7 +124,7 @@ build_wheel() {
     export MXNET_LIBRARY_PATH=${BUILD_DIR}/libmxnet.so
 
     cd ${PYTHON_DIR}
-    python setup.py bdist_wheel --universal
+    python3 setup.py bdist_wheel
 
     # repackage
 
@@ -469,7 +469,6 @@ build_ubuntu_cpu_openblas() {
         USE_LIBJPEG_TURBO=1           \
         USE_SIGNAL_HANDLER=1          \
         -j$(nproc)
-    make cython PYTHON=python2
     make cython PYTHON=python3
 }
 
@@ -786,7 +785,6 @@ build_ubuntu_gpu_cuda101_cudnn7() {
         CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
         USE_SIGNAL_HANDLER=1                      \
         -j$(nproc)
-    make cython PYTHON=python2
     make cython PYTHON=python3
 }
 
@@ -807,7 +805,6 @@ build_ubuntu_gpu_cuda101_cudnn7_mkldnn_cpp_test() {
         USE_SIGNAL_HANDLER=1                      \
         -j$(nproc)
     make test USE_CPP_PACKAGE=1 -j$(nproc)
-    make cython PYTHON=python2
     make cython PYTHON=python3
 }
 
@@ -828,7 +825,6 @@ build_ubuntu_gpu_cuda101_cudnn7_no_tvm_op() {
         USE_SIGNAL_HANDLER=1                      \
         -j$(nproc)
 
-    make cython PYTHON=python2
     make cython PYTHON=python3
 }
 
@@ -1018,34 +1014,6 @@ cd_unittest_ubuntu() {
     fi
 }
 
-unittest_ubuntu_python2_cpu_cython() {
-    set -ex
-    export PYTHONPATH=./python/
-    export MXNET_MKLDNN_DEBUG=1
-    export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
-    export MXNET_SUBGRAPH_VERBOSE=0
-    export MXNET_ENABLE_CYTHON=1
-    export MXNET_ENFORCE_CYTHON=1
-    export DMLC_LOG_STACK_TRACE_DEPTH=10
-    check_cython 2
-    nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest
-    nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_train.xml --verbose tests/python/train
-    nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_quantization.xml --verbose tests/python/quantization
-}
-
-unittest_ubuntu_python2_cpu() {
-    set -ex
-    export PYTHONPATH=./python/
-    export MXNET_MKLDNN_DEBUG=0
-    export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
-    export MXNET_SUBGRAPH_VERBOSE=0
-    export MXNET_ENABLE_CYTHON=0
-    export DMLC_LOG_STACK_TRACE_DEPTH=10
-    nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest
-    nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_train.xml --verbose tests/python/train
-    nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_quantization.xml --verbose tests/python/quantization
-}
-
 unittest_ubuntu_python3_cpu() {
     set -ex
     export PYTHONPATH=./python/
@@ -1070,17 +1038,6 @@ unittest_ubuntu_python3_cpu_mkldnn() {
     nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_mkl.xml --verbose tests/python/mkl
 }
 
-unittest_ubuntu_python2_gpu() {
-    set -ex
-    export PYTHONPATH=./python/
-    export MXNET_MKLDNN_DEBUG=0  # Ignored if not present
-    export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
-    export MXNET_SUBGRAPH_VERBOSE=0
-    export CUDNN_VERSION=${CUDNN_VERSION:-7.0.3}
-    export DMLC_LOG_STACK_TRACE_DEPTH=10
-    nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu
-}
-
 unittest_ubuntu_python3_gpu() {
     set -ex
     export PYTHONPATH=./python/
@@ -1127,24 +1084,10 @@ unittest_ubuntu_tensorrt_gpu() {
     export CUDNN_VERSION=${CUDNN_VERSION:-7.0.3}
     export MXNET_ENABLE_CYTHON=0
     export DMLC_LOG_STACK_TRACE_DEPTH=10
-    python tests/python/tensorrt/lenet5_train.py
+    tests/python/tensorrt/lenet5_train.py
     nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_trt_gpu.xml --verbose --nocapture tests/python/tensorrt/
 }
 
-# quantization gpu currently only runs on P3 instances
-# need to separte it from unittest_ubuntu_python2_gpu()
-unittest_ubuntu_python2_quantization_gpu() {
-    set -ex
-    export PYTHONPATH=./python/
-    export MXNET_MKLDNN_DEBUG=0  # Ignored if not present
-    export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
-    export MXNET_SUBGRAPH_VERBOSE=0
-    export CUDNN_VERSION=${CUDNN_VERSION:-7.0.3}
-    export MXNET_ENABLE_CYTHON=0
-    export DMLC_LOG_STACK_TRACE_DEPTH=10
-    nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_quantization_gpu.xml --verbose tests/python/quantization_gpu
-}
-
 # quantization gpu currently only runs on P3 instances
 # need to separte it from unittest_ubuntu_python3_gpu()
 unittest_ubuntu_python3_quantization_gpu() {
@@ -1305,7 +1248,7 @@ integrationtest_ubuntu_cpu_onnx() {
 	set -ex
 	export PYTHONPATH=./python/
     export DMLC_LOG_STACK_TRACE_DEPTH=10
-	python tests/python-pytest/onnx/backend_test.py
+	tests/python-pytest/onnx/backend_test.py
 	pytest tests/python-pytest/onnx/mxnet_export_test.py
 	pytest tests/python-pytest/onnx/test_models.py
 	pytest tests/python-pytest/onnx/test_node.py
@@ -1317,14 +1260,14 @@ integrationtest_ubuntu_gpu_python() {
     export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
     export MXNET_SUBGRAPH_VERBOSE=0
     export DMLC_LOG_STACK_TRACE_DEPTH=10
-    python example/image-classification/test_score.py
+    example/image-classification/test_score.py
 }
 
 integrationtest_ubuntu_gpu_caffe() {
     set -ex
     export PYTHONPATH=/work/deps/caffe/python:./python
     export DMLC_LOG_STACK_TRACE_DEPTH=10
-    python tools/caffe_converter/test_converter.py
+    tools/caffe_converter/test_converter.py
 }
 
 integrationtest_ubuntu_cpu_asan() {
@@ -1414,24 +1357,6 @@ integrationtest_ubuntu_gpu_dist_kvstore() {
     popd
 }
 
-test_ubuntu_cpu_python2() {
-    set -ex
-    pushd .
-    export MXNET_LIBRARY_PATH=/work/build/libmxnet.so
-    export DMLC_LOG_STACK_TRACE_DEPTH=10
-
-    VENV=mxnet_py2_venv
-    virtualenv -p `which python2` $VENV
-    source $VENV/bin/activate
-    pip install nose nose-timer
-
-    cd /work/mxnet/python
-    pip install -e .
-    cd /work/mxnet
-    python -m "nose" $NOSE_COVERAGE_ARGUMENTS $NOSE_TIMER_ARGUMENTS --verbose tests/python/unittest
-    popd
-}
-
 test_ubuntu_cpu_python3() {
     set -ex
     pushd .
@@ -1513,7 +1438,7 @@ nightly_test_KVStore_singleNode() {
     set -ex
     export PYTHONPATH=./python/
     export DMLC_LOG_STACK_TRACE_DEPTH=10
-    python tests/nightly/test_kvstore.py
+    tests/nightly/test_kvstore.py
 }
 
 #Test Large Tensor Size
@@ -1583,17 +1508,6 @@ nightly_model_backwards_compat_train() {
     ./tests/nightly/model_backwards_compatibility_check/train_mxnet_legacy_models.sh
 }
 
-# Nightly 'MXNet: The Straight Dope' Single-GPU Tests
-nightly_straight_dope_python2_single_gpu_tests() {
-    set -ex
-    cd /work/mxnet/tests/nightly/straight_dope
-    export PYTHONPATH=/work/mxnet/python/
-    export MXNET_TEST_KERNEL=python2
-    export DMLC_LOG_STACK_TRACE_DEPTH=10
-    nosetests-2.7 $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_straight_dope_python2_single_gpu.xml \
-      test_notebooks_single_gpu.py --nologcapture
-}
-
 nightly_straight_dope_python3_single_gpu_tests() {
     set -ex
     cd /work/mxnet/tests/nightly/straight_dope
@@ -1604,17 +1518,6 @@ nightly_straight_dope_python3_single_gpu_tests() {
       test_notebooks_single_gpu.py --nologcapture
 }
 
-# Nightly 'MXNet: The Straight Dope' Multi-GPU Tests
-nightly_straight_dope_python2_multi_gpu_tests() {
-    set -ex
-    cd /work/mxnet/tests/nightly/straight_dope
-    export PYTHONPATH=/work/mxnet/python/
-    export MXNET_TEST_KERNEL=python2
-    export DMLC_LOG_STACK_TRACE_DEPTH=10
-    nosetests-2.7 $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_straight_dope_python2_multi_gpu.xml \
-      test_notebooks_multi_gpu.py --nologcapture
-}
-
 nightly_straight_dope_python3_multi_gpu_tests() {
     set -ex
     cd /work/mxnet/tests/nightly/straight_dope
@@ -1640,21 +1543,6 @@ nightly_tutorial_test_ubuntu_python3_gpu() {
     nosetests-3.4 $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_tutorials.xml test_tutorials.py --nologcapture
 }
 
-nightly_tutorial_test_ubuntu_python2_gpu() {
-    set -ex
-    cd /work/mxnet/docs
-    export BUILD_VER=tutorial
-    export MXNET_DOCS_BUILD_MXNET=0
-    export DMLC_LOG_STACK_TRACE_DEPTH=10
-    make html
-    export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
-    export MXNET_SUBGRAPH_VERBOSE=0
-    export PYTHONPATH=/work/mxnet/python/
-    export MXNET_TUTORIAL_TEST_KERNEL=python2
-    cd /work/mxnet/tests/tutorials
-    nosetests-3.4 $NOSE_TIMER_ARGUMENTS --with-xunit --xunit-file nosetests_tutorials.xml test_tutorials.py --nologcapture
-}
-
 nightly_java_demo_test_cpu() {
     set -ex
     cd /work/mxnet/scala-package/mxnet-demo/java-demo
@@ -1898,6 +1786,7 @@ build_julia_docs() {
 
    echo "Julia will check for MXNet in $MXNET_HOME/lib"
 
+
    make -C julia/docs
 
    GZIP=-9 tar -zcvf $julia_doc_artifact -C $julia_doc_path .