Merge pull request #200 from sony/feature/20191205-remove-python27

Remove python2 dependencies.
sony · Jan 10, 2020 · 6d25d9c · 6d25d9c
2 parents 5cd38e9 + 4c6cbac
commit 6d25d9c
Show file tree

Hide file tree

Showing 14 changed files with 51 additions and 141 deletions.
diff --git a/build-tools/make/build.mk b/build-tools/make/build.mk
@@ -71,6 +71,11 @@ nnabla-ext-cuda-cpplib-rpm: nnabla-ext-cuda-cpplib
 	@cd $(BUILD_EXT_CUDA_DIRECTORY_CPPLIB) && cpack -G RPM CPackConfig.cmake
 	@cd $(BUILD_EXT_CUDA_DIRECTORY_CPPLIB) && cpack -G TBZ2 CPackConfig.cmake
 
+.PHONY: nnabla-ext-cuda-cpplib-deb
+nnabla-ext-cuda-cpplib-deb: nnabla-ext-cuda-cpplib
+	@cd $(BUILD_EXT_CUDA_DIRECTORY_CPPLIB) && cpack -G DEB CPackConfig.cmake
+	@cd $(BUILD_EXT_CUDA_DIRECTORY_CPPLIB) && cpack -G TBZ2 CPackConfig.cmake
+
 .PHONY: nnabla-ext-cuda-cpplib-multi-gpu
 nnabla-ext-cuda-cpplib-multi-gpu:
 	mkdir -p $(BUILD_EXT_CUDA_DIRECTORY_CPPLIB_MULTI_GPU)
@@ -98,7 +103,7 @@ nnabla-ext-cuda-cpplib-multi-gpu-deb: nnabla-ext-cuda-cpplib-multi-gpu
 # wheel
 .PHONY: nnabla-ext-cuda-wheel
 nnabla-ext-cuda-wheel:
-	$(call with-virtualenv, \
+	$(call with-venv, \
 		$(NNABLA_EXT_CUDA_DIRECTORY), \
 		$(BUILD_EXT_CUDA_DIRECTORY_WHEEL)/env, \
 		-f build-tools/make/build.mk, \
@@ -171,7 +176,7 @@ nnabla-ext-cuda-shell:
 # test
 .PHONY: nnabla-ext-cuda-test
 nnabla-ext-cuda-test:
-	$(call with-virtualenv, \
+	$(call with-venv, \
 		$(NNABLA_EXT_CUDA_DIRECTORY), \
 		$(BUILD_EXT_CUDA_DIRECTORY_WHEEL)/env, \
 		-f build-tools/make/build.mk, \

diff --git a/build-tools/msvc/build_wheel.bat b/build-tools/msvc/build_wheel.bat
@@ -37,7 +37,7 @@ IF NOT DEFINED WHL (
 )
 
 IF EXIST env RD /s /q env
-virtualenv --system-site-packages env || GOTO :error
+python -m venv --system-site-packages env || GOTO :error
 CALL env\scripts\activate.bat || GOTO :error
 
 pip install --no-deps %WHL% || GOTO :error

diff --git a/build-tools/msvc/test.bat b/build-tools/msvc/test.bat
@@ -43,10 +43,13 @@ IF NOT DEFINED WHLCUDA (
 )
 
 IF EXIST env RD /s /q env
-virtualenv --system-site-packages env || GOTO :error
+python -m venv --system-site-packages env || GOTO :error
 CALL env\scripts\activate.bat || GOTO :error
 
-pip install --no-deps %WHL% || GOTO :error
+REM Workaround for ONNX installation
+pip install "onnx<1.6"
+
+pip install %WHL% || GOTO :error
 pip install --no-deps %WHLCUDA% || GOTO :error
 
 SET PYTHONPATH=%nnabla_ext_cuda_root%\python\test;%PYTHONPATH%

diff --git a/docker/README.md b/docker/README.md
@@ -11,8 +11,6 @@ The available tags are as following.
 | ------ |:------------:|:-----:|:------:|:---------------:|:------------------- |
 | latest | 8.0          | 7.1   | 3.5    | >=361           | py3/cuda80/         |
 | 92     | 9.2          | 7.1   | 3.5    | >=396           | py3/cuda92/         |
-| py2    | 8.0          | 7.1   | 2.7    | >=361           | py2/cuda80/         |
-| py2-92 | 9.2          | 7.1   | 2.7    | >=396           | py2/cuda92/         |
 
 The docker image can be executed as below.
 

diff --git a/docker/development/Dockerfile.build b/docker/development/Dockerfile.build
@@ -74,7 +74,7 @@ ENV PATH=/opt/rh/git19/root/usr/bin:/opt/rh/devtoolset-4/root/usr/bin:$PATH
 ENV LD_LIBRARY_PATH=/opt/rh/gti19/root/usr/lib64:/opt/rh/devtoolset-4/root/usr/lib64:/opt/rh/devtoolset-4/root/usr/lib:$LD_LIBRARY_PATH
 ENV PKG_CONFIG_PATH=/opt/rh/git19/root/usr/lib64/pkgconfig:${PKG_CONFIG_PATH}
 
-################################################### cmake
+############################################################# cmake
 ENV CMAKEVER=3.14.3
 RUN mkdir /tmp/deps \
     && cd /tmp/deps \
@@ -93,8 +93,8 @@ RUN mkdir /tmp/deps \
     && cd / \
     && rm -rf /tmp/*
 
-################################################## protobuf
-ENV PROTOVER=3.4.1
+############################################################ protobuf
+ENV PROTOVER=3.10.1
 RUN mkdir /tmp/deps \
     && cd /tmp/deps \
     && curl -L https://github.com/google/protobuf/archive/v${PROTOVER}.tar.gz -o protobuf-v${PROTOVER}.tar.gz \
@@ -111,7 +111,7 @@ RUN mkdir /tmp/deps \
     && cd / \
     && rm -rf /tmp/*
 
-################################################## libarchive
+############################################################ libarchive
 RUN mkdir /tmp/deps \
     && cd /tmp/deps \
     && curl -L https://www.libarchive.org/downloads/libarchive-3.3.2.tar.gz -o libarchive-3.3.2.tar.gz \
@@ -131,7 +131,7 @@ RUN mkdir /tmp/deps \
     && cd / \
     && rm -rf /tmp/*
 
-################################################## patchelf
+############################################################ patchelf
 RUN mkdir /tmp/deps \
     && cd /tmp/deps \
     && wget http://nixos.org/releases/patchelf/patchelf-0.9/patchelf-0.9.tar.bz2 \
@@ -143,12 +143,11 @@ RUN mkdir /tmp/deps \
     && cd / \
     && rm -rf /tmp/*
 
-################################################## miniconda3
+############################################################ miniconda3
 ARG PYTHON_VERSION_MAJOR
 ARG PYTHON_VERSION_MINOR
 ENV PYVERNAME=${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}
 
-# Limitation: numpy>=1.17 does not support python2.7
 ADD python/requirements.txt /tmp/deps/
 
 ARG CUDA_VERSION_MAJOR_MINOR
@@ -161,7 +160,7 @@ RUN umask 0 \
     && . /opt/miniconda3/bin/activate \
     && conda create -n nnabla-build python=${PYVERNAME} \
     && conda activate nnabla-build \
-    && pip install numpy\<1.17 \
+    && pip install numpy \
     && pip install -U -r /tmp/deps/requirements.txt \
     && ( [ "$CUDA_VERSION_MAJOR_MINOR" = "10.0" ] || [ "$CUDA_VERSION_MAJOR_MINOR" = "9.0" ] \
         && [ "x$ARCH_SUFFIX" = "x" ] \

diff --git a/docker/development/Dockerfile.build-multi-gpu b/docker/development/Dockerfile.build-multi-gpu
@@ -37,7 +37,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     wget \
     zip
 
-################################################### cmake
+############################################################# cmake
 ENV CMAKEVER=3.14.3
 RUN mkdir /tmp/deps \
     && cd /tmp/deps \
@@ -55,8 +55,8 @@ RUN mkdir /tmp/deps \
     && cd / \
     && rm -rf /tmp/*
 
-################################################## protobuf
-ENV PROTOVER=3.4.1
+############################################################ protobuf
+ENV PROTOVER=3.10.1
 RUN mkdir /tmp/deps \
     && cd /tmp/deps \
     && curl -L https://github.com/google/protobuf/archive/v${PROTOVER}.tar.gz -o protobuf-v${PROTOVER}.tar.gz \
@@ -73,12 +73,11 @@ RUN mkdir /tmp/deps \
     && cd / \
     && rm -rf /tmp/*
 
-################################################## miniconda3
+############################################################ miniconda3
 ARG PYTHON_VERSION_MAJOR
 ARG PYTHON_VERSION_MINOR
 ENV PYVERNAME=${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}
 
-# Limitation: numpy>=1.17 does not support python2.7
 ADD python/requirements.txt /tmp/deps/
 
 ARG CUDA_VERSION_MAJOR_MINOR
@@ -91,7 +90,7 @@ RUN umask 0 \
     && . /opt/miniconda3/bin/activate \
     && conda create -n nnabla-build python=${PYVERNAME} \
     && conda activate nnabla-build \
-    && pip install numpy\<1.17 \
+    && pip install numpy \
     && pip install -U -r /tmp/deps/requirements.txt \
     && ( [ "$CUDA_VERSION_MAJOR_MINOR" = "10.0" ] || [ "$CUDA_VERSION_MAJOR_MINOR" = "9.0" ] \
         && [ "x$ARCH_SUFFIX" = "x" ] \

diff --git a/docker/development/Dockerfile.build-multi-gpu-ppc64le b/docker/development/Dockerfile.build-multi-gpu-ppc64le
@@ -42,7 +42,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     wget \
     zip
 
-################################################### cmake
+############################################################# cmake
 ENV CMAKEVER=3.14.3
 RUN mkdir /tmp/deps \
     && cd /tmp/deps \
@@ -60,8 +60,8 @@ RUN mkdir /tmp/deps \
     && cd / \
     && rm -rf /tmp/*
 
-################################################## protobuf
-ENV PROTOVER=3.4.1
+############################################################ protobuf
+ENV PROTOVER=3.10.1
 RUN mkdir /tmp/deps \
     && cd /tmp/deps \
     && curl -L https://github.com/google/protobuf/archive/v${PROTOVER}.tar.gz -o protobuf-v${PROTOVER}.tar.gz \
@@ -78,12 +78,11 @@ RUN mkdir /tmp/deps \
     && cd / \
     && rm -rf /tmp/*
 
-################################################## miniconda3
+############################################################ miniconda3
 ARG PYTHON_VERSION_MAJOR
 ARG PYTHON_VERSION_MINOR
 ENV PYVERNAME=${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}
 
-# Limitation: numpy>=1.17 does not support python2.7
 ADD python/requirements.txt /tmp/deps/
 
 RUN umask 0 \
@@ -96,8 +95,9 @@ RUN umask 0 \
     && conda activate nnabla-build \
     && conda update --all -y \
     && pip install -U pip wheel mako pyyaml \
-    && pip install -U numpy\<1.17 \
-    && pip install -U -r /tmp/deps/requirements.txt \
+    && pip install -U numpy \
+    && pip install scipy\<1.4 \
+    && pip install -r /tmp/deps/requirements.txt \
     && conda clean -y --all \
     && cd / \
     && rm -rf /tmp/*

diff --git a/docker/development/Dockerfile.build-ppc64le b/docker/development/Dockerfile.build-ppc64le
@@ -44,7 +44,7 @@ RUN yum install -y \
     && yum group install -y "Development Tools" \
     && yum clean all
 
-################################################### cmake
+############################################################# cmake
 ENV CMAKEVER=3.14.3
 RUN mkdir /tmp/deps \
     && cd /tmp/deps \
@@ -63,8 +63,8 @@ RUN mkdir /tmp/deps \
     && cd / \
     && rm -rf /tmp/*
 
-################################################## protobuf
-ENV PROTOVER=3.4.1
+############################################################ protobuf
+ENV PROTOVER=3.10.1
 RUN mkdir /tmp/deps \
     && cd /tmp/deps \
     && curl -L https://github.com/google/protobuf/archive/v${PROTOVER}.tar.gz -o protobuf-v${PROTOVER}.tar.gz \
@@ -81,7 +81,7 @@ RUN mkdir /tmp/deps \
     && cd / \
     && rm -rf /tmp/*
 
-################################################## libarchive
+############################################################ libarchive
 RUN mkdir /tmp/deps \
     && cd /tmp/deps \
     && curl -L https://www.libarchive.org/downloads/libarchive-3.3.2.tar.gz -o libarchive-3.3.2.tar.gz \
@@ -101,7 +101,7 @@ RUN mkdir /tmp/deps \
     && cd / \
     && rm -rf /tmp/*
 
-################################################## patchelf
+############################################################ patchelf
 RUN mkdir /tmp/deps \
     && cd /tmp/deps \
     && wget http://nixos.org/releases/patchelf/patchelf-0.9/patchelf-0.9.tar.bz2 \
@@ -113,12 +113,10 @@ RUN mkdir /tmp/deps \
     && cd / \
     && rm -rf /tmp/*
 
-################################################## miniconda3
+############################################################ miniconda3
 ARG PYTHON_VERSION_MAJOR
 ARG PYTHON_VERSION_MINOR
-ENV PYVERNAME=${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}
 
-# Limitation: numpy>=1.17 does not support python2.7
 ADD python/requirements.txt /tmp/deps/
 
 RUN umask 0 \
@@ -128,15 +126,16 @@ RUN umask 0 \
     && bash Miniconda3-latest-Linux-ppc64le.sh -b -p /opt/miniconda3 \
     && rm -rf Miniconda3-latest-Linux-ppc64le.sh \
     && . /opt/miniconda3/bin/activate \
-    && conda create -n nnabla-build python=${PYVERNAME} \
-    && conda activate nnabla-build \
+    && conda create -n nnabla-build${PYTHON_VERSION_MAJOR}${PYTHON_VERSION_MINOR} python=${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR} \
+    && conda activate nnabla-build${PYTHON_VERSION_MAJOR}${PYTHON_VERSION_MINOR} \
     && conda update --all -y \
-    && pip install -U pip wheel mako pyyaml \
-    && pip install -U cython \
-    && pip install -U numpy\<1.17 \
-    && pip install -U -r /tmp/deps/requirements.txt \
+    && pip install pip wheel mako pyyaml \
+    && pip install cython \
+    && CFLAGS="--std=c99" pip install numpy \
+    && CFLAGS="--std=c99" pip install scipy\<1.4 \
+    && pip install -r /tmp/deps/requirements.txt \
     && conda clean -y --all \
     && cd / \
     && rm -rf /tmp/*
 
-ENV PATH /opt/miniconda3/envs/nnabla-build/bin:$PATH
+ENV PATH /opt/miniconda3/envs/nnabla-build${PYTHON_VERSION_MAJOR}${PYTHON_VERSION_MINOR}/bin:$PATH
diff --git a/docker/py27/cuda100/Dockerfile b/docker/py27/cuda100/Dockerfile
diff --git a/docker/py27/cuda80/Dockerfile b/docker/py27/cuda80/Dockerfile
diff --git a/docker/py27/cuda90/Dockerfile b/docker/py27/cuda90/Dockerfile
diff --git a/docker/py27/cuda92/Dockerfile b/docker/py27/cuda92/Dockerfile