Skip to content
Permalink
Browse files

Update PyTorch to 1.2.0 (#1320)

* Update PyTorch for Dockerfile

Signed-off-by: Alex Sergeev <alsrgv@users.noreply.github.com>

* Switch CI to PyTorch 1.2.0 for latest release

Signed-off-by: Alex Sergeev <alsrgv@users.noreply.github.com>

* Update PyTorch package links

Signed-off-by: Alex Sergeev <alsrgv@users.noreply.github.com>

* Split torch tag

Signed-off-by: Alex Sergeev <alsrgv@users.noreply.github.com>
  • Loading branch information...
alsrgv committed Aug 20, 2019
1 parent 2fed041 commit a77e50cd84b9c1fedfa9497840e70c5cb1135b28
Showing with 45 additions and 46 deletions.
  1. +5 −5 .buildkite/gen-pipeline.sh
  2. +3 −4 Dockerfile
  3. +5 −5 Dockerfile.test.cpu
  4. +5 −5 Dockerfile.test.gpu
  5. +1 −1 build-docker-images.sh
  6. +26 −26 docker-compose.test.yml
@@ -23,9 +23,9 @@ tests=( \
test-cpu-openmpi-gloo-py2_7-tf1_14_0-keras2_2_4-torch1_1_0-mxnet1_4_1-pyspark2_4_0 \
test-cpu-openmpi-gloo-py3_5-tf1_14_0-keras2_2_4-torch1_1_0-mxnet1_4_1-pyspark2_4_0 \
test-cpu-openmpi-gloo-py3_6-tf1_14_0-keras2_2_4-torch1_1_0-mxnet1_4_1-pyspark2_4_0 \
test-cpu-openmpi-py2_7-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0 \
test-cpu-openmpi-py3_5-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0 \
test-cpu-openmpi-py3_6-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0 \
test-cpu-openmpi-py2_7-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0 \
test-cpu-openmpi-py3_5-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0 \
test-cpu-openmpi-py3_6-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0 \
test-cpu-openmpi-py2_7-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0 \
test-cpu-openmpi-py3_6-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0 \
test-cpu-mpich-py3_6-tf1_14_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0 \
@@ -38,8 +38,8 @@ tests=( \
test-gpu-gloo-py3_6-tf1_14_0-keras2_2_4-torch1_1_0-mxnet1_4_1-pyspark2_4_0 \
test-gpu-openmpi-gloo-py2_7-tf1_14_0-keras2_2_4-torch1_1_0-mxnet1_4_1-pyspark2_4_0 \
test-gpu-openmpi-gloo-py3_6-tf1_14_0-keras2_2_4-torch1_1_0-mxnet1_4_1-pyspark2_4_0 \
test-gpu-openmpi-py2_7-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0 \
test-gpu-openmpi-py3_6-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0 \
test-gpu-openmpi-py2_7-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0 \
test-gpu-openmpi-py3_6-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0 \
test-gpu-openmpi-py2_7-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0 \
test-gpu-openmpi-py3_6-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0 \
test-gpu-mpich-py3_6-tf1_14_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0 \
@@ -2,8 +2,8 @@ FROM nvidia/cuda:10.0-devel-ubuntu18.04

# TensorFlow version is tightly coupled to CUDA and cuDNN so it should be selected carefully
ENV TENSORFLOW_VERSION=1.14.0
ENV PYTORCH_VERSION=1.1.0
ENV TORCHVISION_VERSION=0.3.0
ENV PYTORCH_VERSION=1.2.0
ENV TORCHVISION_VERSION=0.4.0
ENV CUDNN_VERSION=7.6.0.64-1+cuda10.0
ENV NCCL_VERSION=2.4.7-1+cuda10.0
ENV MXNET_VERSION=1.5.0
@@ -50,8 +50,7 @@ RUN pip install numpy \
tensorflow-gpu==${TENSORFLOW_VERSION} \
keras \
h5py
RUN pip install https://download.pytorch.org/whl/cu100/torch-${PYTORCH_VERSION}-$(python -c "import wheel.pep425tags as w; print('-'.join(w.get_supported()[0]))").whl \
https://download.pytorch.org/whl/cu100/torchvision-${TORCHVISION_VERSION}-$(python -c "import wheel.pep425tags as w; print('-'.join(w.get_supported()[0]))").whl
RUN pip install -f https://download.pytorch.org/whl/torch_stable.html torch==${PYTORCH_VERSION} torchvision==${TORCHVISION_VERSION}
RUN pip install mxnet-cu100==${MXNET_VERSION}

# Install Open MPI
@@ -8,9 +8,9 @@ ARG MPI_KIND=OpenMPI
ARG PYTHON_VERSION=2.7
ARG TENSORFLOW_PACKAGE=tensorflow==1.14.0
ARG KERAS_PACKAGE=keras==2.2.4
ARG PYTORCH_PACKAGE=torch==1.1.0
ARG TORCHVISION_PACKAGE=https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp27-cp27mu-linux_x86_64.whl
ARG MXNET_PACKAGE=mxnet==1.4.1
ARG PYTORCH_PACKAGE=torch==1.2.0+cpu
ARG TORCHVISION_PACKAGE=torchvision==0.4.0+cpu
ARG MXNET_PACKAGE=mxnet==1.5.0
ARG PYSPARK_PACKAGE=pyspark==2.4.0

# Set default shell to /bin/bash
@@ -108,9 +108,9 @@ RUN pip install future typing
RUN if [[ ${PYTORCH_PACKAGE} == "torch-nightly" ]]; then \
pip install torch_nightly -v -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html; \
else \
pip install ${PYTORCH_PACKAGE}; \
pip install ${PYTORCH_PACKAGE} -f https://download.pytorch.org/whl/torch_stable.html; \
fi
RUN pip install ${TORCHVISION_PACKAGE} Pillow --no-deps
RUN pip install ${TORCHVISION_PACKAGE} Pillow --no-deps -f https://download.pytorch.org/whl/torch_stable.html

# Install MXNet.
RUN pip install ${MXNET_PACKAGE}
@@ -10,9 +10,9 @@ ARG MPI_KIND=OpenMPI
ARG PYTHON_VERSION=2.7
ARG TENSORFLOW_PACKAGE=tensorflow-gpu==1.14.0
ARG KERAS_PACKAGE=keras==2.2.4
ARG PYTORCH_PACKAGE=https://download.pytorch.org/whl/cu100/torch-1.1.0-cp27-cp27mu-linux_x86_64.whl
ARG TORCHVISION_PACKAGE=https://download.pytorch.org/whl/cu100/torchvision-0.3.0-cp27-cp27mu-linux_x86_64.whl
ARG MXNET_PACKAGE=mxnet-cu100==1.4.1
ARG PYTORCH_PACKAGE=torch==1.2.0
ARG TORCHVISION_PACKAGE=torchvision==0.4.0
ARG MXNET_PACKAGE=mxnet-cu100==1.5.0
ARG PYSPARK_PACKAGE=pyspark==2.4.0
ARG HOROVOD_BUILD_FLAGS=HOROVOD_GPU_ALLREDUCE=NCCL
ARG HOROVOD_MIXED_INSTALL=0
@@ -90,9 +90,9 @@ RUN if [[ ${PYTORCH_PACKAGE} == "torch-nightly" ]]; then \
PYTORCH_CUDA=$(echo ${CUDA_DOCKER_VERSION} | awk -F- '{print $1}' | sed 's/\.//'); \
pip install torch_nightly -v -f https://download.pytorch.org/whl/nightly/cu${PYTORCH_CUDA}/torch_nightly.html; \
else \
pip install ${PYTORCH_PACKAGE}; \
pip install ${PYTORCH_PACKAGE} -f https://download.pytorch.org/whl/torch_stable.html; \
fi
RUN pip install ${TORCHVISION_PACKAGE} Pillow --no-deps
RUN pip install ${TORCHVISION_PACKAGE} Pillow --no-deps -f https://download.pytorch.org/whl/torch_stable.html

# Install MXNet.
RUN pip install ${MXNET_PACKAGE}
@@ -10,7 +10,7 @@ function build_one()
docker build -t ${tag} --build-arg python=${py} --no-cache .
horovod_version=$(docker run ${tag} pip freeze | grep ^horovod= | awk -F== '{print $2}')
tensorflow_version=$(docker run ${tag} pip freeze | grep ^tensorflow-gpu= | awk -F== '{print $2}')
pytorch_version=$(docker run ${tag} pip freeze | grep ^torch= | awk -F== '{print $2}')
pytorch_version=$(docker run ${tag} pip freeze | grep ^torch= | awk -F== '{print $2}' | awk -F+ '{print $1}')
mxnet_version=$(docker run ${tag} pip freeze | grep ^mxnet | awk -F== '{print $2}')
final_tag=horovod/horovod:${horovod_version}-tf${tensorflow_version}-torch${pytorch_version}-mxnet${mxnet_version}-py${py}
docker tag ${tag} ${final_tag}
@@ -87,7 +87,7 @@ services:
PYTHON_VERSION: 2.7
TENSORFLOW_PACKAGE: tensorflow==1.14.0
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp27-cp27mu-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp27-cp27mu-linux_x86_64.whl
MXNET_PACKAGE: mxnet==1.4.1
PYSPARK_PACKAGE: pyspark==2.4.0
@@ -99,7 +99,7 @@ services:
PYTHON_VERSION: 3.5
TENSORFLOW_PACKAGE: tensorflow==1.14.0
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp35-cp35m-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp35-cp35m-linux_x86_64.whl
MXNET_PACKAGE: mxnet==1.4.1
PYSPARK_PACKAGE: pyspark==2.4.0
@@ -112,7 +112,7 @@ services:
PYTHON_VERSION: 3.6
TENSORFLOW_PACKAGE: tensorflow==1.14.0
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp36-cp36m-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl
MXNET_PACKAGE: mxnet==1.4.1
PYSPARK_PACKAGE: pyspark==2.4.0
@@ -124,7 +124,7 @@ services:
PYTHON_VERSION: 2.7
TENSORFLOW_PACKAGE: tensorflow==1.14.0
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp27-cp27mu-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp27-cp27mu-linux_x86_64.whl
MXNET_PACKAGE: mxnet==1.4.1
PYSPARK_PACKAGE: pyspark==2.4.0
@@ -136,7 +136,7 @@ services:
PYTHON_VERSION: 3.5
TENSORFLOW_PACKAGE: tensorflow==1.14.0
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp35-cp35m-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp35-cp35m-linux_x86_64.whl
MXNET_PACKAGE: mxnet==1.4.1
PYSPARK_PACKAGE: pyspark==2.4.0
@@ -149,7 +149,7 @@ services:
PYTHON_VERSION: 3.6
TENSORFLOW_PACKAGE: tensorflow==1.14.0
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp36-cp36m-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl
MXNET_PACKAGE: mxnet==1.4.1
PYSPARK_PACKAGE: pyspark==2.4.0
@@ -161,7 +161,7 @@ services:
PYTHON_VERSION: 2.7
TENSORFLOW_PACKAGE: tensorflow==1.14.0
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp27-cp27mu-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp27-cp27mu-linux_x86_64.whl
MXNET_PACKAGE: mxnet==1.4.1
PYSPARK_PACKAGE: pyspark==2.4.0
@@ -173,7 +173,7 @@ services:
PYTHON_VERSION: 3.5
TENSORFLOW_PACKAGE: tensorflow==1.14.0
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp35-cp35m-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp35-cp35m-linux_x86_64.whl
MXNET_PACKAGE: mxnet==1.4.1
PYSPARK_PACKAGE: pyspark==2.4.0
@@ -186,35 +186,35 @@ services:
PYTHON_VERSION: 3.6
TENSORFLOW_PACKAGE: tensorflow==1.14.0
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp36-cp36m-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl
MXNET_PACKAGE: mxnet==1.4.1
PYSPARK_PACKAGE: pyspark==2.4.0
test-cpu-openmpi-py2_7-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0:
test-cpu-openmpi-py2_7-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0:
extends: test-cpu-base
build:
args:
MPI_KIND: OpenMPI
PYTHON_VERSION: 2.7
TENSORFLOW_PACKAGE: tf-nightly-2.0-preview
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp27-cp27mu-linux_x86_64.whl
PYTORCH_PACKAGE: torch==1.2.0+cpu
TORCHVISION_PACKAGE: torchvision==0.4.0+cpu
MXNET_PACKAGE: mxnet==1.5.0
PYSPARK_PACKAGE: pyspark==2.4.0
test-cpu-openmpi-py3_5-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0:
test-cpu-openmpi-py3_5-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0:
extends: test-cpu-base
build:
args:
MPI_KIND: OpenMPI
PYTHON_VERSION: 3.5
TENSORFLOW_PACKAGE: tf-nightly-2.0-preview
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp35-cp35m-linux_x86_64.whl
PYTORCH_PACKAGE: torch==1.2.0+cpu
TORCHVISION_PACKAGE: torchvision==0.4.0+cpu
MXNET_PACKAGE: mxnet==1.5.0
PYSPARK_PACKAGE: pyspark==2.4.0
test-cpu-openmpi-py3_6-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0:
test-cpu-openmpi-py3_6-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0:
extends: test-cpu-base
build:
args:
@@ -223,8 +223,8 @@ services:
PYTHON_VERSION: 3.6
TENSORFLOW_PACKAGE: tf-nightly-2.0-preview
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl
PYTORCH_PACKAGE: torch==1.2.0+cpu
TORCHVISION_PACKAGE: torchvision==0.4.0+cpu
MXNET_PACKAGE: mxnet==1.5.0
PYSPARK_PACKAGE: pyspark==2.4.0
test-cpu-openmpi-py2_7-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0:
@@ -261,7 +261,7 @@ services:
PYTHON_VERSION: 3.6
TENSORFLOW_PACKAGE: tensorflow==1.14.0
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp36-cp36m-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl
MXNET_PACKAGE: mxnet==1.5.0
PYSPARK_PACKAGE: pyspark==2.4.0
@@ -274,7 +274,7 @@ services:
PYTHON_VERSION: 3.6
TENSORFLOW_PACKAGE: tensorflow==1.14.0
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp36-cp36m-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl
MXNET_PACKAGE: mxnet==1.5.0
PYSPARK_PACKAGE: pyspark==2.4.0
@@ -394,19 +394,19 @@ services:
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cu100/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl
MXNET_PACKAGE: mxnet-cu100==1.4.1
PYSPARK_PACKAGE: pyspark==2.4.0
test-gpu-openmpi-py2_7-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0:
test-gpu-openmpi-py2_7-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0:
extends: test-gpu-base
build:
args:
MPI_KIND: OpenMPI
PYTHON_VERSION: 2.7
TENSORFLOW_PACKAGE: tf-nightly-gpu-2.0-preview
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cu100/torch-1.1.0-cp27-cp27mu-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cu100/torchvision-0.3.0-cp27-cp27mu-linux_x86_64.whl
PYTORCH_PACKAGE: torch==1.2.0
TORCHVISION_PACKAGE: torchvision==0.4.0
MXNET_PACKAGE: mxnet-cu100==1.5.0
PYSPARK_PACKAGE: pyspark==2.4.0
test-gpu-openmpi-py3_6-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0:
test-gpu-openmpi-py3_6-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0:
extends: test-gpu-base
build:
args:
@@ -415,8 +415,8 @@ services:
PYTHON_VERSION: 3.6
TENSORFLOW_PACKAGE: tf-nightly-gpu-2.0-preview
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cu100/torch-1.1.0-cp36-cp36m-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cu100/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl
PYTORCH_PACKAGE: torch==1.2.0
TORCHVISION_PACKAGE: torchvision==0.4.0
MXNET_PACKAGE: mxnet-cu100==1.5.0
PYSPARK_PACKAGE: pyspark==2.4.0
test-gpu-openmpi-py2_7-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0:

0 comments on commit a77e50c

Please sign in to comment.
You can’t perform that action at this time.