Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update PyTorch to 1.2.0 #1320

Merged
merged 4 commits into from Aug 20, 2019
Merged

Update PyTorch to 1.2.0 #1320

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b3ee14e
Update PyTorch for Dockerfile
alsrgv Aug 20, 2019
b61e1a2
Switch CI to PyTorch 1.2.0 for latest release
alsrgv Aug 20, 2019
6d24160
Update PyTorch package links
alsrgv Aug 20, 2019
e4051b7
Split torch tag
alsrgv Aug 20, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
10 changes: 5 additions & 5 deletions .buildkite/gen-pipeline.sh
View file
Open in desktop
Expand Up @@ -23,9 +23,9 @@ tests=( \
test-cpu-openmpi-gloo-py2_7-tf1_14_0-keras2_2_4-torch1_1_0-mxnet1_4_1-pyspark2_4_0 \
test-cpu-openmpi-gloo-py3_5-tf1_14_0-keras2_2_4-torch1_1_0-mxnet1_4_1-pyspark2_4_0 \
test-cpu-openmpi-gloo-py3_6-tf1_14_0-keras2_2_4-torch1_1_0-mxnet1_4_1-pyspark2_4_0 \
test-cpu-openmpi-py2_7-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0 \
test-cpu-openmpi-py3_5-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0 \
test-cpu-openmpi-py3_6-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0 \
test-cpu-openmpi-py2_7-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0 \
test-cpu-openmpi-py3_5-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0 \
test-cpu-openmpi-py3_6-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0 \
test-cpu-openmpi-py2_7-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0 \
test-cpu-openmpi-py3_6-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0 \
test-cpu-mpich-py3_6-tf1_14_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0 \
Expand All @@ -38,8 +38,8 @@ tests=( \
test-gpu-gloo-py3_6-tf1_14_0-keras2_2_4-torch1_1_0-mxnet1_4_1-pyspark2_4_0 \
test-gpu-openmpi-gloo-py2_7-tf1_14_0-keras2_2_4-torch1_1_0-mxnet1_4_1-pyspark2_4_0 \
test-gpu-openmpi-gloo-py3_6-tf1_14_0-keras2_2_4-torch1_1_0-mxnet1_4_1-pyspark2_4_0 \
test-gpu-openmpi-py2_7-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0 \
test-gpu-openmpi-py3_6-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0 \
test-gpu-openmpi-py2_7-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0 \
test-gpu-openmpi-py3_6-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0 \
test-gpu-openmpi-py2_7-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0 \
test-gpu-openmpi-py3_6-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0 \
test-gpu-mpich-py3_6-tf1_14_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0 \
Expand Down
7 changes: 3 additions & 4 deletions Dockerfile
View file
Open in desktop
Expand Up @@ -2,8 +2,8 @@ FROM nvidia/cuda:10.0-devel-ubuntu18.04

# TensorFlow version is tightly coupled to CUDA and cuDNN so it should be selected carefully
ENV TENSORFLOW_VERSION=1.14.0
ENV PYTORCH_VERSION=1.1.0
ENV TORCHVISION_VERSION=0.3.0
ENV PYTORCH_VERSION=1.2.0
ENV TORCHVISION_VERSION=0.4.0
ENV CUDNN_VERSION=7.6.0.64-1+cuda10.0
ENV NCCL_VERSION=2.4.7-1+cuda10.0
ENV MXNET_VERSION=1.5.0
Expand Down Expand Up @@ -50,8 +50,7 @@ RUN pip install numpy \
tensorflow-gpu==${TENSORFLOW_VERSION} \
keras \
h5py
RUN pip install https://download.pytorch.org/whl/cu100/torch-${PYTORCH_VERSION}-$(python -c "import wheel.pep425tags as w; print('-'.join(w.get_supported()[0]))").whl \
https://download.pytorch.org/whl/cu100/torchvision-${TORCHVISION_VERSION}-$(python -c "import wheel.pep425tags as w; print('-'.join(w.get_supported()[0]))").whl
RUN pip install -f https://download.pytorch.org/whl/torch_stable.html torch==${PYTORCH_VERSION} torchvision==${TORCHVISION_VERSION}
RUN pip install mxnet-cu100==${MXNET_VERSION}

# Install Open MPI
Expand Down
10 changes: 5 additions & 5 deletions Dockerfile.test.cpu
View file
Open in desktop
Expand Up @@ -8,9 +8,9 @@ ARG MPI_KIND=OpenMPI
ARG PYTHON_VERSION=2.7
ARG TENSORFLOW_PACKAGE=tensorflow==1.14.0
ARG KERAS_PACKAGE=keras==2.2.4
ARG PYTORCH_PACKAGE=torch==1.1.0
ARG TORCHVISION_PACKAGE=https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp27-cp27mu-linux_x86_64.whl
ARG MXNET_PACKAGE=mxnet==1.4.1
ARG PYTORCH_PACKAGE=torch==1.2.0+cpu
ARG TORCHVISION_PACKAGE=torchvision==0.4.0+cpu
ARG MXNET_PACKAGE=mxnet==1.5.0
romerojosh marked this conversation as resolved.
Show resolved Hide resolved
ARG PYSPARK_PACKAGE=pyspark==2.4.0

# Set default shell to /bin/bash
Expand Down Expand Up @@ -108,9 +108,9 @@ RUN pip install future typing
RUN if [[ ${PYTORCH_PACKAGE} == "torch-nightly" ]]; then \
pip install torch_nightly -v -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html; \
else \
pip install ${PYTORCH_PACKAGE}; \
pip install ${PYTORCH_PACKAGE} -f https://download.pytorch.org/whl/torch_stable.html; \
fi
RUN pip install ${TORCHVISION_PACKAGE} Pillow --no-deps
RUN pip install ${TORCHVISION_PACKAGE} Pillow --no-deps -f https://download.pytorch.org/whl/torch_stable.html

# Install MXNet.
RUN pip install ${MXNET_PACKAGE}
Expand Down
10 changes: 5 additions & 5 deletions Dockerfile.test.gpu
View file
Open in desktop
Expand Up @@ -10,9 +10,9 @@ ARG MPI_KIND=OpenMPI
ARG PYTHON_VERSION=2.7
ARG TENSORFLOW_PACKAGE=tensorflow-gpu==1.14.0
ARG KERAS_PACKAGE=keras==2.2.4
ARG PYTORCH_PACKAGE=https://download.pytorch.org/whl/cu100/torch-1.1.0-cp27-cp27mu-linux_x86_64.whl
ARG TORCHVISION_PACKAGE=https://download.pytorch.org/whl/cu100/torchvision-0.3.0-cp27-cp27mu-linux_x86_64.whl
ARG MXNET_PACKAGE=mxnet-cu100==1.4.1
ARG PYTORCH_PACKAGE=torch==1.2.0
ARG TORCHVISION_PACKAGE=torchvision==0.4.0
ARG MXNET_PACKAGE=mxnet-cu100==1.5.0
ARG PYSPARK_PACKAGE=pyspark==2.4.0
ARG HOROVOD_BUILD_FLAGS=HOROVOD_GPU_ALLREDUCE=NCCL
ARG HOROVOD_MIXED_INSTALL=0
Expand Down Expand Up @@ -90,9 +90,9 @@ RUN if [[ ${PYTORCH_PACKAGE} == "torch-nightly" ]]; then \
PYTORCH_CUDA=$(echo ${CUDA_DOCKER_VERSION} | awk -F- '{print $1}' | sed 's/\.//'); \
pip install torch_nightly -v -f https://download.pytorch.org/whl/nightly/cu${PYTORCH_CUDA}/torch_nightly.html; \
else \
pip install ${PYTORCH_PACKAGE}; \
pip install ${PYTORCH_PACKAGE} -f https://download.pytorch.org/whl/torch_stable.html; \
fi
RUN pip install ${TORCHVISION_PACKAGE} Pillow --no-deps
RUN pip install ${TORCHVISION_PACKAGE} Pillow --no-deps -f https://download.pytorch.org/whl/torch_stable.html

# Install MXNet.
RUN pip install ${MXNET_PACKAGE}
Expand Down
2 changes: 1 addition & 1 deletion build-docker-images.sh
View file
Open in desktop
Expand Up @@ -10,7 +10,7 @@ function build_one()
docker build -t ${tag} --build-arg python=${py} --no-cache .
horovod_version=$(docker run ${tag} pip freeze | grep ^horovod= | awk -F== '{print $2}')
tensorflow_version=$(docker run ${tag} pip freeze | grep ^tensorflow-gpu= | awk -F== '{print $2}')
pytorch_version=$(docker run ${tag} pip freeze | grep ^torch= | awk -F== '{print $2}')
pytorch_version=$(docker run ${tag} pip freeze | grep ^torch= | awk -F== '{print $2}' | awk -F+ '{print $1}')
mxnet_version=$(docker run ${tag} pip freeze | grep ^mxnet | awk -F== '{print $2}')
final_tag=horovod/horovod:${horovod_version}-tf${tensorflow_version}-torch${pytorch_version}-mxnet${mxnet_version}-py${py}
docker tag ${tag} ${final_tag}
Expand Down
52 changes: 26 additions & 26 deletions docker-compose.test.yml
View file
Open in desktop
Expand Up @@ -87,7 +87,7 @@ services:
PYTHON_VERSION: 2.7
TENSORFLOW_PACKAGE: tensorflow==1.14.0
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp27-cp27mu-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp27-cp27mu-linux_x86_64.whl
MXNET_PACKAGE: mxnet==1.4.1
PYSPARK_PACKAGE: pyspark==2.4.0
Expand All @@ -99,7 +99,7 @@ services:
PYTHON_VERSION: 3.5
TENSORFLOW_PACKAGE: tensorflow==1.14.0
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp35-cp35m-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp35-cp35m-linux_x86_64.whl
MXNET_PACKAGE: mxnet==1.4.1
PYSPARK_PACKAGE: pyspark==2.4.0
Expand All @@ -112,7 +112,7 @@ services:
PYTHON_VERSION: 3.6
TENSORFLOW_PACKAGE: tensorflow==1.14.0
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp36-cp36m-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl
MXNET_PACKAGE: mxnet==1.4.1
PYSPARK_PACKAGE: pyspark==2.4.0
Expand All @@ -124,7 +124,7 @@ services:
PYTHON_VERSION: 2.7
TENSORFLOW_PACKAGE: tensorflow==1.14.0
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp27-cp27mu-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp27-cp27mu-linux_x86_64.whl
MXNET_PACKAGE: mxnet==1.4.1
PYSPARK_PACKAGE: pyspark==2.4.0
Expand All @@ -136,7 +136,7 @@ services:
PYTHON_VERSION: 3.5
TENSORFLOW_PACKAGE: tensorflow==1.14.0
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp35-cp35m-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp35-cp35m-linux_x86_64.whl
MXNET_PACKAGE: mxnet==1.4.1
PYSPARK_PACKAGE: pyspark==2.4.0
Expand All @@ -149,7 +149,7 @@ services:
PYTHON_VERSION: 3.6
TENSORFLOW_PACKAGE: tensorflow==1.14.0
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp36-cp36m-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl
MXNET_PACKAGE: mxnet==1.4.1
PYSPARK_PACKAGE: pyspark==2.4.0
Expand All @@ -161,7 +161,7 @@ services:
PYTHON_VERSION: 2.7
TENSORFLOW_PACKAGE: tensorflow==1.14.0
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp27-cp27mu-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp27-cp27mu-linux_x86_64.whl
MXNET_PACKAGE: mxnet==1.4.1
PYSPARK_PACKAGE: pyspark==2.4.0
Expand All @@ -173,7 +173,7 @@ services:
PYTHON_VERSION: 3.5
TENSORFLOW_PACKAGE: tensorflow==1.14.0
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp35-cp35m-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp35-cp35m-linux_x86_64.whl
MXNET_PACKAGE: mxnet==1.4.1
PYSPARK_PACKAGE: pyspark==2.4.0
Expand All @@ -186,35 +186,35 @@ services:
PYTHON_VERSION: 3.6
TENSORFLOW_PACKAGE: tensorflow==1.14.0
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp36-cp36m-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl
MXNET_PACKAGE: mxnet==1.4.1
PYSPARK_PACKAGE: pyspark==2.4.0
test-cpu-openmpi-py2_7-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0:
test-cpu-openmpi-py2_7-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0:
extends: test-cpu-base
build:
args:
MPI_KIND: OpenMPI
PYTHON_VERSION: 2.7
TENSORFLOW_PACKAGE: tf-nightly-2.0-preview
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp27-cp27mu-linux_x86_64.whl
PYTORCH_PACKAGE: torch==1.2.0+cpu
TORCHVISION_PACKAGE: torchvision==0.4.0+cpu
MXNET_PACKAGE: mxnet==1.5.0
PYSPARK_PACKAGE: pyspark==2.4.0
test-cpu-openmpi-py3_5-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0:
test-cpu-openmpi-py3_5-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0:
extends: test-cpu-base
build:
args:
MPI_KIND: OpenMPI
PYTHON_VERSION: 3.5
TENSORFLOW_PACKAGE: tf-nightly-2.0-preview
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp35-cp35m-linux_x86_64.whl
PYTORCH_PACKAGE: torch==1.2.0+cpu
TORCHVISION_PACKAGE: torchvision==0.4.0+cpu
MXNET_PACKAGE: mxnet==1.5.0
PYSPARK_PACKAGE: pyspark==2.4.0
test-cpu-openmpi-py3_6-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0:
test-cpu-openmpi-py3_6-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0:
extends: test-cpu-base
build:
args:
Expand All @@ -223,8 +223,8 @@ services:
PYTHON_VERSION: 3.6
TENSORFLOW_PACKAGE: tf-nightly-2.0-preview
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl
PYTORCH_PACKAGE: torch==1.2.0+cpu
TORCHVISION_PACKAGE: torchvision==0.4.0+cpu
MXNET_PACKAGE: mxnet==1.5.0
PYSPARK_PACKAGE: pyspark==2.4.0
test-cpu-openmpi-py2_7-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0:
Expand Down Expand Up @@ -261,7 +261,7 @@ services:
PYTHON_VERSION: 3.6
TENSORFLOW_PACKAGE: tensorflow==1.14.0
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp36-cp36m-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl
MXNET_PACKAGE: mxnet==1.5.0
PYSPARK_PACKAGE: pyspark==2.4.0
Expand All @@ -274,7 +274,7 @@ services:
PYTHON_VERSION: 3.6
TENSORFLOW_PACKAGE: tensorflow==1.14.0
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: torch==1.1.0
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp36-cp36m-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl
MXNET_PACKAGE: mxnet==1.5.0
PYSPARK_PACKAGE: pyspark==2.4.0
Expand Down Expand Up @@ -394,19 +394,19 @@ services:
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cu100/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl
MXNET_PACKAGE: mxnet-cu100==1.4.1
PYSPARK_PACKAGE: pyspark==2.4.0
test-gpu-openmpi-py2_7-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0:
test-gpu-openmpi-py2_7-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0:
extends: test-gpu-base
build:
args:
MPI_KIND: OpenMPI
PYTHON_VERSION: 2.7
TENSORFLOW_PACKAGE: tf-nightly-gpu-2.0-preview
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cu100/torch-1.1.0-cp27-cp27mu-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cu100/torchvision-0.3.0-cp27-cp27mu-linux_x86_64.whl
PYTORCH_PACKAGE: torch==1.2.0
TORCHVISION_PACKAGE: torchvision==0.4.0
MXNET_PACKAGE: mxnet-cu100==1.5.0
PYSPARK_PACKAGE: pyspark==2.4.0
test-gpu-openmpi-py3_6-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0:
test-gpu-openmpi-py3_6-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0:
extends: test-gpu-base
build:
args:
Expand All @@ -415,8 +415,8 @@ services:
PYTHON_VERSION: 3.6
TENSORFLOW_PACKAGE: tf-nightly-gpu-2.0-preview
KERAS_PACKAGE: keras==2.2.4
PYTORCH_PACKAGE: https://download.pytorch.org/whl/cu100/torch-1.1.0-cp36-cp36m-linux_x86_64.whl
TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cu100/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl
PYTORCH_PACKAGE: torch==1.2.0
TORCHVISION_PACKAGE: torchvision==0.4.0
MXNET_PACKAGE: mxnet-cu100==1.5.0
PYSPARK_PACKAGE: pyspark==2.4.0
test-gpu-openmpi-py2_7-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0:
Expand Down