diff --git a/.buildkite/gen-pipeline.sh b/.buildkite/gen-pipeline.sh
index da1cb2d289..af2fb71ce2 100755
--- a/.buildkite/gen-pipeline.sh
+++ b/.buildkite/gen-pipeline.sh
@@ -23,9 +23,9 @@ tests=( \
        test-cpu-openmpi-gloo-py2_7-tf1_14_0-keras2_2_4-torch1_1_0-mxnet1_4_1-pyspark2_4_0 \
        test-cpu-openmpi-gloo-py3_5-tf1_14_0-keras2_2_4-torch1_1_0-mxnet1_4_1-pyspark2_4_0 \
        test-cpu-openmpi-gloo-py3_6-tf1_14_0-keras2_2_4-torch1_1_0-mxnet1_4_1-pyspark2_4_0 \
-       test-cpu-openmpi-py2_7-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0 \
-       test-cpu-openmpi-py3_5-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0 \
-       test-cpu-openmpi-py3_6-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0 \
+       test-cpu-openmpi-py2_7-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0 \
+       test-cpu-openmpi-py3_5-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0 \
+       test-cpu-openmpi-py3_6-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0 \
        test-cpu-openmpi-py2_7-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0 \
        test-cpu-openmpi-py3_6-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0 \
        test-cpu-mpich-py3_6-tf1_14_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0 \
@@ -38,8 +38,8 @@ tests=( \
        test-gpu-gloo-py3_6-tf1_14_0-keras2_2_4-torch1_1_0-mxnet1_4_1-pyspark2_4_0 \
        test-gpu-openmpi-gloo-py2_7-tf1_14_0-keras2_2_4-torch1_1_0-mxnet1_4_1-pyspark2_4_0 \
        test-gpu-openmpi-gloo-py3_6-tf1_14_0-keras2_2_4-torch1_1_0-mxnet1_4_1-pyspark2_4_0 \
-       test-gpu-openmpi-py2_7-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0 \
-       test-gpu-openmpi-py3_6-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0 \
+       test-gpu-openmpi-py2_7-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0 \
+       test-gpu-openmpi-py3_6-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0 \
        test-gpu-openmpi-py2_7-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0 \
        test-gpu-openmpi-py3_6-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0 \
        test-gpu-mpich-py3_6-tf1_14_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0 \
diff --git a/Dockerfile b/Dockerfile
index b241aaf2e0..de717742f8 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -2,8 +2,8 @@ FROM nvidia/cuda:10.0-devel-ubuntu18.04
 
 # TensorFlow version is tightly coupled to CUDA and cuDNN so it should be selected carefully
 ENV TENSORFLOW_VERSION=1.14.0
-ENV PYTORCH_VERSION=1.1.0
-ENV TORCHVISION_VERSION=0.3.0
+ENV PYTORCH_VERSION=1.2.0
+ENV TORCHVISION_VERSION=0.4.0
 ENV CUDNN_VERSION=7.6.0.64-1+cuda10.0
 ENV NCCL_VERSION=2.4.7-1+cuda10.0
 ENV MXNET_VERSION=1.5.0
@@ -50,8 +50,7 @@ RUN pip install numpy \
         tensorflow-gpu==${TENSORFLOW_VERSION} \
         keras \
         h5py
-RUN pip install https://download.pytorch.org/whl/cu100/torch-${PYTORCH_VERSION}-$(python -c "import wheel.pep425tags as w; print('-'.join(w.get_supported()[0]))").whl \
-        https://download.pytorch.org/whl/cu100/torchvision-${TORCHVISION_VERSION}-$(python -c "import wheel.pep425tags as w; print('-'.join(w.get_supported()[0]))").whl
+RUN pip install -f https://download.pytorch.org/whl/torch_stable.html torch==${PYTORCH_VERSION} torchvision==${TORCHVISION_VERSION}
 RUN pip install mxnet-cu100==${MXNET_VERSION}
 
 # Install Open MPI
diff --git a/Dockerfile.test.cpu b/Dockerfile.test.cpu
index 0513bd9203..d95f70d2bd 100644
--- a/Dockerfile.test.cpu
+++ b/Dockerfile.test.cpu
@@ -8,9 +8,9 @@ ARG MPI_KIND=OpenMPI
 ARG PYTHON_VERSION=2.7
 ARG TENSORFLOW_PACKAGE=tensorflow==1.14.0
 ARG KERAS_PACKAGE=keras==2.2.4
-ARG PYTORCH_PACKAGE=torch==1.1.0
-ARG TORCHVISION_PACKAGE=https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp27-cp27mu-linux_x86_64.whl
-ARG MXNET_PACKAGE=mxnet==1.4.1
+ARG PYTORCH_PACKAGE=torch==1.2.0+cpu
+ARG TORCHVISION_PACKAGE=torchvision==0.4.0+cpu
+ARG MXNET_PACKAGE=mxnet==1.5.0
 ARG PYSPARK_PACKAGE=pyspark==2.4.0
 
 # Set default shell to /bin/bash
@@ -108,9 +108,9 @@ RUN pip install future typing
 RUN if [[ ${PYTORCH_PACKAGE} == "torch-nightly" ]]; then \
         pip install torch_nightly -v -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html; \
     else \
-        pip install ${PYTORCH_PACKAGE}; \
+        pip install ${PYTORCH_PACKAGE} -f https://download.pytorch.org/whl/torch_stable.html; \
     fi
-RUN pip install ${TORCHVISION_PACKAGE} Pillow --no-deps
+RUN pip install ${TORCHVISION_PACKAGE} Pillow --no-deps -f https://download.pytorch.org/whl/torch_stable.html
 
 # Install MXNet.
 RUN pip install ${MXNET_PACKAGE}
diff --git a/Dockerfile.test.gpu b/Dockerfile.test.gpu
index 65e80a361d..6eeb97d898 100644
--- a/Dockerfile.test.gpu
+++ b/Dockerfile.test.gpu
@@ -10,9 +10,9 @@ ARG MPI_KIND=OpenMPI
 ARG PYTHON_VERSION=2.7
 ARG TENSORFLOW_PACKAGE=tensorflow-gpu==1.14.0
 ARG KERAS_PACKAGE=keras==2.2.4
-ARG PYTORCH_PACKAGE=https://download.pytorch.org/whl/cu100/torch-1.1.0-cp27-cp27mu-linux_x86_64.whl
-ARG TORCHVISION_PACKAGE=https://download.pytorch.org/whl/cu100/torchvision-0.3.0-cp27-cp27mu-linux_x86_64.whl
-ARG MXNET_PACKAGE=mxnet-cu100==1.4.1
+ARG PYTORCH_PACKAGE=torch==1.2.0
+ARG TORCHVISION_PACKAGE=torchvision==0.4.0
+ARG MXNET_PACKAGE=mxnet-cu100==1.5.0
 ARG PYSPARK_PACKAGE=pyspark==2.4.0
 ARG HOROVOD_BUILD_FLAGS=HOROVOD_GPU_ALLREDUCE=NCCL
 ARG HOROVOD_MIXED_INSTALL=0
@@ -90,9 +90,9 @@ RUN if [[ ${PYTORCH_PACKAGE} == "torch-nightly" ]]; then \
         PYTORCH_CUDA=$(echo ${CUDA_DOCKER_VERSION} | awk -F- '{print $1}' | sed 's/\.//'); \
         pip install torch_nightly -v -f https://download.pytorch.org/whl/nightly/cu${PYTORCH_CUDA}/torch_nightly.html; \
     else \
-        pip install ${PYTORCH_PACKAGE}; \
+        pip install ${PYTORCH_PACKAGE} -f https://download.pytorch.org/whl/torch_stable.html; \
     fi
-RUN pip install ${TORCHVISION_PACKAGE} Pillow --no-deps
+RUN pip install ${TORCHVISION_PACKAGE} Pillow --no-deps -f https://download.pytorch.org/whl/torch_stable.html
 
 # Install MXNet.
 RUN pip install ${MXNET_PACKAGE}
diff --git a/build-docker-images.sh b/build-docker-images.sh
index a2f7b4dc1b..178fe724ac 100755
--- a/build-docker-images.sh
+++ b/build-docker-images.sh
@@ -10,7 +10,7 @@ function build_one()
     docker build -t ${tag} --build-arg python=${py} --no-cache .
     horovod_version=$(docker run ${tag} pip freeze | grep ^horovod= | awk -F== '{print $2}')
     tensorflow_version=$(docker run ${tag} pip freeze | grep ^tensorflow-gpu= | awk -F== '{print $2}')
-    pytorch_version=$(docker run ${tag} pip freeze | grep ^torch= | awk -F== '{print $2}')
+    pytorch_version=$(docker run ${tag} pip freeze | grep ^torch= | awk -F== '{print $2}' | awk -F+ '{print $1}')
     mxnet_version=$(docker run ${tag} pip freeze | grep ^mxnet | awk -F== '{print $2}')
     final_tag=horovod/horovod:${horovod_version}-tf${tensorflow_version}-torch${pytorch_version}-mxnet${mxnet_version}-py${py}
     docker tag ${tag} ${final_tag}
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index b3794f1f46..670ef92f84 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -87,7 +87,7 @@ services:
         PYTHON_VERSION: 2.7
         TENSORFLOW_PACKAGE: tensorflow==1.14.0
         KERAS_PACKAGE: keras==2.2.4
-        PYTORCH_PACKAGE: torch==1.1.0
+        PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp27-cp27mu-linux_x86_64.whl
         TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp27-cp27mu-linux_x86_64.whl
         MXNET_PACKAGE: mxnet==1.4.1
         PYSPARK_PACKAGE: pyspark==2.4.0
@@ -99,7 +99,7 @@ services:
         PYTHON_VERSION: 3.5
         TENSORFLOW_PACKAGE: tensorflow==1.14.0
         KERAS_PACKAGE: keras==2.2.4
-        PYTORCH_PACKAGE: torch==1.1.0
+        PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp35-cp35m-linux_x86_64.whl
         TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp35-cp35m-linux_x86_64.whl
         MXNET_PACKAGE: mxnet==1.4.1
         PYSPARK_PACKAGE: pyspark==2.4.0
@@ -112,7 +112,7 @@ services:
         PYTHON_VERSION: 3.6
         TENSORFLOW_PACKAGE: tensorflow==1.14.0
         KERAS_PACKAGE: keras==2.2.4
-        PYTORCH_PACKAGE: torch==1.1.0
+        PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp36-cp36m-linux_x86_64.whl
         TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl
         MXNET_PACKAGE: mxnet==1.4.1
         PYSPARK_PACKAGE: pyspark==2.4.0
@@ -124,7 +124,7 @@ services:
         PYTHON_VERSION: 2.7
         TENSORFLOW_PACKAGE: tensorflow==1.14.0
         KERAS_PACKAGE: keras==2.2.4
-        PYTORCH_PACKAGE: torch==1.1.0
+        PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp27-cp27mu-linux_x86_64.whl
         TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp27-cp27mu-linux_x86_64.whl
         MXNET_PACKAGE: mxnet==1.4.1
         PYSPARK_PACKAGE: pyspark==2.4.0
@@ -136,7 +136,7 @@ services:
         PYTHON_VERSION: 3.5
         TENSORFLOW_PACKAGE: tensorflow==1.14.0
         KERAS_PACKAGE: keras==2.2.4
-        PYTORCH_PACKAGE: torch==1.1.0
+        PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp35-cp35m-linux_x86_64.whl
         TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp35-cp35m-linux_x86_64.whl
         MXNET_PACKAGE: mxnet==1.4.1
         PYSPARK_PACKAGE: pyspark==2.4.0
@@ -149,7 +149,7 @@ services:
         PYTHON_VERSION: 3.6
         TENSORFLOW_PACKAGE: tensorflow==1.14.0
         KERAS_PACKAGE: keras==2.2.4
-        PYTORCH_PACKAGE: torch==1.1.0
+        PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp36-cp36m-linux_x86_64.whl
         TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl
         MXNET_PACKAGE: mxnet==1.4.1
         PYSPARK_PACKAGE: pyspark==2.4.0
@@ -161,7 +161,7 @@ services:
         PYTHON_VERSION: 2.7
         TENSORFLOW_PACKAGE: tensorflow==1.14.0
         KERAS_PACKAGE: keras==2.2.4
-        PYTORCH_PACKAGE: torch==1.1.0
+        PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp27-cp27mu-linux_x86_64.whl
         TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp27-cp27mu-linux_x86_64.whl
         MXNET_PACKAGE: mxnet==1.4.1
         PYSPARK_PACKAGE: pyspark==2.4.0
@@ -173,7 +173,7 @@ services:
         PYTHON_VERSION: 3.5
         TENSORFLOW_PACKAGE: tensorflow==1.14.0
         KERAS_PACKAGE: keras==2.2.4
-        PYTORCH_PACKAGE: torch==1.1.0
+        PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp35-cp35m-linux_x86_64.whl
         TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp35-cp35m-linux_x86_64.whl
         MXNET_PACKAGE: mxnet==1.4.1
         PYSPARK_PACKAGE: pyspark==2.4.0
@@ -186,11 +186,11 @@ services:
         PYTHON_VERSION: 3.6
         TENSORFLOW_PACKAGE: tensorflow==1.14.0
         KERAS_PACKAGE: keras==2.2.4
-        PYTORCH_PACKAGE: torch==1.1.0
+        PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp36-cp36m-linux_x86_64.whl
         TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl
         MXNET_PACKAGE: mxnet==1.4.1
         PYSPARK_PACKAGE: pyspark==2.4.0
-  test-cpu-openmpi-py2_7-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0:
+  test-cpu-openmpi-py2_7-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0:
     extends: test-cpu-base
     build:
       args:
@@ -198,11 +198,11 @@ services:
         PYTHON_VERSION: 2.7
         TENSORFLOW_PACKAGE: tf-nightly-2.0-preview
         KERAS_PACKAGE: keras==2.2.4
-        PYTORCH_PACKAGE: torch==1.1.0
-        TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp27-cp27mu-linux_x86_64.whl
+        PYTORCH_PACKAGE: torch==1.2.0+cpu
+        TORCHVISION_PACKAGE: torchvision==0.4.0+cpu
         MXNET_PACKAGE: mxnet==1.5.0
         PYSPARK_PACKAGE: pyspark==2.4.0
-  test-cpu-openmpi-py3_5-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0:
+  test-cpu-openmpi-py3_5-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0:
     extends: test-cpu-base
     build:
       args:
@@ -210,11 +210,11 @@ services:
         PYTHON_VERSION: 3.5
         TENSORFLOW_PACKAGE: tf-nightly-2.0-preview
         KERAS_PACKAGE: keras==2.2.4
-        PYTORCH_PACKAGE: torch==1.1.0
-        TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp35-cp35m-linux_x86_64.whl
+        PYTORCH_PACKAGE: torch==1.2.0+cpu
+        TORCHVISION_PACKAGE: torchvision==0.4.0+cpu
         MXNET_PACKAGE: mxnet==1.5.0
         PYSPARK_PACKAGE: pyspark==2.4.0
-  test-cpu-openmpi-py3_6-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0:
+  test-cpu-openmpi-py3_6-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0:
     extends: test-cpu-base
     build:
       args:
@@ -223,8 +223,8 @@ services:
         PYTHON_VERSION: 3.6
         TENSORFLOW_PACKAGE: tf-nightly-2.0-preview
         KERAS_PACKAGE: keras==2.2.4
-        PYTORCH_PACKAGE: torch==1.1.0
-        TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl
+        PYTORCH_PACKAGE: torch==1.2.0+cpu
+        TORCHVISION_PACKAGE: torchvision==0.4.0+cpu
         MXNET_PACKAGE: mxnet==1.5.0
         PYSPARK_PACKAGE: pyspark==2.4.0
   test-cpu-openmpi-py2_7-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0:
@@ -261,7 +261,7 @@ services:
         PYTHON_VERSION: 3.6
         TENSORFLOW_PACKAGE: tensorflow==1.14.0
         KERAS_PACKAGE: keras==2.2.4
-        PYTORCH_PACKAGE: torch==1.1.0
+        PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp36-cp36m-linux_x86_64.whl
         TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl
         MXNET_PACKAGE: mxnet==1.5.0
         PYSPARK_PACKAGE: pyspark==2.4.0
@@ -274,7 +274,7 @@ services:
         PYTHON_VERSION: 3.6
         TENSORFLOW_PACKAGE: tensorflow==1.14.0
         KERAS_PACKAGE: keras==2.2.4
-        PYTORCH_PACKAGE: torch==1.1.0
+        PYTORCH_PACKAGE: https://download.pytorch.org/whl/cpu/torch-1.1.0-cp36-cp36m-linux_x86_64.whl
         TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cpu/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl
         MXNET_PACKAGE: mxnet==1.5.0
         PYSPARK_PACKAGE: pyspark==2.4.0
@@ -394,7 +394,7 @@ services:
         TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cu100/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl
         MXNET_PACKAGE: mxnet-cu100==1.4.1
         PYSPARK_PACKAGE: pyspark==2.4.0
-  test-gpu-openmpi-py2_7-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0:
+  test-gpu-openmpi-py2_7-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0:
     extends: test-gpu-base
     build:
       args:
@@ -402,11 +402,11 @@ services:
         PYTHON_VERSION: 2.7
         TENSORFLOW_PACKAGE: tf-nightly-gpu-2.0-preview
         KERAS_PACKAGE: keras==2.2.4
-        PYTORCH_PACKAGE: https://download.pytorch.org/whl/cu100/torch-1.1.0-cp27-cp27mu-linux_x86_64.whl
-        TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cu100/torchvision-0.3.0-cp27-cp27mu-linux_x86_64.whl
+        PYTORCH_PACKAGE: torch==1.2.0
+        TORCHVISION_PACKAGE: torchvision==0.4.0
         MXNET_PACKAGE: mxnet-cu100==1.5.0
         PYSPARK_PACKAGE: pyspark==2.4.0
-  test-gpu-openmpi-py3_6-tf2_0_0-keras2_2_4-torch1_1_0-mxnet1_5_0-pyspark2_4_0:
+  test-gpu-openmpi-py3_6-tf2_0_0-keras2_2_4-torch1_2_0-mxnet1_5_0-pyspark2_4_0:
     extends: test-gpu-base
     build:
       args:
@@ -415,8 +415,8 @@ services:
         PYTHON_VERSION: 3.6
         TENSORFLOW_PACKAGE: tf-nightly-gpu-2.0-preview
         KERAS_PACKAGE: keras==2.2.4
-        PYTORCH_PACKAGE: https://download.pytorch.org/whl/cu100/torch-1.1.0-cp36-cp36m-linux_x86_64.whl
-        TORCHVISION_PACKAGE: https://download.pytorch.org/whl/cu100/torchvision-0.3.0-cp36-cp36m-linux_x86_64.whl
+        PYTORCH_PACKAGE: torch==1.2.0
+        TORCHVISION_PACKAGE: torchvision==0.4.0
         MXNET_PACKAGE: mxnet-cu100==1.5.0
         PYSPARK_PACKAGE: pyspark==2.4.0
   test-gpu-openmpi-py2_7-tfhead-kerashead-torchhead-mxnethead-pyspark2_4_0: