diff --git a/common/install_mnist.sh b/common/install_mnist.sh
index b325e0027..b0d6937e1 100644
--- a/common/install_mnist.sh
+++ b/common/install_mnist.sh
@@ -7,6 +7,6 @@ mkdir -p /usr/local/mnist/
 cd /usr/local/mnist
 
 for img in train-images-idx3-ubyte.gz train-labels-idx1-ubyte.gz t10k-images-idx3-ubyte.gz t10k-labels-idx1-ubyte.gz; do
-  wget -q http://yann.lecun.com/exdb/mnist/$img
+  wget -q https://ossci-datasets.s3.amazonaws.com/mnist/$img
   gzip -d $img
 done
diff --git a/common/install_openssl.sh b/common/install_openssl.sh
index ea234d46d..de1dcd797 100644
--- a/common/install_openssl.sh
+++ b/common/install_openssl.sh
@@ -7,7 +7,8 @@ OPENSSL=openssl-1.1.1k
 wget -q -O ${OPENSSL}.tar.gz https://www.openssl.org/source/${OPENSSL}.tar.gz
 tar xf ${OPENSSL}.tar.gz
 cd ${OPENSSL}
-./config -d '-Wl,--enable-new-dtags,-rpath,$(LIBRPATH)'
-make install
+./config --prefix=/opt/openssl -d '-Wl,--enable-new-dtags,-rpath,$(LIBRPATH)'
+# NOTE: opensl errors out when built with the -j option
+make install_sw
 cd ..
 rm -rf ${OPENSSL}
diff --git a/conda/Dockerfile b/conda/Dockerfile
index d411dc1b0..3b6cc90d0 100644
--- a/conda/Dockerfile
+++ b/conda/Dockerfile
@@ -83,14 +83,14 @@ COPY --from=cuda11.0  /usr/local/cuda-11.0 /usr/local/cuda-11.0
 COPY --from=cuda11.1  /usr/local/cuda-11.1 /usr/local/cuda-11.1
 COPY --from=cuda11.2  /usr/local/cuda-11.2 /usr/local/cuda-11.2
 
-# Install LLVM
-COPY --from=pytorch/llvm:9.0.1 /opt/llvm /opt/llvm
-COPY --from=pytorch/llvm:9.0.1 /opt/llvm_no_cxx_abi /opt/llvm_no_cxx_abi
-
 FROM ${BASE_TARGET} as final
-COPY --from=patchelf /patchelf            /usr/local/bin/patchelf
-COPY --from=conda    /opt/conda           /opt/conda
-ADD  ./java/jni.h    /usr/local/include/jni.h
+# Install LLVM
+COPY --from=pytorch/llvm:9.0.1 /opt/llvm              /opt/llvm
+COPY --from=pytorch/llvm:9.0.1 /opt/llvm_no_cxx11_abi /opt/llvm_no_cxx11_abi
+COPY --from=openssl            /opt/openssl           /opt/openssl
+COPY --from=patchelf           /patchelf              /usr/local/bin/patchelf
+COPY --from=conda              /opt/conda             /opt/conda
+ADD  ./java/jni.h /usr/local/include/jni.h
 ENV  PATH /opt/conda/bin:$PATH
 COPY --from=mnist  /usr/local/mnist /usr/local/mnist
 RUN rm -rf /usr/local/cuda
diff --git a/conda/build_all_docker.sh b/conda/build_all_docker.sh
index 2e45fef03..fcb92becf 100755
--- a/conda/build_all_docker.sh
+++ b/conda/build_all_docker.sh
@@ -4,6 +4,6 @@ set -eou pipefail
 
 TOPDIR=$(git rev-parse --show-toplevel)
 
-for CUDA_VERSION in 11.2 11.1 11.0 10.2 10.1 cpu; do
+for CUDA_VERSION in 11.1 10.2 cpu; do
   CUDA_VERSION="${CUDA_VERSION}" conda/build_docker.sh
 done
diff --git a/conda/build_docker.sh b/conda/build_docker.sh
index 082a2669c..3a74267b8 100755
--- a/conda/build_docker.sh
+++ b/conda/build_docker.sh
@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -eou pipefail
+
 export DOCKER_BUILDKIT=1
 TOPDIR=$(git rev-parse --show-toplevel)
 
@@ -31,11 +33,21 @@ esac
     ${TOPDIR}
 )
 
-if [[ ${DOCKER_TAG} =~ ^cuda* ]]; then
+if [[ "${DOCKER_TAG}" =~ ^cuda* ]]; then
   # Meant for legacy scripts since they only do the version without the "."
   # TODO: Eventually remove this
   (
     set -x
-    docker tag "pytorch/conda-builder:${DOCKER_TAG}" pytorch/conda-builder:cuda${CUDA_VERSION/./}
+    docker tag "pytorch/conda-builder:${DOCKER_TAG}" "pytorch/conda-builder:cuda${CUDA_VERSION/./}"
+  )
+fi
+
+if [[ -n "${WITH_PUSH:-}" ]]; then
+  (
+    set -x
+    docker push "pytorch/conda-builder:${DOCKER_TAG}"
+    if [[ "${DOCKER_TAG}" =~ ^cuda* ]]; then
+      docker push "pytorch/conda-builder:cuda${CUDA_VERSION/./}"
+    fi
   )
 fi
diff --git a/libtorch/build_all_docker.sh b/libtorch/build_all_docker.sh
index 2e45fef03..93fb19a0f 100755
--- a/libtorch/build_all_docker.sh
+++ b/libtorch/build_all_docker.sh
@@ -4,6 +4,6 @@ set -eou pipefail
 
 TOPDIR=$(git rev-parse --show-toplevel)
 
-for CUDA_VERSION in 11.2 11.1 11.0 10.2 10.1 cpu; do
-  CUDA_VERSION="${CUDA_VERSION}" conda/build_docker.sh
+for CUDA_VERSION in 11.1 10.2 cpu; do
+  CUDA_VERSION="${CUDA_VERSION}" libtorch/build_docker.sh
 done
diff --git a/libtorch/build_docker.sh b/libtorch/build_docker.sh
index b21cae3ae..f8ab39ff5 100755
--- a/libtorch/build_docker.sh
+++ b/libtorch/build_docker.sh
@@ -26,3 +26,10 @@ esac
     -f "${TOPDIR}/libtorch/ubuntu16.04/Dockerfile" \
     ${TOPDIR}
 )
+
+if [[ -n "${WITH_PUSH:-}" ]]; then
+  (
+    set -x
+    docker push pytorch/libtorch-cxx11-builder:${DOCKER_TAG}
+  )
+fi
diff --git a/libtorch/ubuntu16.04/Dockerfile b/libtorch/ubuntu16.04/Dockerfile
index 9585d8183..845f7e8b5 100644
--- a/libtorch/ubuntu16.04/Dockerfile
+++ b/libtorch/ubuntu16.04/Dockerfile
@@ -11,6 +11,7 @@ ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US.UTF-8
 
 # Install openssl
+FROM base as openssl
 ADD ./common/install_openssl.sh install_openssl.sh
 RUN bash ./install_openssl.sh && rm install_openssl.sh
 
@@ -53,11 +54,11 @@ FROM cuda as cuda11.2
 RUN bash ./install_cuda.sh 11.2
 RUN bash ./install_magma.sh 11.2
 
-# Install LLVM
-COPY --from=pytorch/llvm:9.0.1 /opt/llvm /opt/llvm
-COPY --from=pytorch/llvm:9.0.1 /opt/llvm_no_cxx_abi /opt/llvm_no_cxx_abi
-
 FROM ${BASE_TARGET} as final
+# Install LLVM
+COPY --from=pytorch/llvm:9.0.1 /opt/llvm              /opt/llvm
+COPY --from=pytorch/llvm:9.0.1 /opt/llvm_no_cxx11_abi /opt/llvm_no_cxx11_abi
+COPY --from=openssl            /opt/openssl           /opt/openssl
 # Install patchelf
 ADD ./common/install_patchelf.sh install_patchelf.sh
 RUN bash ./install_patchelf.sh && rm install_patchelf.sh
diff --git a/llvm/Dockerfile b/llvm/Dockerfile
index 243038c49..51ef9726c 100644
--- a/llvm/Dockerfile
+++ b/llvm/Dockerfile
@@ -44,6 +44,6 @@ RUN cmake -G "Unix Makefiles" \
 RUN make -j"$(nproc --ignore=2)" && make install
 
 
-FROM scratch as final
+FROM alpine as final
 COPY --from=dev /opt/llvm /opt/llvm
 COPY --from=dev /opt/llvm_no_cxx11_abi /opt/llvm_no_cxx11_abi
diff --git a/manywheel/Dockerfile b/manywheel/Dockerfile
index 68d700c61..a08f67426 100644
--- a/manywheel/Dockerfile
+++ b/manywheel/Dockerfile
@@ -108,18 +108,22 @@ RUN yum install -y \
 RUN yum swap -y git git224-core
 
 ENV SSL_CERT_FILE=/opt/_internal/certs.pem
-COPY --from=python   /opt/python                           /opt/python
-COPY --from=python   /opt/_internal                        /opt/_internal
-COPY --from=python   /opt/python/cp36-cp36m/bin/auditwheel /usr/local/bin/auditwheel
-COPY --from=intel    /opt/intel                            /opt/intel
-COPY --from=patchelf /usr/local/bin/patchelf               /usr/local/bin/patchelf
-COPY --from=jni      /usr/local/include/jni.h              /usr/local/include/jni.h
-COPY --from=libpng   /usr/local/bin/png*                   /usr/local/bin/
-COPY --from=libpng   /usr/local/bin/libpng*                /usr/local/bin/
-COPY --from=libpng   /usr/local/include/png*               /usr/local/include/
-COPY --from=libpng   /usr/local/include/libpng*            /usr/local/include/
-COPY --from=libpng   /usr/local/lib/libpng*                /usr/local/lib/
-COPY --from=libpng   /usr/local/lib/pkgconfig              /usr/local/lib/pkgconfig
+# Install LLVM version
+COPY --from=pytorch/llvm:9.0.1 /opt/llvm                             /opt/llvm
+COPY --from=pytorch/llvm:9.0.1 /opt/llvm_no_cxx11_abi                /opt/llvm_no_cxx11_abi
+COPY --from=openssl            /opt/openssl                          /opt/openssl
+COPY --from=python             /opt/python                           /opt/python
+COPY --from=python             /opt/_internal                        /opt/_internal
+COPY --from=python             /opt/python/cp36-cp36m/bin/auditwheel /usr/local/bin/auditwheel
+COPY --from=intel              /opt/intel                            /opt/intel
+COPY --from=patchelf           /usr/local/bin/patchelf               /usr/local/bin/patchelf
+COPY --from=jni                /usr/local/include/jni.h              /usr/local/include/jni.h
+COPY --from=libpng             /usr/local/bin/png*                   /usr/local/bin/
+COPY --from=libpng             /usr/local/bin/libpng*                /usr/local/bin/
+COPY --from=libpng             /usr/local/include/png*               /usr/local/include/
+COPY --from=libpng             /usr/local/include/libpng*            /usr/local/include/
+COPY --from=libpng             /usr/local/lib/libpng*                /usr/local/lib/
+COPY --from=libpng             /usr/local/lib/pkgconfig              /usr/local/lib/pkgconfig
 
 FROM common as cpu_final
 ARG BASE_CUDA_VERSION=10.1
@@ -138,10 +142,6 @@ RUN rm -rf /usr/local/cuda-${BASE_CUDA_VERSION}
 COPY --from=cuda     /usr/local/cuda-${BASE_CUDA_VERSION}  /usr/local/cuda-${BASE_CUDA_VERSION}
 COPY --from=magma    /usr/local/cuda-${BASE_CUDA_VERSION}  /usr/local/cuda-${BASE_CUDA_VERSION}
 
-# Install LLVM version
-COPY --from=pytorch/llvm:9.0.1 /opt/llvm /opt/llvm
-COPY --from=pytorch/llvm:9.0.1 /opt/llvm_no_cxx11_abi /opt/llvm_no_cxx11_abi
-
 FROM common as rocm_final
 ARG ROCM_VERSION=3.7
 # Install ROCm
diff --git a/manywheel/build_scripts/build.sh b/manywheel/build_scripts/build.sh
index cc84e1221..7b374bd88 100644
--- a/manywheel/build_scripts/build.sh
+++ b/manywheel/build_scripts/build.sh
@@ -7,6 +7,10 @@ set -ex
 # Python versions to be installed in /opt/$VERSION_NO
 CPYTHON_VERSIONS=${CPYTHON_VERSIONS:-"3.6.6 3.7.5 3.8.1 3.9.0"}
 
+# openssl version to build, with expected sha256 hash of .tar.gz
+# archive
+OPENSSL_ROOT=openssl-1.0.2k
+OPENSSL_HASH=6b3977c61f2aedf0f96367dcfb5c6e578cf37e7b8d913b4ecb6643c3cb88d8c0
 DEVTOOLS_HASH=a8ebeb4bed624700f727179e6ef771dafe47651131a00a78b342251415646acc
 PATCHELF_HASH=d9afdff4baeacfbc64861454f368b7f2c15c44d245293f7587bbf726bfe722fb
 CURL_ROOT=curl-7.49.1
@@ -36,6 +40,10 @@ build_autoconf $AUTOCONF_ROOT $AUTOCONF_HASH
 autoconf --version
 
 # Compile the latest Python releases.
+# (In order to have a proper SSL module, Python is compiled
+# against a recent openssl [see env vars above], which is linked
+# statically. We delete openssl afterwards.)
+build_openssl $OPENSSL_ROOT $OPENSSL_HASH
 mkdir -p /opt/python
 build_cpythons $CPYTHON_VERSIONS
 
diff --git a/manywheel/build_scripts/build_utils.sh b/manywheel/build_scripts/build_utils.sh
index e957addfb..cef47640c 100755
--- a/manywheel/build_scripts/build_utils.sh
+++ b/manywheel/build_scripts/build_utils.sh
@@ -2,6 +2,10 @@
 # Helper utilities for build
 
 PYTHON_DOWNLOAD_URL=https://www.python.org/ftp/python
+# XXX: the official https server at www.openssl.org cannot be reached
+# with the old versions of openssl and curl in Centos 5.11 hence the fallback
+# to the ftp mirror:
+OPENSSL_DOWNLOAD_URL=ftp://ftp.openssl.org/source/old/1.0.2/
 # Ditto the curl sources
 CURL_DOWNLOAD_URL=http://curl.askapache.com/download
 
@@ -90,7 +94,8 @@ function build_cpythons {
 
 
 function do_openssl_build {
-    ./config -d '-Wl,--enable-new-dtags,-rpath,$(LIBRPATH)' > /dev/null
+    ./config no-ssl2 no-shared -fPIC --prefix=/usr/local/ssl > /dev/null
+    make > /dev/null
     make install > /dev/null
 }
 
diff --git a/manywheel/deploy.sh b/manywheel/deploy.sh
index 9b5af69f6..34f042bf0 100755
--- a/manywheel/deploy.sh
+++ b/manywheel/deploy.sh
@@ -2,31 +2,35 @@
 
 set -eou pipefail
 
-for rocm_version in 3.7 3.8 3.9 3.10 4.0 4.0.1 4.1; do
-    (
-        set -x
-        DOCKER_BUILDKIT=1 docker build \
-            -t "pytorch/manylinux-rocm:${rocm_version}" \
-            --build-arg "ROCM_VERSION=${rocm_version}" \
-            --build-arg "GPU_IMAGE=rocm/dev-centos-7:${rocm_version}" \
-            --target rocm_final \
-            -f manywheel/Dockerfile \
-            .
-        docker push "pytorch/manylinux-rocm:${rocm_version}"
-    )
-done
+DOCKER_REGISTRY="${DOCKER_REGISTRY:-docker.io}"
+
+# for rocm_version in 4.0.1 4.1; do
+#     DOCKER_IMAGE="${DOCKER_REGISTRY}/pytorch/manylinux-rocm:${rocm_version}"
+#     (
+#         set -x
+#         DOCKER_BUILDKIT=1 docker build \
+#             -t "${DOCKER_IMAGE}" \
+#             --build-arg "ROCM_VERSION=${rocm_version}" \
+#             --build-arg "GPU_IMAGE=rocm/dev-centos-7:${rocm_version}" \
+#             --target rocm_final \
+#             -f manywheel/Dockerfile \
+#             .
+#         docker push "${DOCKER_IMAGE}"
+#     )
+# done
 
-for cuda_version in 9.2 10.1 10.2 11.0 11.1 11.2; do
+for cuda_version in 10.2 11.1; do
+    DOCKER_IMAGE="${DOCKER_REGISTRY}/pytorch/manylinux-cuda${cuda_version//./}"
     (
         set -x
         DOCKER_BUILDKIT=1 docker build \
-            -t "pytorch/manylinux-cuda${cuda_version//./}" \
+            -t "${DOCKER_IMAGE}" \
             --build-arg "BASE_CUDA_VERSION=${cuda_version}" \
             --build-arg "GPU_IMAGE=nvidia/cuda:${cuda_version}-devel-centos7" \
             --target cuda_final \
             -f manywheel/Dockerfile \
             .
-        docker push "pytorch/manylinux-cuda${cuda_version//./}"
+        docker push ${DOCKER_IMAGE}
     )
 done
 
@@ -38,5 +42,5 @@ done
         --target cpu_final \
         -f manywheel/Dockerfile \
         .
-        docker push "pytorch/manylinux-cpu"
+        docker push "${DOCKER_REGISTRY}/pytorch/manylinux-cpu"
 )