Fix for HVD and OpenMPI: open-mpi/ompi#4948

Dockerfile.tensorflow

@@ -35,9 +35,7 @@ ARG HOROVOD_VERSION='0.15.2'
 WORKDIR /workspace
 
 ENV DEBIAN_FRONTEND=noninteractive \
-    CI_BUILD_PYTHON=python \
-    HOROVOD_WITH_TENSORFLOW='1' \
-    HOROVOD_WITHOUT_TENSORFLOW=''
+    CI_BUILD_PYTHON=python
 
 RUN bash -c 'echo -e "###############################################################"' && \
     bash -c 'echo -e "   [-] BAZEL_VERSION   = $BAZEL_VERSION"' && \
@@ -123,9 +121,12 @@ RUN rm -rf /opt/tensorflow && \
     bazel-bin/tensorflow/tools/pip_package/build_pip_package /opt/tensorflow/pip_pkg --gpu && \
     pip --no-cache-dir install --upgrade /opt/tensorflow/pip_pkg/tensorflow_*.whl && \
     rm -f /usr/local/cuda/lib64/stubs/libcuda.so.1 && \
-    rm -rf /root/.cache
+    rm -rf /root/.cache && \
+    rm -rf opt/tensorflow/pip_pkg
 
-# RUN rm -rf opt/tensorflow/pip_pkg
+ARG HOROVOD_WITH_TENSORFLOW='1' \
+    HOROVOD_WITHOUT_TENSORFLOW='' \
+    OMPI_MCA_btl_vader_single_copy_mechanism=none
 
 # Install Horovod, temporarily using CUDA stubs
 RUN ldconfig /usr/local/cuda/targets/x86_64-linux/lib/stubs && \