Switch to packaged Thrust on Ubuntu, enable CentOS as a CI target

.jenkins/pytorch/build.sh

@@ -43,22 +43,24 @@ cmake --version
 pip install -r requirements.txt || true
 
 if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
+
+  if [[ "$BUILD_ENVIRONMENT" == *centos* ]]; then
+    # These environment variables are not set on CI when we were running as the Jenkins user.
+    # The HIP Utility scripts require these environment variables to be set in order to run without error.
+    export LC_ALL=en_US.utf8
+    export LANG=en_US.utf8
+  else
+    export LANG=C.UTF-8
+    export LC_ALL=C.UTF-8
+  fi
+
   # This is necessary in order to cross compile (or else we'll have missing GPU device).
   export HCC_AMDGPU_TARGET=gfx900
 
-  # These environment variables are not set on CI when we were running as the Jenkins user.
-  # The HIP Utility scripts require these environment variables to be set in order to run without error.
-  export LANG=C.UTF-8
-  export LC_ALL=C.UTF-8
-
   # This environment variable enabled HCC Optimizations that speed up the linking stage.
   # https://github.com/RadeonOpenCompute/hcc#hcc-with-thinlto-linking
   export KMTHINLTO=1
 
-  # Need the libc++1 and libc++abi1 libraries to allow torch._C to load at runtime
-  sudo apt-get install libc++1
-  sudo apt-get install libc++abi1
-
   # When hcc runs out of memory, it silently exits without stopping
   # the build process, leaving undefined symbols in the shared lib
   # which will cause undefined symbol errors when later running

.jenkins/pytorch/enabled-configs.txt

@@ -44,6 +44,7 @@ short-perf-test-cpu
 short-perf-test-gpu
 py2-clang7-rocmdeb-ubuntu16.04-build
 py2-clang7-rocmdeb-ubuntu16.04-test
+py2-rocmrpm-centos7.5-build
 pytorch-ppc64le-cuda9.2-cudnn7-py3-build
 pytorch-ppc64le-cuda9.2-cudnn7-py3-test
 pytorch-ppc64le-cuda9.1-cudnn7-py3-build

caffe2/core/hip/context_hip.h

@@ -175,7 +175,7 @@ class HIPContext final : public BaseContext {
     return hip_objects_.GetStream(gpu_id, stream_id);
   }
 
-  rocblas_handle rocblas_handle() {
+  rocblas_handle rocblashandle() {
     return hip_objects_.GetHandle(gpu_id_, stream_id_);
   }
 

caffe2/utils/hip/math_hip.cc

@@ -759,7 +759,7 @@ void Gemm<float, HIPContext>(
       ? rocblas_operation_none
       : rocblas_operation_transpose;
   ROCBLAS_ENFORCE(rocblas_sgemm(
-      context->rocblas_handle(),
+      context->rocblashandle(),
       cuTransB,
       cuTransA,
       N,
@@ -803,7 +803,7 @@ void Gemm<at::Half, HIPContext>(
       : rocblas_operation_transpose;
   if (math_type == TensorProto_DataType_FLOAT) {
     ROCBLAS_CHECK(rocblas_sgemmEx(
-        context->rocblas_handle(),
+        context->rocblashandle(),
         cuTransB,
         cuTransA,
         N,
@@ -828,7 +828,7 @@ void Gemm<at::Half, HIPContext>(
 
     // call cublasHgemm
     ROCBLAS_CHECK(cublasHgemm(
-        context->rocblas_handle(),
+        context->rocblashandle(),
         cuTransB,
         cuTransA,
         N,
@@ -933,7 +933,7 @@ void GemmStridedBatched<float, HIPContext>(
       ? rocblas_operation_none
       : rocblas_operation_transpose;
   ROCBLAS_ENFORCE(rocblas_sgemm_strided_batched(
-      context->rocblas_handle(),
+      context->rocblashandle(),
       cuTransB,
       cuTransA,
       N,
@@ -1004,7 +1004,7 @@ void GemmStridedBatched<at::Half, HIPContext>(
     __half alpha_fp16 = at::Half(alpha);
     __half beta_fp16 = at::Half(beta);
     ROCBLAS_ENFORCE(cublasHgemmStridedBatched(
-        context->rocblas_handle(),
+        context->rocblashandle(),
         cuTransB,
         cuTransA,
         N,
@@ -1051,7 +1051,7 @@ void GemmEx<float, HIPContext>(
       ? rocblas_operation_none
       : rocblas_operation_transpose;
   ROCBLAS_ENFORCE(rocblas_sgemm(
-      context->rocblas_handle(),
+      context->rocblashandle(),
       cuTransB,
       cuTransA,
       N,
@@ -1083,7 +1083,7 @@ void Gemv<float, HIPContext>(
       ? rocblas_operation_transpose
       : rocblas_operation_none;
   ROCBLAS_ENFORCE(rocblas_sgemv(
-      context->rocblas_handle(),
+      context->rocblashandle(),
       cuTransA,
       N,
       M,
@@ -1170,7 +1170,7 @@ void Gemv<at::Half, HIPContext>(
 
   if (math_type == TensorProto_DataType_FLOAT) {
     ROCBLAS_CHECK(cublasSgemmEx(
-        context->rocblas_handle(),
+        context->rocblashandle(),
         cuTransA,
         rocblas_operation_none,
         m,
@@ -1192,7 +1192,7 @@ void Gemv<at::Half, HIPContext>(
     __half beta_fp16 = at::Half(beta);
 
     ROCBLAS_CHECK(cublasHgemm(
-        context->rocblas_handle(),
+        context->rocblashandle(),
         cuTransA,
         rocblas_operation_none,
         m,
@@ -1390,7 +1390,7 @@ void Dot<float, HIPContext>(
     HIPContext* context) {
   float result;
   ROCBLAS_ENFORCE(
-      rocblas_sdot(context->rocblas_handle(), n, a, 1, b, 1, &result));
+      rocblas_sdot(context->rocblashandle(), n, a, 1, b, 1, &result));
   context->CopyFromCPU<float>(1, &result, y);
 }
 
@@ -1406,7 +1406,7 @@ void Dot<at::Half, HIPContext>(
   at::Half result;
   // execute with 32-bit math
   ROCBLAS_CHECK(cublasDotEx(
-      context->rocblas_handle(),
+      context->rocblashandle(),
       n,
       a,
       CUDA_R_16F,
@@ -1879,7 +1879,7 @@ void Axpy<float, HIPContext>(
     float* Y,
     HIPContext* context) {
   ROCBLAS_ENFORCE(
-      rocblas_saxpy(context->rocblas_handle(), N, &alpha, X, 1, Y, 1));
+      rocblas_saxpy(context->rocblashandle(), N, &alpha, X, 1, Y, 1));
 }
 
 template <>
@@ -1891,7 +1891,7 @@ void Axpy<double, HIPContext>(
     HIPContext* context) {
   double alpha_d{alpha};
   ROCBLAS_ENFORCE(
-      rocblas_daxpy(context->rocblas_handle(), N, &alpha_d, X, 1, Y, 1));
+      rocblas_daxpy(context->rocblashandle(), N, &alpha_d, X, 1, Y, 1));
 }
 
 template <>
@@ -1904,7 +1904,7 @@ void Axpy<at::Half, HIPContext>(
   CAFFE_THROW("Unsupported math type");
 #if ROCBLAS_FP16
   ROCBLAS_CHECK(cublasAxpyEx(
-      context->rocblas_handle(),
+      context->rocblashandle(),
       N,
       &alpha,
       CUDA_R_16F,

cmake/public/LoadHIP.cmake

@@ -62,11 +62,8 @@ ENDIF()
 # THRUST_PATH
 IF(DEFINED ENV{THRUST_PATH})
   SET(THRUST_PATH $ENV{THRUST_PATH})
-ELSEIF(DEFINED ENV{THRUST_ROOT})
-  # TODO: Remove support of THRUST_ROOT environment variable
-  SET(THRUST_PATH $ENV{THRUST_ROOT})
 ELSE()
-  SET(THRUST_PATH ${ROCM_PATH}/Thrust)
+  SET(THRUST_PATH ${ROCM_PATH}/include)
 ENDIF()
 
 # HIPRAND_PATH

docker/caffe2/jenkins/build.sh

@@ -39,6 +39,10 @@ fi
 if [[ "$image" == *rocm* ]]; then
   ROCM_VERSION="$(echo "${image}" | perl -n -e'/rocm(\d+\.\d+\.\d+|nightly)/ && print $1')"
   DOCKERFILE="${OS}-rocm/Dockerfile"
+  if [[ "$image" == *centos* ]]; then
+    # newer cmake version needed
+    CMAKE_VERSION=3.6.3
+  fi
 fi
 
 if [[ "$image" == *conda* ]]; then

docker/caffe2/jenkins/centos-rocm/.gitignore

@@ -0,0 +1 @@
+*.sh

docker/caffe2/jenkins/centos-rocm/Dockerfile

@@ -0,0 +1,46 @@
+ARG CENTOS_VERSION
+FROM centos:${CENTOS_VERSION}
+
+# Install required packages to build Caffe2
+ARG EC2
+ADD ./install_base.sh install_base.sh
+RUN bash ./install_base.sh && rm install_base.sh
+
+# Install rocm
+ARG ROCM_VERSION
+ADD ./install_rocm.sh install_rocm.sh
+RUN bash ./install_rocm.sh
+RUN rm install_rocm.sh
+ENV PATH /opt/rocm/bin:$PATH
+ENV PATH /opt/rocm/hcc/bin:$PATH
+ENV PATH /opt/rocm/hip/bin:$PATH
+ENV PATH /opt/rocm/opencl/bin:$PATH
+ENV MIOPEN_DISABLE_CACHE 1
+ENV HIP_PLATFORM hcc
+
+# Install non-default CMake version
+ARG CMAKE_VERSION
+ADD ./install_cmake.sh install_cmake.sh
+RUN if [ -n "${CMAKE_VERSION}" ]; then bash ./install_cmake.sh; fi
+
+# Compile/install ccache for faster builds
+ADD ./install_ccache.sh install_ccache.sh
+RUN bash ./install_ccache.sh && rm install_ccache.sh
+
+# Install Python
+ARG PYTHON_VERSION
+ADD ./install_python.sh install_python.sh
+RUN if [ -n "${PYTHON_VERSION}" ]; then bash ./install_python.sh; fi
+RUN rm install_python.sh
+
+# (optional) Add Jenkins user
+ARG JENKINS
+ARG JENKINS_UID
+ARG JENKINS_GID
+ADD ./add_jenkins_user.sh add_jenkins_user.sh
+RUN if [ -n "${JENKINS}" ]; then bash ./add_jenkins_user.sh; echo "source scl_source enable devtoolset-7" >> /var/lib/jenkins/.bashrc; fi
+RUN rm add_jenkins_user.sh
+
+# Include BUILD_ENVIRONMENT environment variable in image
+ARG BUILD_ENVIRONMENT
+ENV BUILD_ENVIRONMENT ${BUILD_ENVIRONMENT}

docker/caffe2/jenkins/common/install_rocm.sh

@@ -7,6 +7,10 @@ install_ubuntu() {
     apt-get install -y wget
     apt-get install -y libopenblas-dev
 
+    # Need the libc++1 and libc++abi1 libraries to allow torch._C to load at runtime
+    apt-get install libc++1
+    apt-get install libc++abi1
+
     DEB_ROCM_REPO=http://repo.radeon.com/rocm/misc/facebook/apt/.apt_1.9.white_rabbit/debian
     # Add rocm repository
     wget -qO - $DEB_ROCM_REPO/rocm.gpg.key | apt-key add -
@@ -42,19 +46,55 @@ install_ubuntu() {
 }
 
 install_centos() {
-    echo "Not implemented yet"
-    exit 1
+
+  yum update -y
+  yum install -y wget
+  yum install -y openblas-devel
+
+  yum install -y centos-release-scl
+  yum install -y devtoolset-7
+
+  echo "source scl_source enable devtoolset-7" >> /root/.bashrc
+
+  yum install -y epel-release
+  yum install -y dkms kernel-headers-`uname -r` kernel-devel-`uname -r`
+
+  echo "[ROCm]" > /etc/yum.repos.d/rocm.repo
+  echo "name=ROCm" >> /etc/yum.repos.d/rocm.repo
+  echo "baseurl=http://repo.radeon.com/rocm/misc/facebook/yum/.yum_1.9.white_rabbit/" >> /etc/yum.repos.d/rocm.repo
+  echo "enabled=1" >> /etc/yum.repos.d/rocm.repo
+  echo "gpgcheck=0" >> /etc/yum.repos.d/rocm.repo
+
+  yum update -y
+
+  yum install -y \
+                   rocm-dev \
+                   rocm-libs \
+                   rocm-utils \
+                   rocfft \
+                   miopen-hip \
+                   miopengemm \
+                   rocblas \
+                   rocm-profiler \
+                   cxlactivitylogger \
+                   rocsparse \
+                   hipsparse \
+                   rocrand
+
+
+  # Cleanup
+  yum clean all
+  rm -rf /var/cache/yum
+  rm -rf /var/lib/yum/yumdb
+  rm -rf /var/lib/yum/history
+
+  # Needed for now, will be replaced once hip-thrust is packaged for CentOS
+  git clone --recursive https://github.com/ROCmSoftwarePlatform/Thrust.git /data/Thrust
+  rm -rf /data/Thrust/thrust/system/cuda/detail/cub-hip
+  git clone --recursive https://github.com/ROCmSoftwarePlatform/cub-hip.git /data/Thrust/thrust/system/cuda/detail/cub-hip
+  ln -s /data/Thrust/thrust /opt/rocm/include/thrust
 }
 
-install_hip_thrust() {
-    # Needed for now, will be replaced soon
-    # We are now (redundantly) installing the Thrust package into another location (/opt/rocm/include/thrust) which we will
-    # switch over to
-    git clone --recursive https://github.com/ROCmSoftwarePlatform/Thrust.git /data/Thrust
-    rm -rf /data/Thrust/thrust/system/cuda/detail/cub-hip
-    git clone --recursive https://github.com/ROCmSoftwarePlatform/cub-hip.git /data/Thrust/thrust/system/cuda/detail/cub-hip
-}
-
 # Install Python packages depending on the base OS
 if [ -f /etc/lsb-release ]; then
   install_ubuntu
@@ -64,5 +104,3 @@ else
   echo "Unable to determine OS..."
   exit 1
 fi
-
-install_hip_thrust

docker/caffe2/jenkins/ubuntu-rocm/Dockerfile

@@ -60,7 +60,6 @@ ENV PATH /opt/rocm/bin:$PATH
 ENV PATH /opt/rocm/hcc/bin:$PATH
 ENV PATH /opt/rocm/hip/bin:$PATH
 ENV PATH /opt/rocm/opencl/bin:$PATH
-ENV THRUST_ROOT /data/Thrust
 ENV MIOPEN_DISABLE_CACHE 1
 ENV HIP_PLATFORM hcc
 

setup.py

@@ -1029,6 +1029,7 @@ def run(self):
     rocfft_include_path = '/opt/rocm/rocfft/include'
     hiprand_include_path = '/opt/rocm/hiprand/include'
     rocrand_include_path = '/opt/rocm/rocrand/include'
+    thrust_include_path = '/opt/rocm/include/'
     hip_lib_path = '/opt/rocm/hip/lib'
     hcc_lib_path = '/opt/rocm/hcc/lib'
     include_dirs.append(rocm_include_path)
@@ -1038,6 +1039,7 @@ def run(self):
     include_dirs.append(hipsparse_include_path)
     include_dirs.append(hiprand_include_path)
     include_dirs.append(rocrand_include_path)
+    include_dirs.append(thrust_include_path)
     include_dirs.append(tmp_install_path + "/include/THCUNN")
     extra_link_args.append('-L' + hip_lib_path)
     extra_link_args.append('-Wl,-rpath,' + hip_lib_path)

tools/amd_build/pyHIPIFY/cuda_to_hip_mappings.py

@@ -2230,7 +2230,7 @@
     ("HasCudaGPU" , ("HasHipGPU", API_CAFFE2)),
     ("__expf" , ("expf", API_CAFFE2)),
     ("CUBLAS_ENFORCE" , ("ROCBLAS_ENFORCE", API_CAFFE2)),
-    ("cublas_handle" , ("rocblas_handle", API_CAFFE2)),
+    ("cublas_handle" , ("rocblashandle", API_CAFFE2)),
     ("CURAND_ENFORCE" ,("HIPRAND_ENFORCE", API_CAFFE2)),
     ("curandGenerateUniform" , ("hiprandGenerateUniform", API_CAFFE2)),
     ("curand_generator" , ("hiprand_generator", API_CAFFE2)),