[ROCm] add 4.1 manylinux image

common/install_miopen.sh

@@ -33,6 +33,9 @@ if [[ $ROCM_INT -lt 40001 ]]; then
     exit 0
 fi
 
+# Uninstall existing package, to avoid errors during later yum install indicating packages did not change.
+yum remove -y miopen-hip
+
 # Function to retry functions that sometimes timeout or have flaky failures
 retry () {
     $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
@@ -89,6 +92,9 @@ MIOPEN_CMAKE_COMMON_FLAGS="
 if [[ ${ROCM_VERSION} == 4.0.1 ]]; then
     MIOPEN_CMAKE_DB_FLAGS="-DMIOPEN_EMBED_DB=gfx803_36;gfx803_64;gfx900_56;gfx900_64;gfx906_60;gfx906_64;gfx90878"
     MIOPEN_BRANCH="rocm-4.0.1"
+elif [[ ${ROCM_VERSION} == 4.1 ]]; then
+    MIOPEN_CMAKE_DB_FLAGS="-DMIOPEN_EMBED_DB=gfx803_36;gfx803_64;gfx900_56;gfx900_64;gfx906_60;gfx906_64;gfx90878"
+    MIOPEN_BRANCH="rocm-4.1.x"
 else
     echo "Unhandled ROCM_VERSION ${ROCM_VERSION}"
     exit 1

common/install_rocm.sh

@@ -36,7 +36,7 @@ echo 'LIBDIR += -L$(MKLROOT)/lib' >> make.inc
 # overwrite original LIB, because it's wrong; it's missing start/end-group
 echo 'LIB = -Wl,--start-group -lmkl_gf_lp64 -lmkl_gnu_thread -lmkl_core -Wl,--end-group -lpthread -lstdc++ -lm -lgomp' >> make.inc
 echo 'LIB += -Wl,--enable-new-dtags -Wl,--rpath,/opt/rocm/lib -Wl,--rpath,$(MKLROOT)/lib -Wl,--rpath,/opt/rocm/magma/lib' >> make.inc
-echo 'DEVCCFLAGS += --amdgpu-target=gfx803 --amdgpu-target=gfx900 --amdgpu-target=gfx906 --amdgpu-target=gfx908' >> make.inc
+echo 'DEVCCFLAGS += --amdgpu-target=gfx803 --amdgpu-target=gfx900 --amdgpu-target=gfx906 --amdgpu-target=gfx908 --gpu-max-threads-per-block=256' >> make.inc
 export PATH="${PATH}:/opt/rocm/bin"
 make -f make.gen.hipMAGMA -j $(nproc)
 make lib/libmagma.so -j $(nproc) MKLROOT=/opt/intel

cron/update_s3_htmls.sh

@@ -9,7 +9,7 @@ set -e
 
 # Upload for all CUDA/cpu versions if not given one to use
 if [[ -z "$CUDA_VERSIONS" ]]; then
-    export CUDA_VERSIONS=('cpu' 'cu92' 'cu100' 'cu101' 'cu102' 'cu110' 'rocm3.7' 'rocm3.8' 'rocm3.9' 'rocm3.10' 'rocm4.0')
+    export CUDA_VERSIONS=('cpu' 'cu92' 'cu100' 'cu101' 'cu102' 'cu110' 'rocm3.7' 'rocm3.8' 'rocm3.9' 'rocm3.10' 'rocm4.0.1' 'rocm4.1')
 fi
 
 if [[ -z "$HTML_NAME" ]]; then

manywheel/build_rocm.sh

@@ -108,14 +108,31 @@ else
     LIBAMDHIP64=libamdhip64.so.3
 fi;
 
+# in rocm4.1, libamd_comgr.so.1 changed to *.so.2
+# hipfft is a new package, separate from rocfft
+if [[ $ROCM_INT -ge 40100 ]]; then
+    LIBAMDCOMGR=libamd_comgr.so.2
+    KERNELGFX906=gfx906-xnack-
+    KERNELGFX908=gfx908-xnack-
+    HIPFFT_DEP=/opt/rocm/hipfft/lib/libhipfft.so
+    HIPFFT_SO=libhipfft.so
+else
+    LIBAMDCOMGR=libamd_comgr.so.1
+    KERNELGFX906=gfx906
+    KERNELGFX908=gfx908
+    HIPFFT_DEP=
+    HIPFFT_SO=
+fi;
+
 DEPS_LIST=(
     "/opt/rocm/miopen/lib/libMIOpen.so.1"
     "/opt/rocm/hip/lib/$LIBAMDHIP64"
     "/opt/rocm/hipblas/lib/libhipblas.so.0"
+    ${HIPFFT_DEP}
     "/opt/rocm/hiprand/lib/libhiprand.so.1"
     "/opt/rocm/hipsparse/lib/libhipsparse.so.0"
     "/opt/rocm/hsa/lib/libhsa-runtime64.so.1"
-    "/opt/rocm/${COMGR_LIBDIR}/libamd_comgr.so.1"
+    "/opt/rocm/${COMGR_LIBDIR}/${LIBAMDCOMGR}"
     "/opt/rocm/lib64/libhsakmt.so.1"
     "/opt/rocm/magma/lib/libmagma.so"
     "/opt/rocm/rccl/lib/librccl.so.1"
@@ -135,10 +152,11 @@ DEPS_SONAME=(
     "libMIOpen.so.1"
     "$LIBAMDHIP64"
     "libhipblas.so.0"
+    ${HIPFFT_SO}
     "libhiprand.so.1"
     "libhipsparse.so.0"
     "libhsa-runtime64.so.1"
-    "libamd_comgr.so.1"
+    "${LIBAMDCOMGR}"
     "libhsakmt.so.1"
     "libmagma.so"
     "librccl.so.1"
@@ -157,8 +175,8 @@ DEPS_SONAME=(
 DEPS_AUX_SRCLIST=(
     "/opt/rocm/rocblas/lib/library/Kernels.so-000-gfx803.hsaco"
     "/opt/rocm/rocblas/lib/library/Kernels.so-000-gfx900.hsaco"
-    "/opt/rocm/rocblas/lib/library/Kernels.so-000-gfx906.hsaco"
-    "/opt/rocm/rocblas/lib/library/Kernels.so-000-gfx908.hsaco"
+    "/opt/rocm/rocblas/lib/library/Kernels.so-000-${KERNELGFX906}.hsaco"
+    "/opt/rocm/rocblas/lib/library/Kernels.so-000-${KERNELGFX908}.hsaco"
     "/opt/rocm/rocblas/lib/library/TensileLibrary_gfx803.co"
     "/opt/rocm/rocblas/lib/library/TensileLibrary_gfx900.co"
     "/opt/rocm/rocblas/lib/library/TensileLibrary_gfx906.co"
@@ -169,8 +187,8 @@ DEPS_AUX_SRCLIST=(
 DEPS_AUX_DSTLIST=(
     "lib/library/Kernels.so-000-gfx803.hsaco"
     "lib/library/Kernels.so-000-gfx900.hsaco"
-    "lib/library/Kernels.so-000-gfx906.hsaco"
-    "lib/library/Kernels.so-000-gfx908.hsaco"
+    "lib/library/Kernels.so-000-${KERNELGFX906}.hsaco"
+    "lib/library/Kernels.so-000-${KERNELGFX908}.hsaco"
     "lib/library/TensileLibrary_gfx803.co"
     "lib/library/TensileLibrary_gfx900.co"
     "lib/library/TensileLibrary_gfx906.co"

manywheel/deploy.sh

@@ -2,7 +2,7 @@
 
 set -eou pipefail
 
-for rocm_version in 3.7 3.8 3.9 3.10 4.0 4.0.1; do
+for rocm_version in 3.7 3.8 3.9 3.10 4.0 4.0.1 4.1; do
     (
         set -x
         DOCKER_BUILDKIT=1 docker build \