From 986d990fe5d97daf2dc59ac8ba4646fa288480f2 Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Wed, 24 Mar 2021 12:28:43 -0700 Subject: [PATCH 1/7] add ROCm 4.1 manylinux image build --- common/install_miopen.sh | 3 +++ common/install_rocm.sh | 2 +- manywheel/deploy.sh | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/common/install_miopen.sh b/common/install_miopen.sh index 567c6e7c9..75a227575 100644 --- a/common/install_miopen.sh +++ b/common/install_miopen.sh @@ -89,6 +89,9 @@ MIOPEN_CMAKE_COMMON_FLAGS=" if [[ ${ROCM_VERSION} == 4.0.1 ]]; then MIOPEN_CMAKE_DB_FLAGS="-DMIOPEN_EMBED_DB=gfx803_36;gfx803_64;gfx900_56;gfx900_64;gfx906_60;gfx906_64;gfx90878" MIOPEN_BRANCH="rocm-4.0.1" +elif [[ ${ROCM_VERSION} == 4.1 ]]; then + MIOPEN_CMAKE_DB_FLAGS="-DMIOPEN_EMBED_DB=gfx803_36;gfx803_64;gfx900_56;gfx900_64;gfx906_60;gfx906_64;gfx90878" + MIOPEN_BRANCH="rocm-4.1.x" else echo "Unhandled ROCM_VERSION ${ROCM_VERSION}" exit 1 diff --git a/common/install_rocm.sh b/common/install_rocm.sh index cf0f74ab6..a7f092c39 100644 --- a/common/install_rocm.sh +++ b/common/install_rocm.sh @@ -36,7 +36,7 @@ echo 'LIBDIR += -L$(MKLROOT)/lib' >> make.inc # overwrite original LIB, because it's wrong; it's missing start/end-group echo 'LIB = -Wl,--start-group -lmkl_gf_lp64 -lmkl_gnu_thread -lmkl_core -Wl,--end-group -lpthread -lstdc++ -lm -lgomp' >> make.inc echo 'LIB += -Wl,--enable-new-dtags -Wl,--rpath,/opt/rocm/lib -Wl,--rpath,$(MKLROOT)/lib -Wl,--rpath,/opt/rocm/magma/lib' >> make.inc -echo 'DEVCCFLAGS += --amdgpu-target=gfx803 --amdgpu-target=gfx900 --amdgpu-target=gfx906 --amdgpu-target=gfx908' >> make.inc +echo 'DEVCCFLAGS += --amdgpu-target=gfx803 --amdgpu-target=gfx900 --amdgpu-target=gfx906 --amdgpu-target=gfx908 --gpu-max-threads-per-block=256' >> make.inc export PATH="${PATH}:/opt/rocm/bin" make -f make.gen.hipMAGMA -j $(nproc) make lib/libmagma.so -j $(nproc) MKLROOT=/opt/intel diff --git a/manywheel/deploy.sh b/manywheel/deploy.sh index 61ad96201..9b5af69f6 100755 --- a/manywheel/deploy.sh +++ b/manywheel/deploy.sh @@ -2,7 +2,7 @@ set -eou pipefail -for rocm_version in 3.7 3.8 3.9 3.10 4.0 4.0.1; do +for rocm_version in 3.7 3.8 3.9 3.10 4.0 4.0.1 4.1; do ( set -x DOCKER_BUILDKIT=1 docker build \ From f81b80a751a88162e0bd2e6b8d039ac5562b883d Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Wed, 24 Mar 2021 14:08:12 -0700 Subject: [PATCH 2/7] uninstall existing miopen-hip package before reinstalling --- common/install_miopen.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/common/install_miopen.sh b/common/install_miopen.sh index 75a227575..b54bb5a5e 100644 --- a/common/install_miopen.sh +++ b/common/install_miopen.sh @@ -33,6 +33,9 @@ if [[ $ROCM_INT -lt 40001 ]]; then exit 0 fi +# Uninstall existing package, to avoid errors during later yum install indicating packages did not change. +yum remove -y miopen-hip + # Function to retry functions that sometimes timeout or have flaky failures retry () { $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) From 4628e2380eb7a3fc63ee9444aa467d3aec6fce1a Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Wed, 24 Mar 2021 14:51:52 -0700 Subject: [PATCH 3/7] libamd_comgr.so.1 -> *.so.2 --- manywheel/build_rocm.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/manywheel/build_rocm.sh b/manywheel/build_rocm.sh index 456186dec..8032281e3 100755 --- a/manywheel/build_rocm.sh +++ b/manywheel/build_rocm.sh @@ -108,14 +108,21 @@ else LIBAMDHIP64=libamdhip64.so.3 fi; +# in rocm4.1, libamd_comgr.so.1 changed to *.so.2 +if [[ $ROCM_INT -ge 40100 ]]; then + LIBAMDCOMGR=libamd_comgr.so.1 +else + LIBAMDCOMGR=libamd_comgr.so.2 +fi; + DEPS_LIST=( - "/opt/rocm/miopen/lib/libMIOpen.so.1" +o "/opt/rocm/miopen/lib/libMIOpen.so.1" "/opt/rocm/hip/lib/$LIBAMDHIP64" "/opt/rocm/hipblas/lib/libhipblas.so.0" "/opt/rocm/hiprand/lib/libhiprand.so.1" "/opt/rocm/hipsparse/lib/libhipsparse.so.0" "/opt/rocm/hsa/lib/libhsa-runtime64.so.1" - "/opt/rocm/${COMGR_LIBDIR}/libamd_comgr.so.1" + "/opt/rocm/${COMGR_LIBDIR}/${LIBAMDCOMGR}" "/opt/rocm/lib64/libhsakmt.so.1" "/opt/rocm/magma/lib/libmagma.so" "/opt/rocm/rccl/lib/librccl.so.1" @@ -138,7 +145,7 @@ DEPS_SONAME=( "libhiprand.so.1" "libhipsparse.so.0" "libhsa-runtime64.so.1" - "libamd_comgr.so.1" + "${LIBAMDCOMGR}" "libhsakmt.so.1" "libmagma.so" "librccl.so.1" From e534b65c76a497f473da0850e6db5bbf0c9de496 Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Wed, 24 Mar 2021 15:22:43 -0700 Subject: [PATCH 4/7] fix typo --- manywheel/build_rocm.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/manywheel/build_rocm.sh b/manywheel/build_rocm.sh index 8032281e3..f9ad51940 100755 --- a/manywheel/build_rocm.sh +++ b/manywheel/build_rocm.sh @@ -110,13 +110,13 @@ fi; # in rocm4.1, libamd_comgr.so.1 changed to *.so.2 if [[ $ROCM_INT -ge 40100 ]]; then - LIBAMDCOMGR=libamd_comgr.so.1 -else LIBAMDCOMGR=libamd_comgr.so.2 +else + LIBAMDCOMGR=libamd_comgr.so.1 fi; DEPS_LIST=( -o "/opt/rocm/miopen/lib/libMIOpen.so.1" + "/opt/rocm/miopen/lib/libMIOpen.so.1" "/opt/rocm/hip/lib/$LIBAMDHIP64" "/opt/rocm/hipblas/lib/libhipblas.so.0" "/opt/rocm/hiprand/lib/libhiprand.so.1" From 7a07afe525127609225332ca8a3d01781f8aedc0 Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Wed, 24 Mar 2021 15:24:52 -0700 Subject: [PATCH 5/7] update aux list names --- manywheel/build_rocm.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/manywheel/build_rocm.sh b/manywheel/build_rocm.sh index f9ad51940..9337c3c5f 100755 --- a/manywheel/build_rocm.sh +++ b/manywheel/build_rocm.sh @@ -111,8 +111,12 @@ fi; # in rocm4.1, libamd_comgr.so.1 changed to *.so.2 if [[ $ROCM_INT -ge 40100 ]]; then LIBAMDCOMGR=libamd_comgr.so.2 + KERNELGFX906=gfx906-xnack- + KERNELGFX908=gfx908-xnack- else LIBAMDCOMGR=libamd_comgr.so.1 + KERNELGFX906=gfx906 + KERNELGFX908=gfx908 fi; DEPS_LIST=( @@ -164,8 +168,8 @@ DEPS_SONAME=( DEPS_AUX_SRCLIST=( "/opt/rocm/rocblas/lib/library/Kernels.so-000-gfx803.hsaco" "/opt/rocm/rocblas/lib/library/Kernels.so-000-gfx900.hsaco" - "/opt/rocm/rocblas/lib/library/Kernels.so-000-gfx906.hsaco" - "/opt/rocm/rocblas/lib/library/Kernels.so-000-gfx908.hsaco" + "/opt/rocm/rocblas/lib/library/Kernels.so-000-${KERNELGFX906}.hsaco" + "/opt/rocm/rocblas/lib/library/Kernels.so-000-${KERNELGFX908}.hsaco" "/opt/rocm/rocblas/lib/library/TensileLibrary_gfx803.co" "/opt/rocm/rocblas/lib/library/TensileLibrary_gfx900.co" "/opt/rocm/rocblas/lib/library/TensileLibrary_gfx906.co" @@ -176,8 +180,8 @@ DEPS_AUX_SRCLIST=( DEPS_AUX_DSTLIST=( "lib/library/Kernels.so-000-gfx803.hsaco" "lib/library/Kernels.so-000-gfx900.hsaco" - "lib/library/Kernels.so-000-gfx906.hsaco" - "lib/library/Kernels.so-000-gfx908.hsaco" + "lib/library/Kernels.so-000-${KERNELGFX906}.hsaco" + "lib/library/Kernels.so-000-${KERNELGFX908}.hsaco" "lib/library/TensileLibrary_gfx803.co" "lib/library/TensileLibrary_gfx900.co" "lib/library/TensileLibrary_gfx906.co" From 514a20edea14bfe2a62926b265fa6a8f8fe8ab42 Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Thu, 25 Mar 2021 15:03:08 -0700 Subject: [PATCH 6/7] add rocm4.1 to update_s3_htmls.sh --- cron/update_s3_htmls.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron/update_s3_htmls.sh b/cron/update_s3_htmls.sh index deb2413e2..c1eadff3d 100755 --- a/cron/update_s3_htmls.sh +++ b/cron/update_s3_htmls.sh @@ -9,7 +9,7 @@ set -e # Upload for all CUDA/cpu versions if not given one to use if [[ -z "$CUDA_VERSIONS" ]]; then - export CUDA_VERSIONS=('cpu' 'cu92' 'cu100' 'cu101' 'cu102' 'cu110' 'rocm3.7' 'rocm3.8' 'rocm3.9' 'rocm3.10' 'rocm4.0') + export CUDA_VERSIONS=('cpu' 'cu92' 'cu100' 'cu101' 'cu102' 'cu110' 'rocm3.7' 'rocm3.8' 'rocm3.9' 'rocm3.10' 'rocm4.0.1' 'rocm4.1') fi if [[ -z "$HTML_NAME" ]]; then From 5ab8e8d67a16ae2a72452d4327306a0f468a86d9 Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Fri, 26 Mar 2021 10:59:10 -0700 Subject: [PATCH 7/7] add missing hipfft dep --- manywheel/build_rocm.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/manywheel/build_rocm.sh b/manywheel/build_rocm.sh index 9337c3c5f..4006907f9 100755 --- a/manywheel/build_rocm.sh +++ b/manywheel/build_rocm.sh @@ -109,20 +109,26 @@ else fi; # in rocm4.1, libamd_comgr.so.1 changed to *.so.2 +# hipfft is a new package, separate from rocfft if [[ $ROCM_INT -ge 40100 ]]; then LIBAMDCOMGR=libamd_comgr.so.2 KERNELGFX906=gfx906-xnack- KERNELGFX908=gfx908-xnack- + HIPFFT_DEP=/opt/rocm/hipfft/lib/libhipfft.so + HIPFFT_SO=libhipfft.so else LIBAMDCOMGR=libamd_comgr.so.1 KERNELGFX906=gfx906 KERNELGFX908=gfx908 + HIPFFT_DEP= + HIPFFT_SO= fi; DEPS_LIST=( "/opt/rocm/miopen/lib/libMIOpen.so.1" "/opt/rocm/hip/lib/$LIBAMDHIP64" "/opt/rocm/hipblas/lib/libhipblas.so.0" + ${HIPFFT_DEP} "/opt/rocm/hiprand/lib/libhiprand.so.1" "/opt/rocm/hipsparse/lib/libhipsparse.so.0" "/opt/rocm/hsa/lib/libhsa-runtime64.so.1" @@ -146,6 +152,7 @@ DEPS_SONAME=( "libMIOpen.so.1" "$LIBAMDHIP64" "libhipblas.so.0" + ${HIPFFT_SO} "libhiprand.so.1" "libhipsparse.so.0" "libhsa-runtime64.so.1"