From 997798c1fe753b0e6c88f27999366acae3f8afec Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Tue, 26 Jan 2021 10:25:52 -0800 Subject: [PATCH] add support for ROCm 4.0.1, including custom MIOpen package --- common/install_miopen.sh | 115 +++++++++++++++++++++++++++++++++++++++ manywheel/Dockerfile | 3 + manywheel/build_rocm.sh | 46 ++++++++++------ manywheel/deploy.sh | 2 +- 4 files changed, 149 insertions(+), 17 deletions(-) create mode 100644 common/install_miopen.sh diff --git a/common/install_miopen.sh b/common/install_miopen.sh new file mode 100644 index 000000000..567c6e7c9 --- /dev/null +++ b/common/install_miopen.sh @@ -0,0 +1,115 @@ +#!/bin/bash + +set -ex + +ROCM_VERSION=$1 + +if [[ -z $ROCM_VERSION ]]; then + echo "missing ROCM_VERSION" + exit 1; +fi + +# To make version comparison easier, create an integer representation. +save_IFS="$IFS" +IFS=. ROCM_VERSION_ARRAY=(${ROCM_VERSION}) +IFS="$save_IFS" +if [[ ${#ROCM_VERSION_ARRAY[@]} == 2 ]]; then + ROCM_VERSION_MAJOR=${ROCM_VERSION_ARRAY[0]} + ROCM_VERSION_MINOR=${ROCM_VERSION_ARRAY[1]} + ROCM_VERSION_PATCH=0 +elif [[ ${#ROCM_VERSION_ARRAY[@]} == 3 ]]; then + ROCM_VERSION_MAJOR=${ROCM_VERSION_ARRAY[0]} + ROCM_VERSION_MINOR=${ROCM_VERSION_ARRAY[1]} + ROCM_VERSION_PATCH=${ROCM_VERSION_ARRAY[2]} +else + echo "Unhandled ROCM_VERSION ${ROCM_VERSION}" + exit 1 +fi +ROCM_INT=$(($ROCM_VERSION_MAJOR * 10000 + $ROCM_VERSION_MINOR * 100 + $ROCM_VERSION_PATCH)) + +# Install custom MIOpen + COMgr for ROCm >= 4.0.1 +if [[ $ROCM_INT -lt 40001 ]]; then + echo "ROCm version < 4.0.1; will not install custom MIOpen" + exit 0 +fi + +# Function to retry functions that sometimes timeout or have flaky failures +retry () { + $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) +} + +# Build custom MIOpen to use comgr for offline compilation. + +## Need a sanitized ROCM_VERSION without patchlevel; patchlevel version 0 must be added to paths. +ROCM_DOTS=$(echo ${ROCM_VERSION} | tr -d -c '.' | wc -c) +if [[ ${ROCM_DOTS} == 1 ]]; then + ROCM_VERSION_NOPATCH="${ROCM_VERSION}" + ROCM_INSTALL_PATH="/opt/rocm-${ROCM_VERSION}.0" +else + ROCM_VERSION_NOPATCH="${ROCM_VERSION%.*}" + ROCM_INSTALL_PATH="/opt/rocm-${ROCM_VERSION}" +fi + +## MIOpen minimum requirements + +### Boost; No viable yum package exists. Must use static linking with PIC. +retry wget https://dl.bintray.com/boostorg/release/1.72.0/source/boost_1_72_0.tar.gz +tar xzf boost_1_72_0.tar.gz +pushd boost_1_72_0 +./bootstrap.sh +./b2 -j $(nproc) threading=multi link=static cxxflags=-fPIC --with-system --with-filesystem install +popd +rm -rf boost_1_72_0 +rm -f boost_1_72_0.tar.gz + +### sqlite; No viable yum package exists. Must be at least version 3.14. +retry wget https://sqlite.org/2017/sqlite-autoconf-3170000.tar.gz +tar xzf sqlite-autoconf-3170000.tar.gz +pushd sqlite-autoconf-3170000 +./configure --with-pic +make -j $(nproc) +make install +popd +rm -rf sqlite-autoconf-3170000 +rm -f sqlite-autoconf-3170000.tar.gz + +### half header +retry curl -fsSL https://raw.githubusercontent.com/ROCmSoftwarePlatform/half/master/include/half.hpp -o /usr/include/half.hpp + +### bzip2 +yum install -y bzip2-devel + +## Build MIOpen + +# MIOPEN_USE_HIP_KERNELS is a Workaround for COMgr issues +MIOPEN_CMAKE_COMMON_FLAGS=" +-DMIOPEN_USE_COMGR=ON +-DMIOPEN_BUILD_DRIVER=OFF +" +if [[ ${ROCM_VERSION} == 4.0.1 ]]; then + MIOPEN_CMAKE_DB_FLAGS="-DMIOPEN_EMBED_DB=gfx803_36;gfx803_64;gfx900_56;gfx900_64;gfx906_60;gfx906_64;gfx90878" + MIOPEN_BRANCH="rocm-4.0.1" +else + echo "Unhandled ROCM_VERSION ${ROCM_VERSION}" + exit 1 +fi + +git clone https://github.com/ROCmSoftwarePlatform/MIOpen -b ${MIOPEN_BRANCH} +pushd MIOpen +mkdir -p build +cd build +PKG_CONFIG_PATH=/usr/local/lib/pkgconfig CXX=${ROCM_INSTALL_PATH}/llvm/bin/clang++ cmake .. \ + ${MIOPEN_CMAKE_COMMON_FLAGS} \ + ${MIOPEN_CMAKE_DB_FLAGS} \ + -DCMAKE_PREFIX_PATH="${ROCM_INSTALL_PATH}/hip;${ROCM_INSTALL_PATH}" +make MIOpen -j $(nproc) +make -j $(nproc) package +yum install -y miopen-*.rpm +popd +rm -rf MIOpen + +# Cleanup +yum clean all +rm -rf /var/cache/yum +rm -rf /var/lib/yum/yumdb +rm -rf /var/lib/yum/history diff --git a/manywheel/Dockerfile b/manywheel/Dockerfile index 28f8c2808..90e45dec1 100644 --- a/manywheel/Dockerfile +++ b/manywheel/Dockerfile @@ -136,7 +136,10 @@ ARG ROCM_VERSION=3.7 ADD ./common/install_rocm.sh install_rocm.sh RUN bash ./install_rocm.sh ${ROCM_VERSION} && rm install_rocm.sh # cmake is already installed inside the rocm base image, but both 2 and 3 exist +# cmake3 is needed for the later MIOpen custom build, so that step is last. RUN yum install -y cmake3 && \ rm -f /usr/bin/cmake && \ ln -s /usr/bin/cmake3 /usr/bin/cmake +ADD ./common/install_miopen.sh install_miopen.sh +RUN bash ./install_miopen.sh ${ROCM_VERSION} && rm install_miopen.sh diff --git a/manywheel/build_rocm.sh b/manywheel/build_rocm.sh index 3db4a19f6..2324406d9 100755 --- a/manywheel/build_rocm.sh +++ b/manywheel/build_rocm.sh @@ -62,31 +62,50 @@ elif [[ "$OS_NAME" == *"Ubuntu"* ]]; then LIBELF_PATH="/usr/lib/x86_64-linux-gnu/libelf.so.1" fi -# rocm3.8 and later use TensileLibrary.dat -if [[ $ROCM_VERSION == "rocm3.7" ]]; then - TENSILE_LIBRARY_NAME=TensileLibrary.yaml +# NOTE: Some ROCm versions have identical dependencies, or very close deps. +# We conditionalize as generically as possible, capturing only what changes +# from version to version. + +# To make version comparison easier, create an integer representation. +ROCM_VERSION_CLEAN=$(echo ${ROCM_VERSION} | sed s/rocm//) +save_IFS="$IFS" +IFS=. ROCM_VERSION_ARRAY=(${ROCM_VERSION_CLEAN}) +IFS="$save_IFS" +if [[ ${#ROCM_VERSION_ARRAY[@]} == 2 ]]; then + ROCM_VERSION_MAJOR=${ROCM_VERSION_ARRAY[0]} + ROCM_VERSION_MINOR=${ROCM_VERSION_ARRAY[1]} + ROCM_VERSION_PATCH=0 +elif [[ ${#ROCM_VERSION_ARRAY[@]} == 3 ]]; then + ROCM_VERSION_MAJOR=${ROCM_VERSION_ARRAY[0]} + ROCM_VERSION_MINOR=${ROCM_VERSION_ARRAY[1]} + ROCM_VERSION_PATCH=${ROCM_VERSION_ARRAY[2]} else + echo "Unhandled ROCM_VERSION ${ROCM_VERSION}" + exit 1 +fi +ROCM_INT=$(($ROCM_VERSION_MAJOR * 10000 + $ROCM_VERSION_MINOR * 100 + $ROCM_VERSION_PATCH)) + +# rocm3.8 and later use TensileLibrary.dat +if [[ $ROCM_INT -ge 30800 ]]; then TENSILE_LIBRARY_NAME=TensileLibrary.dat +else + TENSILE_LIBRARY_NAME=TensileLibrary.yaml fi # in rocm3.9, libamd_comgr path changed from lib to lib64 -if [[ $ROCM_VERSION == "rocm3.7" || $ROCM_VERSION == "rocm3.8" ]]; then - COMGR_LIBDIR="lib" -else +if [[ $ROCM_INT -ge 30900 ]]; then COMGR_LIBDIR="lib64" +else + COMGR_LIBDIR="lib" fi # in rocm4.0, libamdhip64.so.3 changed to *.so.4 -if [[ $ROCM_VERSION == "rocm4.0" ]]; then +if [[ $ROCM_INT -ge 40000 ]]; then LIBAMDHIP64=libamdhip64.so.4 else LIBAMDHIP64=libamdhip64.so.3 fi; -# NOTE: Some ROCm versions have identical dependencies, or very close deps. -# To avoid copy/paste mistakes, version condition branches are combined. -if [[ $ROCM_VERSION == "rocm3.7" || $ROCM_VERSION == "rocm3.8" || $ROCM_VERSION == "rocm3.9" || $ROCM_VERSION == "rocm3.10" || $ROCM_VERSION == "rocm4.0" ]]; then - DEPS_LIST=( "/opt/rocm/miopen/lib/libMIOpen.so.1" "/opt/rocm/hip/lib/$LIBAMDHIP64" @@ -151,11 +170,6 @@ DEPS_AUX_DSTLIST=( "lib/library/$TENSILE_LIBRARY_NAME" ) -else - echo "Unknown ROCm version $ROCM_VERSION" - exit 1 -fi - SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" if [[ -z "$BUILD_PYTHONLESS" ]]; then BUILD_SCRIPT=build_common.sh diff --git a/manywheel/deploy.sh b/manywheel/deploy.sh index 9baacada2..61ad96201 100755 --- a/manywheel/deploy.sh +++ b/manywheel/deploy.sh @@ -2,7 +2,7 @@ set -eou pipefail -for rocm_version in 3.7 3.8 3.9 3.10 4.0; do +for rocm_version in 3.7 3.8 3.9 3.10 4.0 4.0.1; do ( set -x DOCKER_BUILDKIT=1 docker build \