[HIP][Regression] Cannot override ROCm include and library paths

[biergaizi]

Describe the bug
In the past, it was possible to override ROCm include and library paths using the following variables:

• SYCL_BUILD_PI_HIP_ROCM_DIR
• SYCL_BUILD_PI_HIP_ROCM_INCLUDE_DIR
• SYCL_BUILD_PI_HIP_ROCM_LIB_DIR
• SYCL_BUILD_PI_HIP_INCLUDE_DIR
• SYCL_BUILD_PI_HIP_HSA_INCLUDE_DIR
• SYCL_BUILD_PI_HIP_LIB_DIR

However, they no longer work in recent months due to the introduction of an external dependency Unified Runtime which doesn't respect these settings.

Furthermore, it's currently also not possible to set an override in Unified Runtime to work around the problem, because unlike DPC++, UR_HIP_INCLUDE_DIR, UR_HIP_HSA_INCLUDE_DIR and UR_HIP_LIB_DIR are hardcoded to derive from UR_HIP_ROCM_DIR, and cannot be set separately.

To Reproduce

1. Run:

    python buildbot/configure.py --hip \
        --cmake-opt=-DSYCL_BUILD_PI_HIP_ROCM_DIR=/usr \
        --cmake-opt=-DSYCL_BUILD_PI_HIP_ROCM_INCLUDE_DIR=/usr/include \
        --cmake-opt=-DSYCL_BUILD_PI_HIP_ROCM_LIB_DIR=/usr/lib64 \
        --cmake-opt=-DSYCL_BUILD_PI_HIP_INCLUDE_DIR=/usr/include/hsa \
        --cmake-opt=-DSYCL_BUILD_PI_HIP_HSA_INCLUDE_DIR=/usr/include/hsa \
        --cmake-opt=-DSYCL_BUILD_PI_HIP_LIB_DIR=/usr/lib64 \
   

   Unified Runtime doesn't respect these SYCL_BUILD_PI variables.

    -- Will fetch Unified Runtime from https://github.com/oneapi-src/unified-runtime.git
    -- Performing Test USING_LIBSTDCXX
    -- Performing Test USING_LIBSTDCXX - Success
    -- Could NOT find Libbacktrace (missing: LIBBACKTRACE_LIBRARIES LIBBACKTRACE_INCLUDE_DIR)
    -- Using default backtrace for validation
    -- Download Level Zero loader and headers from github.com
    -- Will fetch Level Zero Loader from https://github.com/oneapi-src/level-zero.git
    -- Using patch version from commit count in git repository: 0
    -- Performing Test COMPILER_SUPPORTS_CXX14
    -- Performing Test COMPILER_SUPPORTS_CXX14 - Success
    CMake Warning at /usr/share/cmake/Modules/CPack.cmake:507 (message):
      CPack.cmake has already been included!!
    Call Stack (most recent call first):
      /home/user/dpcpp/llvm/build/_deps/level-zero-loader-src/CMakeLists.txt:255 (INCLUDE)
    
    
    CMake Error at /home/user/dpcpp/llvm/build/_deps/unified-runtime-src/source/adapters/hip/CMakeLists.txt:24 (message):
      Couldn't find the HIP library directory at '/opt/rocm/lib', please check
      ROCm installation.
    
     
    -- Configuring incomplete, errors occurred!
    There is CMakeCache.txt at /home/user/dpcpp/llvm/build/CMakeCache.txt ... you can try to remove it and rerun.
    Configure failed!
   

2. Run:

    python buildbot/configure.py --hip \
            --cmake-opt=-DSYCL_BUILD_PI_HIP_ROCM_DIR=/usr \
            --cmake-opt=-DSYCL_BUILD_PI_HIP_ROCM_INCLUDE_DIR=/usr/include \
            --cmake-opt=-DSYCL_BUILD_PI_HIP_ROCM_LIB_DIR=/usr/lib64 \
            --cmake-opt=-DSYCL_BUILD_PI_HIP_INCLUDE_DIR=/usr/include/hsa \
            --cmake-opt=-DSYCL_BUILD_PI_HIP_HSA_INCLUDE_DIR=/usr/include/hsa \
            --cmake-opt=-DSYCL_BUILD_PI_HIP_LIB_DIR=/usr/lib64 \
            --cmake-opt=-DUR_HIP_ROCM_DIR=/usr \
            --cmake-opt=-DUR_HIP_INCLUDE_DIR=/usr/include \
            --cmake-opt=-DUR_HIP_HSA_INCLUDE_DIR=/usr/include/hsa \
            --cmake-opt=-DUR_HIP_LIB_DIR=/usr/lib64
   

   It's possible to override UR_HIP_ROCM_DIR, but still not possible to override UR_HIP_INCLUDE_DIR, UR_HIP_HSA_INCLUDE_DIR, UR_HIP_HSA_INCLUDE_DIR, or UR_HIP_LIB_DIR.

    CMake Error at /home/user/dpcpp/llvm/build/_deps/unified-runtime-src/source/adapters/hip/CMakeLists.txt:35 (message):
      Couldn't find the HSA include directory at '/usr/hsa/include', please check
      ROCm installation.
   

   This is because Unified Runtime devires these variables from UR_HIP_ROCM_DIR in a hardcoded manner in build/_deps/unified-runtime-src/source/adapters/hip/CMakeLists.txt.

    # Set default ROCm installation directory
    set(UR_HIP_ROCM_DIR "/opt/rocm" CACHE STRING "ROCm installation dir")
   
    set(UR_HIP_INCLUDE_DIR "${UR_HIP_ROCM_DIR}/include")
   
    set(UR_HIP_HSA_INCLUDE_DIR "${UR_HIP_ROCM_DIR}/hsa/include")
   
    # Set HIP lib dir
    set(UR_HIP_LIB_DIR "${UR_HIP_ROCM_DIR}/lib")
   

   Meanwhile, DPC++ checks whether the string is non-empty, before using the default relative paths derived from SYCL_BUILD_PI_HIP_ROCM_DIR - Unified Runtime should do the same. DPC++ should also pass the respective SYCL_BUILD_PI_ variables to Unified Runtime during configuration.

   if("${SYCL_BUILD_PI_HIP_INCLUDE_DIR}" STREQUAL "")
     set(PI_HIP_INCLUDE_DIR "${SYCL_BUILD_PI_HIP_ROCM_DIR}/include")
   else()
     set(PI_HIP_INCLUDE_DIR "${SYCL_BUILD_PI_HIP_INCLUDE_DIR}")
   endif()
   
   if("${SYCL_BUILD_PI_HIP_HSA_INCLUDE_DIR}" STREQUAL "")
     set(PI_HIP_HSA_INCLUDE_DIR "${SYCL_BUILD_PI_HIP_ROCM_DIR}/hsa/include")
   else()
     set(PI_HIP_HSA_INCLUDE_DIR "${SYCL_BUILD_PI_HIP_INCLUDE_DIR}")
   endif()
   
   # Set HIP lib dir
   set(SYCL_BUILD_PI_HIP_LIB_DIR "" CACHE STRING "Override HIP lib dir path (set to \"\" for default behavior)")
   if("${SYCL_BUILD_PI_HIP_LIB_DIR}" STREQUAL "")
     set(PI_HIP_LIB_DIR "${SYCL_BUILD_PI_HIP_ROCM_DIR}/lib")
   else()
     set(PI_HIP_LIB_DIR "${SYCL_BUILD_PI_HIP_LIB_DIR}")
   endif()
   

Environment (please complete the following information):

• OS: Gentoo
• Target device and vendor: AMD HIP
• DPC++ version: 096676e

Additional context

Gentoo is one of the few systems that install ROCm "properly" to standard system paths like /usr, /usr/lib64 or /usr/include. The ability to override all paths under /opt/rocm is a critical feature, otherwise DPC++ is uninstallable. Today, I was asked to verify a fix to a bug that I reported, but because of this regression, I can no longer install DPC++.

Workaround

As a workaround, install DPC++ this way. First, use CMake to override SYCL_BUILD and UR_HIP variables as one normally would (the following example is known to work under Gentoo):

python buildbot/configure.py --hip \
        --cmake-opt=-DSYCL_BUILD_PI_HIP_ROCM_DIR=/usr \
        --cmake-opt=-DSYCL_BUILD_PI_HIP_ROCM_INCLUDE_DIR=/usr/include \
        --cmake-opt=-DSYCL_BUILD_PI_HIP_ROCM_LIB_DIR=/usr/lib64 \
        --cmake-opt=-DSYCL_BUILD_PI_HIP_INCLUDE_DIR=/usr/include/hsa \
        --cmake-opt=-DSYCL_BUILD_PI_HIP_HSA_INCLUDE_DIR=/usr/include/hsa \
        --cmake-opt=-DSYCL_BUILD_PI_HIP_LIB_DIR=/usr/lib64 \
        --cmake-opt=-DUR_HIP_ROCM_DIR=/usr \
        --cmake-opt=-DUR_HIP_INCLUDE_DIR=/usr/include \
        --cmake-opt=-DUR_HIP_HSA_INCLUDE_DIR=/usr/include/hsa \
        --cmake-opt=-DUR_HIP_LIB_DIR=/usr/lib64

Then, when you see the error message:

CMake Error at /home/user/dpcpp/llvm/build/_deps/unified-runtime-src/source/adapters/hip/CMakeLists.txt:35 (message):
 Couldn't find the HSA include directory at '/usr/hsa/include', please check
 ROCm installation

Now open build/_deps/unified-runtime-src/source/adapters/hip/CMakeLists.txt, and delete the following lines from this auto-generated file:

# Set default ROCm installation directory
set(UR_HIP_ROCM_DIR "/opt/rocm" CACHE STRING "ROCm installation dir")

set(UR_HIP_INCLUDE_DIR "${UR_HIP_ROCM_DIR}/include")

set(UR_HIP_HSA_INCLUDE_DIR "${UR_HIP_ROCM_DIR}/hsa/include")

# Set HIP lib dir
set(UR_HIP_LIB_DIR "${UR_HIP_ROCM_DIR}/lib")

Finally repeat Step 1.

[jchlanda]

@biergaizi you should be able to use:

• SYCL_BUILD_PI_HIP_INCLUDE_DIR
• SYCL_BUILD_PI_HIP_HSA_INCLUDE_DIR
• SYCL_BUILD_PI_HIP_LIB_DIR
  to override the location of HIP include/library directories.

Please see: oneapi-src/unified-runtime#1170 and: #12127 for details.

Closing this ticket now, but if you experience any more problems please feel free to either re-open or submit a new ticket.

Thank you.