Fix HIP, hwloc and test_install

CMakeLists.txt

@@ -65,15 +65,16 @@ set(GINKGO_HIP_COMPILER_FLAGS "" CACHE STRING
     "Set the required HIP compiler flags. Current default is an empty string.")
 set(GINKGO_HIP_NVCC_COMPILER_FLAGS "" CACHE STRING
     "Set the required HIP nvcc compiler flags. Current default is an empty string.")
-set(GINKGO_HIP_HCC_COMPILER_FLAGS "" CACHE STRING
-    "Set the required HIP HCC compiler flags. Current default is an empty string.")
 set(GINKGO_HIP_CLANG_COMPILER_FLAGS "" CACHE STRING
     "Set the required HIP CLANG compiler flags. Current default is an empty string.")
 set(GINKGO_HIP_AMDGPU "" CACHE STRING
     "The amdgpu_target(s) variable passed to hipcc. The default is none (auto).")
 option(GINKGO_JACOBI_FULL_OPTIMIZATIONS "Use all the optimizations for the CUDA Jacobi algorithm" OFF)
 option(BUILD_SHARED_LIBS "Build shared (.so, .dylib, .dll) libraries" ON)
 option(GINKGO_BUILD_HWLOC "Build Ginkgo with HWLOC. Default is ON. If a system HWLOC is not found, then we try to build it ourselves. Switch this OFF to disable HWLOC." ON)
+option(GINKGO_INSTALL_RPATH "Set the RPATH when installing its libraries." ON)
+option(GINKGO_INSTALL_RPATH_ORIGIN "Add $ORIGIN (Linux) or @loader_path (MacOS) to the installation RPATH." ON)
+option(GINKGO_INSTALL_RPATH_DEPENDENCIES "Add dependencies to the installation RPATH." OFF)
 
 set(GINKGO_CIRCULAR_DEPS_FLAGS "-Wl,--no-undefined")
 
@@ -191,12 +192,22 @@ if(GINKGO_BUILD_HIP)
     endif()
 endif()
 
-configure_file(${Ginkgo_SOURCE_DIR}/include/ginkgo/config.hpp.in
-    ${Ginkgo_BINARY_DIR}/include/ginkgo/config.hpp @ONLY)
+
+# Try to find the third party packages before using our subdirectories
+include(cmake/package_helpers.cmake)
+ginkgo_find_package(GTest "GTest::GTest;GTest::Main" FALSE 1.8.1)
+ginkgo_find_package(gflags gflags FALSE 2.2.2)
+ginkgo_find_package(RapidJSON rapidjson TRUE 1.1.0)
+if(GINKGO_HAVE_HWLOC)
+    ginkgo_find_package(HWLOC hwloc FALSE 2.1)
+    if (NOT HWLOC_FOUND)
+        set(GINKGO_USE_EXTERNAL_HWLOC 0)
+    endif()
+endif()
+add_subdirectory(third_party)    # Third-party tools and libraries
 
 # Load CMake helpers
 include(cmake/build_helpers.cmake)
-include(cmake/hip_helpers.cmake)
 include(cmake/install_helpers.cmake)
 include(cmake/windows_helpers.cmake)
 
@@ -214,40 +225,30 @@ if(MSVC)
     endif()
 endif()
 
-# Try to find the third party packages before using our subdirectories
-include(cmake/package_helpers.cmake)
-ginkgo_find_package(GTest "GTest::GTest;GTest::Main" FALSE 1.8.1)
-ginkgo_find_package(gflags gflags FALSE 2.2.2)
-ginkgo_find_package(RapidJSON rapidjson TRUE 1.1.0)
-if(GINKGO_HAVE_HWLOC)
-    ginkgo_find_package(HWLOC hwloc FALSE 2.1)
-endif()
-set(GKO_HWLOC_XMLFILE "$ENV{HWLOC_XMLFILE}")
-message(STATUS "HWLOC XML file set to: ${GKO_HWLOC_XMLFILE}")
-add_subdirectory(third_party)    # Third-party tools and libraries
+configure_file(${Ginkgo_SOURCE_DIR}/include/ginkgo/config.hpp.in
+    ${Ginkgo_BINARY_DIR}/include/ginkgo/config.hpp @ONLY)
 
 # Ginkgo core libraries
 # Needs to be first in order for `CMAKE_CUDA_DEVICE_LINK_EXECUTABLE` to be
 # propagated to the other parts of Ginkgo in case of building as static libraries
+add_subdirectory(devices)        # Basic device functionalities. Always compiled.
 if(GINKGO_BUILD_CUDA)
     add_subdirectory(cuda)       # High-performance kernels for NVIDIA GPUs
 endif()
-add_subdirectory(core)           # Core Ginkgo types and top-level functions
-add_subdirectory(include)        # Public API self-contained check
 if (GINKGO_BUILD_REFERENCE)
     add_subdirectory(reference)  # Reference kernel implementations
 endif()
-if (GINKGO_BUILD_OMP)
-    add_subdirectory(omp)        # High-performance omp kernels
+if(GINKGO_BUILD_HIP)
+    add_subdirectory(hip)        # High-performance kernels for AMD or NVIDIA GPUs
 endif()
 if (GINKGO_BUILD_DPCPP)
     add_subdirectory(dpcpp)        # High-performance DPC++ kernels
 endif()
-# HIP needs to be last because it builds the GINKGO_RPATH_FOR_HIP variable
-# which needs to know the `ginkgo` target.
-if(GINKGO_BUILD_HIP)
-    add_subdirectory(hip)        # High-performance kernels for AMD or NVIDIA GPUs
+if (GINKGO_BUILD_OMP)
+    add_subdirectory(omp)        # High-performance omp kernels
 endif()
+add_subdirectory(core)           # Core Ginkgo types and top-level functions
+add_subdirectory(include)        # Public API self-contained check
 
 # Non core directories and targets
 if(GINKGO_BUILD_EXAMPLES)
@@ -322,27 +323,31 @@ if(MSVC)
     set(GINKGO_TEST_INSTALL_COMMAND "${Ginkgo_BINARY_DIR}/test_install/$<CONFIG>/test_install")
     set(GINKGO_TEST_EXPORTBUILD_COMMAND "${Ginkgo_BINARY_DIR}/test_exportbuild/$<CONFIG>/test_exportbuild")
     if(GINKGO_BUILD_CUDA)
-        set(GINKGO_TEST_INSTALL_COMMAND "${GINKGO_TEST_INSTALL_COMMAND}" "${Ginkgo_BINARY_DIR}/test_install/$<CONFIG>/test_install_cuda")
+        set(GINKGO_TEST_INSTALL_CUDA_COMMAND "${Ginkgo_BINARY_DIR}/test_install/$<CONFIG>/test_install_cuda")
     endif()
 else()
     set(GINKGO_TEST_INSTALL_COMMAND "${Ginkgo_BINARY_DIR}/test_install/test_install")
     set(GINKGO_TEST_EXPORTBUILD_COMMAND "${Ginkgo_BINARY_DIR}/test_exportbuild/test_exportbuild")
     if(GINKGO_BUILD_CUDA)
-        set(GINKGO_TEST_INSTALL_COMMAND "${GINKGO_TEST_INSTALL_COMMAND}" "${Ginkgo_BINARY_DIR}/test_install/test_install_cuda")
+        set(GINKGO_TEST_INSTALL_CUDA_COMMAND "${Ginkgo_BINARY_DIR}/test_install/test_install_cuda")
+    endif()
+    if(GINKGO_BUILD_HIP)
+        set(GINKGO_TEST_INSTALL_HIP_COMMAND "${Ginkgo_BINARY_DIR}/test_install/test_install_hip")
     endif()
 endif()
 add_custom_target(test_install
     COMMAND ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} -H${Ginkgo_SOURCE_DIR}/test_install
     -B${Ginkgo_BINARY_DIR}/test_install
     -DCMAKE_PREFIX_PATH=${CMAKE_INSTALL_PREFIX}/${GINKGO_INSTALL_CONFIG_DIR}
-    -DHWLOC_DIR=${HWLOC_DIR}
     -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
     -DCMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER}
     # `--config cfg` is ignored by single-configuration generator.
     # `$<CONFIG>` is always be the same as `CMAKE_BUILD_TYPE` in
     # single-configuration generator.
     COMMAND ${CMAKE_COMMAND} --build ${Ginkgo_BINARY_DIR}/test_install --config $<CONFIG>
     COMMAND ${GINKGO_TEST_INSTALL_COMMAND}
+    COMMAND ${GINKGO_TEST_INSTALL_CUDA_COMMAND}
+    COMMAND ${GINKGO_TEST_INSTALL_HIP_COMMAND}
     COMMENT "Running a test on the installed binaries. This requires running `(sudo) make install` first.")
 
 add_custom_target(test_exportbuild

INSTALL.md

@@ -65,6 +65,13 @@ Ginkgo adds the following additional switches to control what is being built:
 *   `-DGINKGO_VERBOSE_LEVEL=integer` sets the verbosity of Ginkgo.
     * `0` disables all output in the main libraries,
     * `1` enables a few important messages related to unexpected behavior (default).
+*   `GINKGO_INSTALL_RPATH` allows setting any RPATH information when installing
+    the Ginkgo libraries. If this is `OFF`, the behavior is the same as if all
+    other RPATH flags are set to `OFF` as well. The default is `ON`.
+*   `GINKGO_INSTALL_RPATH_ORIGIN` adds $ORIGIN (Linux) or @loader_path (MacOS)
+    to the installation RPATH. The default is `ON`.
+*   `GINKGO_INSTALL_RPATH_DEPENDENCIES` adds the dependencies to the
+    installation RPATH. The default is `OFF`.
 *   `-DCMAKE_INSTALL_PREFIX=path` sets the installation path for `make install`.
     The default value is usually something like `/usr/local`.
 *   `-DCMAKE_BUILD_TYPE=type` specifies which configuration will be used for
@@ -192,14 +199,21 @@ Ginkgo's HIP backend adds a dependency to the following packages:
 All HIP installation paths can be configured through the use of environment
 variables or CMake variables. This way of configuring the paths is currently
 imposed by the `HIP` tool suite. The variables are the following:
-+ CMake `-DHIP_PATH=` or  environment `export HIP_PATH=`: sets the `HIP`
-  installation path. The default value is `/opt/rocm/hip`.
-+ CMake `-DHIPBLAS_PATH=` or  environment `export HIPBLAS_PATH=`: sets the
-  `hipBLAS` installation path. The default value is `/opt/rocm/hipblas`.
-+ CMake `-DHIPSPARSE_PATH=` or  environment `export HIPSPARSE_PATH=`: sets the
-  `hipSPARSE` installation path. The default value is `/opt/rocm/hipsparse`.
-+ CMake `-DHCC_PATH=` or  environment `export HCC_PATH=`: sets the `HCC`
-  installation path, for AMD backends. The default value is `/opt/rocm/hcc`.
++ CMake `-DROCM_PATH=` or environment `export ROCM_PATH=`: sets the `ROCM`
+  installation path. The default value is `/opt/rocm/`.
++ CMake `-DHIP_CLANG_PATH` or environment `export HIP_CLANG_PATH=`: sets the
+  `HIP` compatible `clang` binary path. The default value is
+  `${ROCM_PATH}/llvm/bin`.
++ CMake `-DHIP_PATH=` or environment `export HIP_PATH=`: sets the `HIP`
+  installation path. The default value is `${ROCM_PATH}/hip`.
++ CMake `-DHIPBLAS_PATH=` or environment `export HIPBLAS_PATH=`: sets the
+  `hipBLAS` installation path. The default value is `${ROCM_PATH}/hipblas`.
++ CMake `-DHIPSPARSE_PATH=` or environment `export HIPSPARSE_PATH=`: sets the
+  `hipSPARSE` installation path. The default value is `${ROCM_PATH}/hipsparse`.
++ CMake `-DROCRAND_PATH=` or environment `export ROCRAND_PATH=`: sets the
+  `rocRAND` installation path. The default value is `${ROCM_PATH}/rocrand`.
++ CMake `-DHIPRAND_PATH=` or environment `export HIPRAND_PATH=`: sets the
+  `hipRAND` installation path. The default value is `${ROCM_PATH}/hiprand`.
 + environment `export CUDA_PATH=`: where `hipcc` can find `CUDA` if it is not in
   the default `/usr/local/cuda` path.
 
@@ -216,9 +230,9 @@ export HIP_PLATFORM=nvcc
 ```
 
 When using `HIP_PLATFORM=hcc`, note that two `HIP` compilers can be set, the old
-`hcc` or since ROCm 3.5, `clang`. On newer installations, `clang` should be set
-by default, but if encountering any problem try to manually set the `HIP`
-compiler to `clang`:
+`hcc` or since ROCm 3.5, `clang`. Ginkgo is only compatible with the `clang`
+based installations. Although this setting should be automatically done, it is
+also possible to manually set the `HIP` compiler to `clang`:
 ```
 export HIP_COMPILER=clang
 ```
@@ -227,7 +241,6 @@ export HIP_COMPILER=clang
 Platform specific compilation flags can be given through the following
 CMake variables:
 + `-DGINKGO_HIP_COMPILER_FLAGS=`: compilation flags given to all platforms.
-+ `-DGINKGO_HIP_HCC_COMPILER_FLAGS=`: compilation flags given to AMD platforms.
 + `-DGINKGO_HIP_NVCC_COMPILER_FLAGS=`: compilation flags given to NVIDIA platforms.
 + `-DGINKGO_HIP_CLANG_COMPILER_FLAGS=`: compilation flags given to AMD clang compiler.
 
@@ -257,7 +270,9 @@ packages `GTEST`, `GFLAGS`, `RAPIDJSON` and `CAS`, it is possible to force
 Ginkgo to try to use an external version of a package. For this, Ginkgo provides
 two ways to find packages. To rely on the CMake `find_package` command, use the
 CMake option `-DGINKGO_USE_EXTERNAL_<package>=ON`. `HWLOC` works the opposite
-way, Ginkgo always looks for the system's `hwloc` first.
+way, Ginkgo always looks for the system's `hwloc` first. In addition, when
+installing Ginkgo, `hwloc` will be installed as well into the Ginkgo directory
+if there were no system `hwloc` detected.
 
 Note that, if the external packages were not installed to the default location,
 the CMake option `-DCMAKE_PREFIX_PATH=<path-list>` needs to be set to the
@@ -277,6 +292,9 @@ format):
 When applicable (e.g. for `GTest` libraries), a `;` separated list can be given
 to the `TPL_<package>_{LIBRARIES|INCLUDE_DIRS}` variables.
 
+Note that for convenience, the options `GINKGO_INSTALL_RPATH[_.*]` can be used
+to make bind the Ginkgo shared libraries to the path of its dependencies.
+
 ### Installing Ginkgo
 
 To install Ginkgo into the specified folder, execute the following command in

README.md

@@ -61,10 +61,10 @@ following:
 
 The Ginkgo HIP module has the following __additional__ requirements:
 
-* _ROCm 2.8+_
-*    the HIP, hipBLAS and hipSPARSE packages compiled with either:
-    * _AMD_ backend
-    * _CUDA 9.0+_ backend. When using CUDA 10+, _cmake 3.12.2+_ is required.
+* _ROCm 3.5+_
+*    the HIP, hipBLAS, hipSPARSE, hip/rocRAND packages compiled with either:
+    * _AMD_ backend (using the `clang` compiler)
+    * _CUDA 9.2+_ backend. When using CUDA 10+, _cmake 3.12.2+_ is required.
 
 ### Windows
 

benchmark/CMakeLists.txt

@@ -45,18 +45,6 @@ function(ginkgo_benchmark_hipsp_linops name)
         ${HSA_HEADER} ${HIP_INCLUDE_DIRS}
         ${HIPBLAS_INCLUDE_DIRS} ${HIPSPARSE_INCLUDE_DIRS})
 
-    if(GINKGO_HIP_PLATFORM MATCHES "hcc")
-        ginkgo_hip_ban_link_hcflag(hcc::hccrt)
-        ginkgo_hip_ban_link_hcflag(hcc::hc_am)
-        ginkgo_hip_ban_link_hcflag(hcc::mcwamp)
-        ginkgo_hip_ban_compile_hcflag(hcc::hccrt)
-        ginkgo_hip_ban_compile_hcflag(hcc::hc_am)
-        ginkgo_hip_ban_compile_hcflag(hcc::mcwamp)
-        ginkgo_hip_clang_ban_hip_device_flags()
-        target_link_libraries("${name}" hip::device)
-    else()
-        target_link_libraries("${name}" ${HIP_CUDA_LIBRARIES})
-    endif()
     target_link_libraries("${name}" ${HIPSPARSE_LIBRARIES})
 endfunction()
 

cmake/GinkgoConfig.cmake.in

@@ -107,6 +107,7 @@ set(GINKGO_INSTALL_LIBRARY_DIR "${GINKGO_INSTALL_PREFIX}/@GINKGO_INSTALL_LIBRARY
 set(GINKGO_INSTALL_PKGCONFIG_DIR "${GINKGO_INSTALL_PREFIX}/@GINKGO_INSTALL_PKGCONFIG_DIR@")
 set(GINKGO_INSTALL_CONFIG_DIR "${GINKGO_INSTALL_PREFIX}/@GINKGO_INSTALL_CONFIG_DIR@")
 set(GINKGO_INSTALL_MODULE_DIR "${GINKGO_INSTALL_PREFIX}/@GINKGO_INSTALL_MODULE_DIR@")
+set(GINKGO_INSTALL_RPATH_FOR_HIP "-Wl,-rpath,${GINKGO_INSTALL_LIBRARY_DIR}")
 
 # Forward Ginkgo's MODULE PATH and the PREFIX PATH for HIP and more
 list(APPEND CMAKE_MODULE_PATH "@CMAKE_MODULE_PATH@" "${GINKGO_INSTALL_MODULE_DIR}")
@@ -139,10 +140,6 @@ set(GINKGO_OPENMP_LIBRARIES @OpenMP_CXX_LIBRARIES@)
 
 set(GINKGO_OPENMP_FLAGS "@OpenMP_CXX_FLAGS@")
 
-# Provide useful HIP helper functions
-if(GINKGO_BUILD_HIP)
-    include(${CMAKE_CURRENT_LIST_DIR}/hip_helpers.cmake)
-endif()
 if (WIN32 OR CYGWIN)
     include(${CMAKE_CURRENT_LIST_DIR}/windows_helpers.cmake)
 endif()
@@ -181,15 +178,6 @@ if((NOT GINKGO_BUILD_SHARED_LIBS) AND GINKGO_BUILD_HIP)
     find_package(hiprand REQUIRED)
     find_package(hipsparse REQUIRED)
     find_package(rocrand REQUIRED)
-    if(GINKGO_HIP_PLATFORM MATCHES "hcc")
-        ginkgo_hip_ban_link_hcflag(hcc::hccrt)
-        ginkgo_hip_ban_link_hcflag(hcc::hc_am)
-        ginkgo_hip_ban_link_hcflag(hcc::mcwamp)
-        ginkgo_hip_ban_compile_hcflag(hcc::hccrt)
-        ginkgo_hip_ban_compile_hcflag(hcc::hc_am)
-        ginkgo_hip_ban_compile_hcflag(hcc::mcwamp)
-    endif()
 endif()
 
-
 include(${CMAKE_CURRENT_LIST_DIR}/GinkgoTargets.cmake)

cmake/Modules/FindHWLOC.cmake

@@ -40,20 +40,17 @@ include(hwloc_helpers)
 find_path(HWLOC_INCLUDE_DIRS
     NAMES "hwloc.h"
     HINTS ${HWLOC_DIR} $ENV{HWLOC_DIR}
-    PATH_SUFFIXES include src/include
+    PATH_SUFFIXES include
     DOC "Find the hwloc.h main header"
     )
 
 find_library(HWLOC_LIBRARIES "hwloc"
     HINTS ${HWLOC_DIR} $ENV{HWLOC_DIR}
-    PATH_SUFFIXES hwloc/lib lib lib64
+    PATH_SUFFIXES lib lib64
     DOC "Find the hwloc library"
     )
 
 if (HWLOC_INCLUDE_DIRS)
-    unset(HWLOC_FOUND CACHE)
-    set(HWLOC_FOUND 1)
-
     # Find the version of hwloc found
     if(NOT HWLOC_VERSION)
         file(READ "${HWLOC_INCLUDE_DIRS}/hwloc.h"
@@ -68,25 +65,26 @@ if (HWLOC_INCLUDE_DIRS)
         get_dec_from_hex("${HEX_PATCH}" DEC_PATCH)
         set(HWLOC_VERSION "${DEC_MAJOR}.${DEC_MINOR}.${DEC_PATCH}" CACHE STRING "HWLOC version")
     endif()
+endif()
 
-    if (NOT HWLOC_FIND_QUIETLY)
-        if (HWLOC_FOUND AND HWLOC_LIBRARIES)
-            message(STATUS "Looking for HWLOC - found version ${HWLOC_VERSION}")
-        else()
-            message(STATUS "${Magenta}Looking for HWLOC - not found"
-                "\n   Please check that your environment variable HWLOC_DIR"
-                "\n   has been set properly.${ColourReset}")
-        endif()
+if (NOT HWLOC_FIND_QUIETLY)
+    if (HWLOC_INCLUDE_DIRS AND HWLOC_LIBRARIES)
+        message(STATUS "Looking for HWLOC - found version ${HWLOC_VERSION}")
+    else()
+        message(STATUS "${Magenta}Looking for HWLOC - not found"
+            "\n   Please check that your environment variable HWLOC_DIR"
+            "\n   has been set properly.${ColourReset}")
+        unset(HWLOC_LIBRARIES CACHE)
+        unset(HWLOC_INCLUDE_DIRS CACHE)
     endif()
 endif()
 
-# check a function to validate the find
-if(HWLOC_FOUND AND HWLOC_LIBRARIES)
-
-    # set required libraries for link
+# check a function to validate what was found
+if(HWLOC_INCLUDE_DIRS AND HWLOC_LIBRARIES)
+    # set required libraries for linking
     ginkgo_set_required_test_lib_link(HWLOC)
 
-    # test link
+    # test linking
     unset(HWLOC_WORKS CACHE)
     include(CheckFunctionExists)
     check_function_exists(hwloc_topology_init HWLOC_WORKS)
@@ -108,29 +106,23 @@ if(HWLOC_FOUND AND HWLOC_LIBRARIES)
     string(SUBSTRING "${HWLOC_VERSION}" 0 3 HWLOC_VERSION)
     if(HWLOC_VERSION VERSION_LESS HWLOC_FIND_VERSION)
         message(STATUS "Required version ${HWLOC_FIND_VERSION}, but found version ${HWLOC_VERSION}")
-        set(HWLOC_FOUND 0)
-        unset(HWLOC_LIBRARIES)
-        unset(HWLOC_INCLUDE_DIRS)
+        unset(HWLOC_FOUND CACHE)
+        unset(HWLOC_LIBRARIES CACHE)
+        unset(HWLOC_INCLUDE_DIRS CACHE)
     else()
+        unset(HWLOC_FOUND CACHE)
+        set(HWLOC_FOUND 1)
         include(FindPackageHandleStandardArgs)
         find_package_handle_standard_args(HWLOC
             REQUIRED_VARS HWLOC_LIBRARIES HWLOC_INCLUDE_DIRS HWLOC_WORKS
             VERSION_VAR HWLOC_VERSION)
         mark_as_advanced(HWLOC_INCLUDE_DIRS HWLOC_LIBRARIES HWLOC_VERSION HWLOC_WORKS)
     endif()
-
-endif(HWLOC_FOUND AND HWLOC_LIBRARIES)
+endif(HWLOC_INCLUDE_DIRS AND HWLOC_LIBRARIES)
 
 if(HWLOC_FOUND)
     add_library(hwloc SHARED IMPORTED GLOBAL)
     set_target_properties(hwloc PROPERTIES IMPORTED_LOCATION ${HWLOC_LIBRARIES})
     set_target_properties(hwloc PROPERTIES INTERFACE_LINK_LIBRARIES ${HWLOC_LIBRARIES})
     set_target_properties(hwloc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${HWLOC_INCLUDE_DIRS})
 endif()
-
-if(HWLOC_FIND_REQUIRED AND NOT HWLOC_FOUND)
-    unset(HWLOC_LIBRARIES)
-    unset(HWLOC_INCLUDE_DIRS)
-    message(SEND_ERROR "HWLOC could not be found. A version mismatch could have occured. The version found was ${HWLOC_VERSION}.\n"
-        "Hints where hwloc is installed can be given in the HWLOC_DIR variable. Current HWLOC_DIR: ${HWLOC_DIR}")
-endif()