Add three new options to control RPATH behavior.

CMakeLists.txt

@@ -72,6 +72,9 @@ set(GINKGO_HIP_AMDGPU "" CACHE STRING
 option(GINKGO_JACOBI_FULL_OPTIMIZATIONS "Use all the optimizations for the CUDA Jacobi algorithm" OFF)
 option(BUILD_SHARED_LIBS "Build shared (.so, .dylib, .dll) libraries" ON)
 option(GINKGO_BUILD_HWLOC "Build Ginkgo with HWLOC. Default is ON. If a system HWLOC is not found, then we try to build it ourselves. Switch this OFF to disable HWLOC." ON)
+option(GINKGO_INSTALL_RPATH "Set the RPATH when installing its libraries." ON)
+option(GINKGO_INSTALL_RPATH_ORIGIN "Add $ORIGIN (Linux) or @loader_path (MacOS) to the installation RPATH." ON)
+option(GINKGO_INSTALL_RPATH_DEPENDENCIES "Add dependencies to the installation RPATH." OFF)
 
 set(GINKGO_CIRCULAR_DEPS_FLAGS "-Wl,--no-undefined")
 

INSTALL.md

@@ -65,6 +65,12 @@ Ginkgo adds the following additional switches to control what is being built:
 *   `-DGINKGO_VERBOSE_LEVEL=integer` sets the verbosity of Ginkgo.
     * `0` disables all output in the main libraries,
     * `1` enables a few important messages related to unexpected behavior (default).
+*   `GINKGO_INSTALL_RPATH` sets the RPATH when installing the Ginkgo libraries.
+    The default is `ON`.
+*   `GINKGO_INSTALL_RPATH_ORIGIN` adds $ORIGIN (Linux) or @loader_path (MacOS)
+    to the installation RPATH. The default is `ON`.
+*   `GINKGO_INSTALL_RPATH_DEPENDENCIES` adds the dependencies to the
+    installation RPATH. The default is `OFF`.
 *   `-DCMAKE_INSTALL_PREFIX=path` sets the installation path for `make install`.
     The default value is usually something like `/usr/local`.
 *   `-DCMAKE_BUILD_TYPE=type` specifies which configuration will be used for
@@ -285,6 +291,9 @@ format):
 When applicable (e.g. for `GTest` libraries), a `;` separated list can be given
 to the `TPL_<package>_{LIBRARIES|INCLUDE_DIRS}` variables.
 
+Note that for convenience, the options `GINKGO_INSTALL_RPATH[_.*]` can be used
+to make bind the Ginkgo shared libraries to the path of its dependencies.
+
 ### Installing Ginkgo
 
 To install Ginkgo into the specified folder, execute the following command in

cmake/install_helpers.cmake

@@ -8,27 +8,32 @@ set(GINKGO_INSTALL_PKGCONFIG_DIR "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
 set(GINKGO_INSTALL_CONFIG_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/Ginkgo")
 set(GINKGO_INSTALL_MODULE_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/Ginkgo/Modules")
 
-function(ginkgo_install_library name subdir)
-    # This setting sets the directory of the library as the `INSTALL_RPATH`.
-    # This is required for `libginkgo.so` to find its dependencies such as
-    # `libginkgo_omp.so` etc. Note that with this setting it's not possible on
-    # Linux to control which Ginkgo libraries are found and where: if all the
-    # libraries are present in the directory, these will be used first and
-    # foremost and the environment variable `LD_LIBRARY_PATH` will have no
-    # effect.
+function(ginkgo_add_install_rpath name subdir)
     if (BUILD_SHARED_LIBS)
-        set (HWLOC_LIB_PATH "")
-        if  (GINKGO_HAVE_HWLOC AND GINKGO_USE_EXTERNAL_HWLOC)
-            get_filename_component(HWLOC_LIB_PATH ${HWLOC_LIBRARIES} DIRECTORY)
+        if (GINKGO_INSTALL_RPATH_ORIGIN)
+            if (APPLE)
+                set(ORIGIN_OR_LOADER_PATH "@loader_path")
+            else()
+                set(ORIGIN_OR_LOADER_PATH "$ORIGIN")
+            endif()
+        endif()
+        if (GINKGO_INSTALL_RPATH_DEPENDENCIES)
+            set(RPATH_DEPENDENCIES "${ARGN}")
+            if  (GINKGO_HAVE_HWLOC AND GINKGO_USE_EXTERNAL_HWLOC)
+                get_filename_component(HWLOC_LIB_PATH ${HWLOC_LIBRARIES} DIRECTORY)
+                list(APPEND RPATH_DEPENDENCIES "${HWLOC_LIBRARIES}")
+            endif()
         endif()
-        if (APPLE)
-            set(ORIGIN_OR_LOADER_PATH "@loader_path")
-        else()
-            set(ORIGIN_OR_LOADER_PATH "$ORIGIN")
+        if (GINKGO_INSTALL_RPATH)
+            set_property(TARGET "${name}" PROPERTY INSTALL_RPATH
+                "${ORIGIN_OR_LOADER_PATH}" "${RPATH_DEPENDENCIES}")
         endif()
-        set_property(TARGET "${name}" PROPERTY INSTALL_RPATH
-        "${ORIGIN_OR_LOADER_PATH}" "${HWLOC_LIB_PATH}" "${ARGN}")
     endif()
+endfunction()
+
+function(ginkgo_install_library name subdir)
+    ginkgo_add_install_rpath("${name}" "${subdir}" "${ARGN}")
+
     if (WIN32 OR CYGWIN)
         # dll is considered as runtime
         install(TARGETS "${name}"

core/CMakeLists.txt

@@ -64,18 +64,22 @@ add_library(Ginkgo::ginkgo ALIAS ginkgo)
 target_link_libraries(ginkgo
     PUBLIC ginkgo_device ginkgo_omp ginkgo_cuda ginkgo_reference ginkgo_hip ginkgo_dpcpp)
 # The PAPI dependency needs to be exposed to the user.
+set(GKO_RPATH_ADDITIONS "")
 if (GINKGO_HAVE_PAPI_SDE)
     target_link_libraries(ginkgo PUBLIC PAPI::PAPI)
+    list(GET PAPI_LIBRARIES 0 PAPI_FIRST_LIB)
+    get_filename_component(GKO_PAPI_LIBDIR "${PAPI_FIRST_LIB}" DIRECTORY)
+    list(APPEND GKO_RPATH_ADDITIONS "${GKO_PAPI_LIBDIR}")
 endif()
 
-# Since we have a public dependency on HIP, this dependency appears here as well
-set(RPATH_ADD_HIP_PATH "")
+# Since we have a public dependency on HIP, this dependency appears
+# here as well
 if (GINKGO_BUILD_HIP AND GINKGO_HIP_PLATFORM MATCHES "hcc")
-    set(RPATH_ADD_HIP_PATH "${HIP_PATH}/lib")
+    list(APPEND GKO_RPATH_ADDITIONS "${HIP_PATH}/lib")
 endif()
 
 ginkgo_default_includes(ginkgo)
-ginkgo_install_library(ginkgo core "${RPATH_ADD_HIP_PATH}")
+ginkgo_install_library(ginkgo core "${GKO_RPATH_ADDITIONS}")
 
 if (GINKGO_CHECK_CIRCULAR_DEPS)
     ginkgo_check_headers(ginkgo)

cuda/CMakeLists.txt

@@ -163,9 +163,11 @@ target_link_libraries(ginkgo_cuda PRIVATE ${CUDA_RUNTIME_LIBS} ${CUBLAS} ${CUSPA
 target_link_libraries(ginkgo_cuda PUBLIC ginkgo_device)
 target_compile_options(ginkgo_cuda
         PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:${GINKGO_CUDA_ARCH_FLAGS}>")
+list(GET CUDA_RUNTIME_LIBS 0 CUDA_FIRST_LIB)
+get_filename_component(GKO_CUDA_LIBDIR "${CUDA_FIRST_LIB}" DIRECTORY)
 
 ginkgo_default_includes(ginkgo_cuda)
-ginkgo_install_library(ginkgo_cuda cuda)
+ginkgo_install_library(ginkgo_cuda cuda "${GKO_CUDA_LIBDIR}")
 
 if (GINKGO_CHECK_CIRCULAR_DEPS)
     ginkgo_check_headers(ginkgo_cuda)

hip/CMakeLists.txt

@@ -270,15 +270,21 @@ elseif(GINKGO_HIP_PLATFORM MATCHES "nvcc")
     target_link_libraries(ginkgo_hip PUBLIC ${CUDA_LIBRARIES})
 endif()
 
-set(RPATH_ADD_HIP_PATH "")
-if (GINKGO_BUILD_HIP AND GINKGO_HIP_PLATFORM MATCHES "hcc")
-    set(RPATH_ADD_HIP_PATH "${HIP_PATH}/lib")
+# Try to find everything in /opt/rocm/lib first.
+set(GKO_HIP_RPATH "${ROCM_PATH}/lib" )
+if (GINKGO_HIP_PLATFORM MATCHES "nvcc")
+    list(GET CUDA_LIBRARIES 0 CUDA_FIRST_LIB)
+    get_filename_component(GKO_CUDA_LIBDIR "${CUDA_FIRST_LIB}" DIRECTORY)
+    list(APPEND GKO_HIP_RPATH "${GKO_CUDA_LIBDIR}")
+else()
+    list(APPEND GKO_HIP_RPATH "${HIP_PATH}/lib")
 endif()
+list(APPEND GKO_HIP_RPATH "${HIPBLAS_PATH}/lib" "${HIPRAND_PATH}/lib"
+    "${HIPSPARSE_PATH}/lib" "${ROCRAND_PATH}/lib")
 
 ginkgo_compile_features(ginkgo_hip)
 ginkgo_default_includes(ginkgo_hip)
-ginkgo_install_library(ginkgo_hip hip "${RPATH_ADD_HIP_PATH}" "${HIPBLAS_PATH}/lib"
-    "${HIPRAND_PATH}/lib" "${HIPSPARSE_PATH}/lib" "${ROCRAND_PATH}/lib")
+ginkgo_install_library(ginkgo_hip hip "${GKO_HIP_RPATH}")
 
 if (GINKGO_CHECK_CIRCULAR_DEPS)
     ginkgo_check_headers(ginkgo_hip)