Removing RUNPATHS from installed DSOs

[gkaf89]

The build scripts add a RUNPATH entry in the installed library. It may be a good idea to leave the RUNPATH empty so that the linking fails in incorrectly initialized environments, instead of linking with incorrect libraries.

The default behavior of CMake is to remove RUNPATHs during DSO installation with the expectation that the executable linked with the DSO is usually responsible for providing the RPATH/RUNPATH. A hard coded path can cause issues if the library is used in a different environment from the one it was built.

For instance, I am building the MUMPS library in an HPC system, with a few development modules loaded. With the modules loaded, the shared object dependencies are resolved correctly:

$ ldd mump/lib64/libzmumps.so
   linux-vdso.so.1 (0x00007ffc71ea9000)
   libmumps_common.so.5.6.1.0 => /home/users/gkafanas/opt/mumps/lib64/libmumps_common.so.5.6.1.0 (0x00007ff1eca4e000)
   libesmumps.so => /home/users/gkafanas/opt/scotch/lib/libesmumps.so (0x00007ff1eca44000)
   libscotch.so.7.0.3 => /home/users/gkafanas/opt/scotch/lib/libscotch.so.7.0.3 (0x00007ff1ec9a4000)
   libscotcherr.so => /home/users/gkafanas/opt/scotch/lib/libscotcherr.so (0x00007ff1ec9a1000)
   libmetis.so => /home/users/gkafanas/opt/metis/lib/libmetis.so (0x00007ff1ec949000)
   libpord.so => /home/users/gkafanas/opt/mumps/lib64/libpord.so (0x00007ff1ec930000)
   libscalapack.so.2.2 => /home/users/gkafanas/opt/scalapack/lib/libscalapack.so.2.2 (0x00007ff1ec05c000)
   libopenblas.so.0 => /opt/apps/resif/aion/2020b/epyc/software/OpenBLAS/0.3.12-GCC-10.2.0/lib/libopenblas.so.0 (0x00007ff1eb249000)
   libpthread.so.0 => /usr/lib64/libpthread.so.0 (0x00007ff1eb029000)
   libmpi_usempif08.so.40 => /opt/apps/resif/aion/2020b/epyc/software/OpenMPI/4.0.5-GCC-10.2.0/lib/libmpi_usempif08.so.40 (0x00007ff1ec8f1000)
   libmpi_usempi_ignore_tkr.so.40 => /opt/apps/resif/aion/2020b/epyc/software/OpenMPI/4.0.5-GCC-10.2.0/lib/libmpi_usempi_ignore_tkr.so.40 (0x00007ff1ec8e3000)
   libmpi_mpifh.so.40 => /opt/apps/resif/aion/2020b/epyc/software/OpenMPI/4.0.5-GCC-10.2.0/lib/libmpi_mpifh.so.40 (0x00007ff1eafb0000)
   libmpi.so.40 => /opt/apps/resif/aion/2020b/epyc/software/OpenMPI/4.0.5-GCC-10.2.0/lib/libmpi.so.40 (0x00007ff1ead1f000)
   libgomp.so.1 => /opt/apps/resif/aion/2020b/epyc/software/GCCcore/10.2.0/lib64/libgomp.so.1 (0x00007ff1eacdf000)
   libgfortran.so.5 => /opt/apps/resif/aion/2020b/epyc/software/GCCcore/10.2.0/lib64/libgfortran.so.5 (0x00007ff1eaa26000)
   libm.so.6 => /usr/lib64/libm.so.6 (0x00007ff1ea6a4000)
   libgcc_s.so.1 => /opt/apps/resif/aion/2020b/epyc/software/GCCcore/10.2.0/lib64/libgcc_s.so.1 (0x00007ff1ec8c8000)
   libquadmath.so.0 => /opt/apps/resif/aion/2020b/epyc/software/GCCcore/10.2.0/lib64/libquadmath.so.0 (0x00007ff1ea65c000)
   libc.so.6 => /usr/lib64/libc.so.6 (0x00007ff1ea296000)
   libz.so.1 => /opt/apps/resif/aion/2020b/epyc/software/zlib/1.2.11-GCCcore-10.2.0/lib/libz.so.1 (0x00007ff1ea27d000)
   libbz2.so.1.0 => /opt/apps/resif/aion/2020b/epyc/software/bzip2/1.0.8-GCCcore-10.2.0/lib/libbz2.so.1.0 (0x00007ff1ea269000)
   liblzma.so.5 => /opt/apps/resif/aion/2020b/epyc/software/XZ/5.2.5-GCCcore-10.2.0/lib/liblzma.so.5 (0x00007ff1ea241000)
   /lib64/ld-linux-x86-64.so.2 (0x00007ff1ec899000)
   libdl.so.2 => /lib64/libdl.so.2 (0x00007ff1ea034000)
   libmvec.so.1 => /lib64/libmvec.so.1 (0x00007ff1e9e09000)
   liblustreapi.so.1 => /lib64/liblustreapi.so.1 (0x00007ff1e9bd5000)
   libopen-rte.so.40 => /opt/apps/resif/aion/2020b/epyc/software/OpenMPI/4.0.5-GCC-10.2.0/lib/libopen-rte.so.40 (0x00007ff1e9aae000)
   libopen-pal.so.40 => /opt/apps/resif/aion/2020b/epyc/software/OpenMPI/4.0.5-GCC-10.2.0/lib/libopen-pal.so.40 (0x00007ff1e9854000)
   libucp.so.0 => /opt/apps/resif/aion/2020b/epyc/software/UCX/1.9.0-GCCcore-10.2.0/lib/libucp.so.0 (0x00007ff1e97f1000)
   libuct.so.0 => /opt/apps/resif/aion/2020b/epyc/software/UCX/1.9.0-GCCcore-10.2.0/lib/libuct.so.0 (0x00007ff1e97bf000)
   libucs.so.0 => /opt/apps/resif/aion/2020b/epyc/software/UCX/1.9.0-GCCcore-10.2.0/lib/libucs.so.0 (0x00007ff1e9778000)
   libnuma.so.1 => /opt/apps/resif/aion/2020b/epyc/software/numactl/2.0.13-GCCcore-10.2.0/lib/libnuma.so.1 (0x00007ff1e976b000)
   libucm.so.0 => /opt/apps/resif/aion/2020b/epyc/software/UCX/1.9.0-GCCcore-10.2.0/lib/libucm.so.0 (0x00007ff1e9753000)
   libbfd-2.35.2.so => /opt/apps/resif/aion/2020b/epyc/software/binutils/2.35-GCCcore-10.2.0/lib/libbfd-2.35.2.so (0x00007ff1e961a000)
   libpmi2.so.0 => /lib64/libpmi2.so.0 (0x00007ff1e9402000)
   libpmi.so.0 => /lib64/libpmi.so.0 (0x00007ff1e91fc000)
   libnl-3.so.200 => /lib64/libnl-3.so.200 (0x00007ff1e8fd7000)
   libnl-route-3.so.200 => /lib64/libnl-route-3.so.200 (0x00007ff1e8d45000)
   librt.so.1 => /lib64/librt.so.1 (0x00007ff1e8b3d000)
   libutil.so.1 => /lib64/libutil.so.1 (0x00007ff1e8939000)
   libhwloc.so.15 => /opt/apps/resif/aion/2020b/epyc/software/hwloc/2.2.0-GCCcore-10.2.0/lib/libhwloc.so.15 (0x00007ff1e88e7000)
   libpciaccess.so.0 => /opt/apps/resif/aion/2020b/epyc/software/libpciaccess/0.16-GCCcore-10.2.0/lib/libpciaccess.so.0 (0x00007ff1e88dc000)
   libxml2.so.2 => /opt/apps/resif/aion/2020b/epyc/software/libxml2/2.9.10-GCCcore-10.2.0/lib/libxml2.so.2 (0x00007ff1e8764000)
   libevent_core-2.1.so.6 => /lib64/libevent_core-2.1.so.6 (0x00007ff1e852b000)
   libevent_pthreads-2.1.so.6 => /lib64/libevent_pthreads-2.1.so.6 (0x00007ff1e8328000)
   libkeyutils.so.1 => /lib64/libkeyutils.so.1 (0x00007ff1e8124000)
   libresolv.so.2 => /lib64/libresolv.so.2 (0x00007ff1e7f0a000)
   libslurm_pmi.so => /usr/lib64/slurm/libslurm_pmi.so (0x00007ff1e7b2f000)
   libcrypto.so.1.1 => /lib64/libcrypto.so.1.1 (0x00007ff1e7646000)

However, after removing the development modules, there are issues in resolving the shared object dependencies:

$ ldd mump/lib64/libzmumps.so
mumps/lib64/libzmumps.so: /usr/lib64/libgfortran.so.5: version `GFORTRAN_10' not found (required by mumps/lib64/libzmumps.so)
   linux-vdso.so.1 (0x00007ffd5854a000)
   libmumps_common.so.5.6.1.0 => /home/users/gkafanas/opt/mumps/lib64/libmumps_common.so.5.6.1.0 (0x00007f24342ac000)
   libesmumps.so => /home/users/gkafanas/opt/scotch/lib/libesmumps.so (0x00007f24342a4000)
   libscotch.so.7.0.3 => /home/users/gkafanas/opt/scotch/lib/libscotch.so.7.0.3 (0x00007f2434204000)
   libscotcherr.so => /home/users/gkafanas/opt/scotch/lib/libscotcherr.so (0x00007f2434201000)
   libmetis.so => /home/users/gkafanas/opt/metis/lib/libmetis.so (0x00007f24341a7000)
   libpord.so => /home/users/gkafanas/opt/mumps/lib64/libpord.so (0x00007f243418e000)
   libscalapack.so.2.2 => /home/users/gkafanas/opt/scalapack/lib/libscalapack.so.2.2 (0x00007f24338ba000)
   libopenblas.so.0 => /usr/lib64/libopenblas.so.0 (0x00007f243146e000)
   libpthread.so.0 => /usr/lib64/libpthread.so.0 (0x00007f243124e000)
   libmpi_usempif08.so.40 => /opt/apps/resif/aion/2020b/epyc/software/OpenMPI/4.0.5-GCC-10.2.0/lib/libmpi_usempif08.so.40 (0x00007f2434151000)
   libmpi_usempi_ignore_tkr.so.40 => /opt/apps/resif/aion/2020b/epyc/software/OpenMPI/4.0.5-GCC-10.2.0/lib/libmpi_usempi_ignore_tkr.so.40 (0x00007f2434141000)
   libmpi_mpifh.so.40 => /opt/apps/resif/aion/2020b/epyc/software/OpenMPI/4.0.5-GCC-10.2.0/lib/libmpi_mpifh.so.40 (0x00007f24311d5000)
   libmpi.so.40 => /opt/apps/resif/aion/2020b/epyc/software/OpenMPI/4.0.5-GCC-10.2.0/lib/libmpi.so.40 (0x00007f2430f44000)
   libgomp.so.1 => /usr/lib64/libgomp.so.1 (0x00007f2430d0c000)
   libgfortran.so.5 => /usr/lib64/libgfortran.so.5 (0x00007f243088d000)
   libm.so.6 => /usr/lib64/libm.so.6 (0x00007f243050b000)
   libgcc_s.so.1 => /usr/lib64/libgcc_s.so.1 (0x00007f24302f3000)
   libquadmath.so.0 => /usr/lib64/libquadmath.so.0 (0x00007f24300b2000)
   libc.so.6 => /usr/lib64/libc.so.6 (0x00007f242fcec000)
   libz.so.1 => /lib64/libz.so.1 (0x00007f242fad4000)
   libbz2.so.1.0 => not found
   liblzma.so.5 => /lib64/liblzma.so.5 (0x00007f242f8ad000)
   /lib64/ld-linux-x86-64.so.2 (0x00007f24340f7000)
   libbz2.so.1.0 => not found
   libdl.so.2 => /lib64/libdl.so.2 (0x00007f242f6a9000)
   liblustreapi.so.1 => /lib64/liblustreapi.so.1 (0x00007f242f477000)
   libopen-rte.so.40 => /opt/apps/resif/aion/2020b/epyc/software/OpenMPI/4.0.5-GCC-10.2.0/lib/libopen-rte.so.40 (0x00007f242f350000)
   libopen-pal.so.40 => /opt/apps/resif/aion/2020b/epyc/software/OpenMPI/4.0.5-GCC-10.2.0/lib/libopen-pal.so.40 (0x00007f242f0f6000)
   libucp.so.0 => /opt/apps/resif/aion/2020b/epyc/software/UCX/1.9.0-GCCcore-10.2.0/lib/libucp.so.0 (0x00007f242f093000)
   libuct.so.0 => /opt/apps/resif/aion/2020b/epyc/software/UCX/1.9.0-GCCcore-10.2.0/lib/libuct.so.0 (0x00007f242f061000)
   libucs.so.0 => /opt/apps/resif/aion/2020b/epyc/software/UCX/1.9.0-GCCcore-10.2.0/lib/libucs.so.0 (0x00007f242f01c000)
   libnuma.so.1 => /opt/apps/resif/aion/2020b/epyc/software/numactl/2.0.13-GCCcore-10.2.0/lib/libnuma.so.1 (0x00007f242f00f000)
   libucm.so.0 => /opt/apps/resif/aion/2020b/epyc/software/UCX/1.9.0-GCCcore-10.2.0/lib/libucm.so.0 (0x00007f242eff7000)
   libbfd-2.35.2.so => /opt/apps/resif/aion/2020b/epyc/software/binutils/2.35-GCCcore-10.2.0/lib/libbfd-2.35.2.so (0x00007f242eebe000)
   libpmi2.so.0 => /lib64/libpmi2.so.0 (0x00007f242eca6000)
   libpmi.so.0 => /lib64/libpmi.so.0 (0x00007f242eaa0000)
   libnl-3.so.200 => /lib64/libnl-3.so.200 (0x00007f242e87d000)
   libnl-route-3.so.200 => /lib64/libnl-route-3.so.200 (0x00007f242e5eb000)
   librt.so.1 => /lib64/librt.so.1 (0x00007f242e3e3000)
   libutil.so.1 => /lib64/libutil.so.1 (0x00007f242e1df000)
   libhwloc.so.15 => /opt/apps/resif/aion/2020b/epyc/software/hwloc/2.2.0-GCCcore-10.2.0/lib/libhwloc.so.15 (0x00007f242e18d000)
   libpciaccess.so.0 => /opt/apps/resif/aion/2020b/epyc/software/libpciaccess/0.16-GCCcore-10.2.0/lib/libpciaccess.so.0 (0x00007f242e182000)
   libxml2.so.2 => /opt/apps/resif/aion/2020b/epyc/software/libxml2/2.9.10-GCCcore-10.2.0/lib/libxml2.so.2 (0x00007f242e00c000)
   libevent_core-2.1.so.6 => /lib64/libevent_core-2.1.so.6 (0x00007f242ddd3000)
   libevent_pthreads-2.1.so.6 => /lib64/libevent_pthreads-2.1.so.6 (0x00007f242dbd0000)
   libkeyutils.so.1 => /lib64/libkeyutils.so.1 (0x00007f242d9cc000)
   libresolv.so.2 => /lib64/libresolv.so.2 (0x00007f242d7b4000)
   libslurm_pmi.so => /usr/lib64/slurm/libslurm_pmi.so (0x00007f242d3d9000)
   libcrypto.so.1.1 => /lib64/libcrypto.so.1.1 (0x00007f242cef0000)

The dependency on libbz2 (which is introduced by scotch) can no longer be resolved, and more importantly, the loaded FORTRAN library does not support the required version, in the 1st line of the ldd output.

The MUMPS library is provided through a module that sets up the environment, so that users do not have to load whole development modules. Executables that use MUMPS may also have a few paths hard-coded in their RUNPATH.

I believe it would be nicer if the library failed to detect the required DSOs completely when the environment is not set up properly, instead of detecting incorrect DSOs and possibly crashing at runtime. Would it make sense to maintain the default CMake behavior, removing the RPATH from the installed library, and leave it as an option for the user to choose otherwise?

The relevant settings are in file cmake/compilers.cmake:

set(CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_FULL_LIBDIR})
set(CMAKE_INSTALL_RPATH_USE_LINK_PATH true)

[gkaf89]

I am moving this issue to the discussion section.