relocatable: switch from run-time relocation to install-time relocation

Our current relocation works by invoking the dynamic linker with the
executable as an argument. This confuses gdb since the kernel records
the dynamic linker as the executable, not the real executable.

Switch to install-time relocation with patchelf: when installing the
executable and libraries, all paths are known, and we can update the
path to the dynamic loader and to the dynamic libraries.

Since patchelf itself is dynamically linked, we have to relocate it
dynamically (with the old method of invoking it via the dynamic linker).
This is okay since it's a one-time operation and since we don't expect
to debug core dumps of patchelf crashes.

We lose the ability to run scylla directly from the uninstalled
tarball, but since the nonroot installer is already moving in the
direction of requiring install.sh, that is not a great loss, and
certainly the ability to debug is more important.

Fixes scylladb#4673.

install.sh

@@ -89,6 +89,29 @@ if [ -n "$pkg" ] && [ "$pkg" != "server" -a "$pkg" != "conf" -a "$pkg" != "kerne
     exit 1
 fi
 
+patchelf() {
+    # patchelf comes from the build system, so it needs the build system's ld.so and
+    # shared libraries. We can't use patchelf on patchelf itself, so invoke it via
+    # ld.so.
+    LD_LIBRARY_PATH="$PWD/libreloc" libreloc/ld.so libexec/patchelf "$@"
+}
+
+adjust_bin() {
+    local bin="$1"
+    # We could add --set-rpath too, but then debugedit (called by rpmbuild) barfs
+    # on the result. So use LD_LIBRARY_PATH in the thunk, below.
+    patchelf \
+	--set-interpreter "$prefix/libreloc/ld.so" \
+	"$root/$prefix/libexec/$bin"
+    cat > "$root/$prefix/bin/$bin" <<EOF
+#!/bin/bash -e
+export GNUTLS_SYSTEM_PRIORITY_FILE="\${GNUTLS_SYSTEM_PRIORITY_FILE-$prefix/libreloc/gnutls.config}"
+export LD_LIBRARY_PATH="$prefix/libreloc"
+exec -a "\$0" "$prefix/libexec/$bin" "\$@"
+EOF
+    chmod +x "$root/$prefix/bin/$bin"
+}
+
 rprefix="$root/$prefix"
 retc="$root/etc"
 rusr="$root/usr"
@@ -145,16 +168,13 @@ if [ -z "$pkg" ] || [ "$pkg" = "server" ]; then
     install -m644 dist/common/systemd/*.timer -Dt "$rusr"/lib/systemd/system
     install -m755 seastar/scripts/seastar-cpu-map.sh -Dt "$rprefix"/scripts
     install -m755 seastar/dpdk/usertools/dpdk-devbind.py -Dt "$rprefix"/scripts
-    install -m755 bin/* -Dt "$rprefix/bin"
+    install -m755 libreloc/* -Dt "$rprefix/libreloc"
     # some files in libexec are symlinks, which "install" dereferences
     # use cp -P for the symlinks instead.
-    install -m755 libexec/*.bin -Dt "$rprefix/libexec"
-    for f in libexec/*; do
-        if [[ "$f" != *.bin ]]; then
-            cp -P "$f" "$rprefix/libexec"
-        fi
+    install -m755 libexec/* -Dt "$rprefix/libexec"
+    for bin in libexec/*; do
+	adjust_bin "${bin#libexec/}"
     done
-    install -m755 libreloc/* -Dt "$rprefix/libreloc"
     ln -srf "$rprefix/bin/scylla" "$rusr/bin/scylla"
     ln -srf "$rprefix/bin/iotune" "$rusr/bin/iotune"
 

scripts/create-relocatable-package.py

@@ -61,6 +61,7 @@ def ldd(executable):
 
 executables = ['build/{}/scylla'.format(args.mode),
                'build/{}/iotune'.format(args.mode),
+               '/usr/bin/patchelf',
                '/usr/bin/lscpu',
                '/usr/bin/gawk',
                '/usr/bin/gzip',
@@ -96,56 +97,9 @@ def ldd(executable):
 pathlib.Path('build/SCYLLA-RELOCATABLE-FILE').touch()
 ar.add('build/SCYLLA-RELOCATABLE-FILE', arcname='SCYLLA-RELOCATABLE-FILE')
 
-# This thunk is a shell script that arranges for the executable to be invoked,
-# under the following conditions:
-#
-#  - the same argument vector is passed to the executable, including argv[0]
-#  - the executable name (/proc/pid/comm, shown in top(1)) is the same
-#  - the dynamic linker is taken from this package rather than the executable's
-#    default (which is hardcoded to point to /lib64/ld-linux-x86_64.so or similar)
-#  - LD_LIBRARY_PATH points to the lib/ directory so shared library dependencies
-#    are satisified from there rather than the system default (e.g. /lib64)
-
-# To do that, the dynamic linker is invoked using a symbolic link named after the
-# executable, not its standard name. We use "bash -a" to set argv[0].
-
-# The full tangled web looks like:
-#
-# foobar/bin/scylla               a shell script invoking everything
-# foobar/libexec/scylla.bin       the real binary
-# foobar/libexec/scylla           a symlink to ../lib/ld.so
-# foobar/libreloc/ld.so                the dynamic linker
-# foobar/libreloc/lib...               all the other libraries
-
-# the transformations (done by the thunk and symlinks) are:
-#
-#    bin/scylla args -> libexec/scylla libexec/scylla.bin args -> lib/ld.so libexec/scylla.bin args
-
-thunk = b'''\
-#!/bin/bash
-
-x="$(readlink -f "$0")"
-b="$(basename "$x")"
-d="$(dirname "$x")/.."
-ldso="$d/libexec/$b"
-realexe="$d/libexec/$b.bin"
-export GNUTLS_SYSTEM_PRIORITY_FILE="${GNUTLS_SYSTEM_PRIORITY_FILE-$d/libreloc/gnutls.config}"
-LD_LIBRARY_PATH="$d/libreloc" exec -a "$0" "$ldso" "$realexe" "$@"
-'''
-
 for exe in executables:
     basename = os.path.basename(exe)
-    ar.add(exe, arcname='libexec/' + basename + '.bin')
-    ti = tarfile.TarInfo(name='bin/' + basename)
-    ti.size = len(thunk)
-    ti.mode = 0o755
-    ti.mtime = os.stat(exe).st_mtime
-    ar.addfile(ti, fileobj=io.BytesIO(thunk))
-    ti = tarfile.TarInfo(name='libexec/' + basename)
-    ti.type = tarfile.SYMTYPE
-    ti.linkname = '../libreloc/ld.so'
-    ti.mtime = os.stat(exe).st_mtime
-    ar.addfile(ti)
+    ar.add(exe, arcname='libexec/' + basename)
 for lib, libfile in libs.items():
     ar.add(libfile, arcname='libreloc/' + lib)
 if have_gnutls: