add scripts to set up minimal EESSI environment and configure EasyBuild, use init/* scripts in install script, inject RPATH override directory via prepare hook to allow for overriding MPI library

EESSI-pilot-install-software.sh

@@ -17,7 +17,7 @@ function echo_yellow() {
     echo -e "\e[33m$1\e[0m"
 }
 
-function error() {
+function fatal_error() {
     echo_red "ERROR: $1" >&2
     exit 1
 }
@@ -30,7 +30,7 @@ function check_exit_code {
     if [[ $ec -eq 0 ]]; then
         echo_green "${ok_msg}"
     else
-        error "${fail_msg}"
+        fatal_error "${fail_msg}"
     fi
 }
 
@@ -44,20 +44,21 @@ fi
 TMPDIR=$(mktemp -d)
 
 echo ">> Setting up environment..."
-export CVMFS_REPO="/cvmfs/pilot.eessi-hpc.org"
-export EESSI_PILOT_VERSION="2021.06"
 
-if [[ $(uname -s) == 'Linux' ]]; then
-    export EESSI_OS_TYPE='linux'
+source $TOPDIR/init/minimal_eessi_env
+
+if [ -d $EESSI_CVMFS_REPO ]; then
+    echo_green "$EESSI_CVMFS_REPO available, OK!"
 else
-    export EESSI_OS_TYPE='macos'
+    fatal_error "$EESSI_CVMFS_REPO is not available!"
 fi
 
-# aarch64 (Arm 64-bit), ppc64le (POWER 64-bit), x86_64 (x86 64-bit)
-export EESSI_CPU_FAMILY=$(uname -m)
-
-export EESSI_PREFIX=${CVMFS_REPO}/${EESSI_PILOT_VERSION}
-export EPREFIX=${EESSI_PREFIX}/compat/${EESSI_OS_TYPE}/${EESSI_CPU_FAMILY}
+# make sure we're in Prefix environment by checking $SHELL
+if [[ ${SHELL} = ${EPREFIX}/bin/bash ]]; then
+    echo_green ">> It looks like we're in a Gentoo Prefix environment, good!"
+else
+    fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!"
+fi
 
 # avoid that pyc files for EasyBuild are stored in EasyBuild installation directory
 export PYTHONPYCACHEPREFIX=$TMPDIR/pycache
@@ -72,12 +73,20 @@ if [[ "$1" == "--generic" || "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then
     EB='eb --optarch=GENERIC'
 fi
 
+echo ">> Determining software subdirectory to use for current build host..."
+export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
 
-# make sure we're in Prefix environment by which path to 'bash' command
-if [[ $(which bash) = ${EPREFIX}/bin/bash ]]; then
-    echo_green ">> It looks like we're in a Gentoo Prefix environment, good!"
+# Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE)
+# $EESSI_SILENT - don't print any messages
+# $EESSI_BASIC_ENV - give a basic set of environment variables
+EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables
+
+if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then
+    fatal_error "Failed to determine software subdirectory?!"
+elif [[ "${EESSI_SOFTWARE_SUBDIR}" != "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then
+    fatal_error "Values for EESSI_SOFTWARE_SUBDIR_OVERRIDE (${EESSI_SOFTWARE_SUBDIR_OVERRIDE}) and EESSI_SOFTWARE_SUBDIR (${EESSI_SOFTWARE_SUBDIR}) differ!"
 else
-    error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!"
+    echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!"
 fi
 
 echo ">> Initializing Lmod..."
@@ -87,54 +96,11 @@ ml --version &> $ml_version_out
 if [[ $? -eq 0 ]]; then
     echo_green ">> Found Lmod ${LMOD_VERSION}"
 else
-    error "Failed to initialize Lmod?! (see output in ${ml_version_out}"
-fi
-
-echo ">> Determining software subdirectory to use for current build host..."
-export EESSI_SOFTWARE_SUBDIR=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
-if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then
-    error "Failed to determine software subdirectory?!"
-else
-    echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!"
+    fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}"
 fi
 
 echo ">> Configuring EasyBuild..."
-export EASYBUILD_PREFIX=${WORKDIR}/easybuild
-export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR}
-export EASYBUILD_SOURCEPATH=${WORKDIR}/easybuild/sources:${EESSI_SOURCEPATH}
-
-# just ignore OS dependencies for now, see https://github.com/easybuilders/easybuild-framework/issues/3430
-export EASYBUILD_IGNORE_OSDEPS=1
-
-export EASYBUILD_SYSROOT=${EPREFIX}
-
-export EASYBUILD_DEBUG=1
-export EASYBUILD_TRACE=1
-export EASYBUILD_ZIP_LOGS=bzip2
-
-export EASYBUILD_RPATH=1
-export EASYBUILD_FILTER_ENV_VARS=LD_LIBRARY_PATH
-
-export EASYBUILD_HOOKS=$TOPDIR/eb_hooks.py
-# make sure hooks are available, so we can produce a clear error message
-if [ ! -f $EASYBUILD_HOOKS ]; then
-    error "$EASYBUILD_HOOKS does not exist!"
-fi
-
-# note: filtering Bison may break some installations, like Qt5 (see https://github.com/EESSI/software-layer/issues/49)
-# filtering pkg-config breaks R-bundle-Bioconductor installation (see also https://github.com/easybuilders/easybuild-easyconfigs/pull/11104)
-# problems occur when filtering pkg-config with gnuplot too (picks up Lua 5.1 from $EPREFIX rather than from Lua 5.3 dependency)
-DEPS_TO_FILTER=Autoconf,Automake,Autotools,binutils,bzip2,cURL,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,Lua,M4,makeinfo,ncurses,util-linux,XZ,zlib
-# For aarch64 we need to also filter out Yasm.
-# See https://github.com/easybuilders/easybuild-easyconfigs/issues/11190
-if [[ "$EESSI_CPU_FAMILY" == "aarch64" ]]; then
-    DEPS_TO_FILTER="${DEPS_TO_FILTER},Yasm"
-fi
-
-export EASYBUILD_FILTER_DEPS=$DEPS_TO_FILTER
-
-
-export EASYBUILD_MODULE_EXTENSIONS=1
+source configure_easybuild
 
 echo ">> Setting up \$MODULEPATH..."
 # make sure no modules are loaded
@@ -143,7 +109,7 @@ module --force purge
 module unuse $MODULEPATH
 module use $EASYBUILD_INSTALLPATH/modules/all
 if [[ -z ${MODULEPATH} ]]; then
-    error "Failed to set up \$MODULEPATH?!"
+    fatal_error "Failed to set up \$MODULEPATH?!"
 else
     echo_green ">> MODULEPATH set up: ${MODULEPATH}"
 fi
@@ -171,7 +137,7 @@ else
     if [[ $? -eq 0 ]]; then
         echo_green ">> EasyBuild module installed!"
     else
-        error "EasyBuild module failed to install?! (output of 'pip install' in ${pip_install_out}, output of 'eb' in ${eb_install_out}, output of 'ml av easybuild' in ${ml_av_easybuild_out})"
+        fatal_error "EasyBuild module failed to install?! (output of 'pip install' in ${pip_install_out}, output of 'eb' in ${eb_install_out}, output of 'ml av easybuild' in ${ml_av_easybuild_out})"
     fi
 fi
 
@@ -187,12 +153,12 @@ if [[ $? -eq 0 ]]; then
         echo_green "Found EasyBuild version ${REQ_EB_VERSION}, looking good!"
     else
         $EB --version
-        error "Expected to find EasyBuild version ${REQ_EB_VERSION}, giving up here..."
+        fatal_error "Expected to find EasyBuild version ${REQ_EB_VERSION}, giving up here..."
     fi
     $EB --show-config
 else
     cat ${eb_show_system_info_out}
-    error "EasyBuild not working?!"
+    fatal_error "EasyBuild not working?!"
 fi
 
 echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALLPATH}..."

configure_easybuild

@@ -0,0 +1,35 @@
+export EASYBUILD_PREFIX=${WORKDIR}/easybuild
+export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR}
+export EASYBUILD_SOURCEPATH=${WORKDIR}/easybuild/sources:${EESSI_SOURCEPATH}
+
+# just ignore OS dependencies for now, see https://github.com/easybuilders/easybuild-framework/issues/3430
+export EASYBUILD_IGNORE_OSDEPS=1
+
+export EASYBUILD_SYSROOT=${EPREFIX}
+
+export EASYBUILD_DEBUG=1
+export EASYBUILD_TRACE=1
+export EASYBUILD_ZIP_LOGS=bzip2
+
+export EASYBUILD_RPATH=1
+export EASYBUILD_FILTER_ENV_VARS=LD_LIBRARY_PATH
+
+export EASYBUILD_HOOKS=$(realpath eb_hooks.py)
+# make sure hooks are available, so we can produce a clear error message
+if [ ! -f $EASYBUILD_HOOKS ]; then
+    fatal_error "$EASYBUILD_HOOKS does not exist!"
+fi
+
+# note: filtering Bison may break some installations, like Qt5 (see https://github.com/EESSI/software-layer/issues/49)
+# filtering pkg-config breaks R-bundle-Bioconductor installation (see also https://github.com/easybuilders/easybuild-easyconfigs/pull/11104)
+# problems occur when filtering pkg-config with gnuplot too (picks up Lua 5.1 from $EPREFIX rather than from Lua 5.3 dependency)
+DEPS_TO_FILTER=Autoconf,Automake,Autotools,binutils,bzip2,cURL,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,Lua,M4,makeinfo,ncurses,util-linux,XZ,zlib
+# For aarch64 we need to also filter out Yasm.
+# See https://github.com/easybuilders/easybuild-easyconfigs/issues/11190
+if [[ "$EESSI_CPU_FAMILY" == "aarch64" ]]; then
+    DEPS_TO_FILTER="${DEPS_TO_FILTER},Yasm"
+fi
+
+export EASYBUILD_FILTER_DEPS=$DEPS_TO_FILTER
+
+export EASYBUILD_MODULE_EXTENSIONS=1

eb_hooks.py

@@ -3,21 +3,93 @@
 import os
 
 from easybuild.tools.build_log import EasyBuildError, print_msg
+from easybuild.tools.config import build_option, update_build_option
 from easybuild.tools.systemtools import POWER, get_cpu_architecture
 
+EESSI_RPATH_OVERRIDE_ATTR = 'orig_rpath_override_dirs'
+
+
+def get_eessi_envvar(eessi_envvar):
+    """Get an EESSI environment variable from the environment"""
+
+    eessi_envvar_value = os.getenv(eessi_envvar)
+    if eessi_envvar_value is None:
+        raise EasyBuildError("$%s is not defined!", eessi_envvar)
+
+    return eessi_envvar_value
+
+
+def get_rpath_override_dirs(software_name):
+    # determine path to installations in software layer via $EESSI_SOFTWARE_PATH
+    eessi_software_path = get_eessi_envvar('EESSI_SOFTWARE_PATH')
+    eessi_pilot_version = get_eessi_envvar('EESSI_PILOT_VERSION')
+
+    # construct the rpath override directory stub
+    rpath_injection_stub = os.path.join(
+        # Make sure we are looking inside the `host_injections` directory
+        eessi_software_path.replace(eessi_pilot_version, os.path.join('host_injections', eessi_pilot_version), 1),
+        # Add the subdirectory for the specific software
+        'rpath_overrides',
+        software_name,
+        # We can't know the version, but this allows the use of a symlink
+        # to facilitate version upgrades without removing files
+        'system',
+    )
+
+    # Allow for libraries in lib or lib64
+    rpath_injection_dirs = [os.path.join(rpath_injection_stub, x) for x in ('lib', 'lib64')]
+
+    return rpath_injection_dirs
+
 
 def parse_hook(ec, *args, **kwargs):
     """Main parse hook: trigger custom functions based on software name."""
 
     # determine path to Prefix installation in compat layer via $EPREFIX
-    eprefix = os.getenv('EPREFIX')
-    if eprefix is None:
-        raise EasyBuildError("$EPREFIX is not defined!")
+    eprefix = get_eessi_envvar('EPREFIX')
 
     if ec.name in PARSE_HOOKS:
         PARSE_HOOKS[ec.name](ec, eprefix)
 
 
+def pre_prepare_hook(self, *args, **kwargs):
+    """Main pre-prepare hook: trigger custom functions."""
+
+    # Check if we have an MPI family in the toolchain (returns None if there is not)
+    mpi_family = self.toolchain.mpi_family()
+
+    # Inject an RPATH override for MPI (if needed)
+    if mpi_family:
+        # Get list of override directories
+        mpi_rpath_override_dirs = get_rpath_override_dirs(mpi_family)
+
+        # update the relevant option (but keep the original value so we can reset it later)
+        if hasattr(self, EESSI_RPATH_OVERRIDE_ATTR):
+            raise EasyBuildError("'self' already has attribute %s! Can't use pre_prepare hook.",
+                                 EESSI_RPATH_OVERRIDE_ATTR)
+
+        setattr(self, EESSI_RPATH_OVERRIDE_ATTR, build_option('rpath_override_dirs'))
+        if getattr(self, EESSI_RPATH_OVERRIDE_ATTR):
+            # self.EESSI_RPATH_OVERRIDE_ATTR is (already) a colon separated string, let's make it a list
+            orig_rpath_override_dirs = [getattr(self, EESSI_RPATH_OVERRIDE_ATTR)]
+            rpath_override_dirs = ':'.join(orig_rpath_override_dirs + mpi_rpath_override_dirs)
+        else:
+            rpath_override_dirs = ':'.join(mpi_rpath_override_dirs)
+        update_build_option('rpath_override_dirs', rpath_override_dirs)
+        print_msg("Updated rpath_override_dirs (to allow overriding MPI family %s): %s",
+                  mpi_family, rpath_override_dirs)
+
+
+def post_prepare_hook(self, *args, **kwargs):
+    """Main post-prepare hook: trigger custom functions."""
+
+    if hasattr(self, EESSI_RPATH_OVERRIDE_ATTR):
+        # Reset the value of 'rpath_override_dirs' now that we are finished with it
+        update_build_option('rpath_override_dirs', getattr(self, EESSI_RPATH_OVERRIDE_ATTR))
+        print_msg("Resetting rpath_override_dirs to original value: %s", getattr(self, EESSI_RPATH_OVERRIDE_ATTR))
+        delattr(self, EESSI_RPATH_OVERRIDE_ATTR)
+
+
 def cgal_toolchainopts_precise(ec, eprefix):
     """Enable 'precise' rather than 'strict' toolchain option for CGAL on POWER."""
     if ec.name == 'CGAL':

init/Magic_Castle/bash

@@ -15,6 +15,10 @@ export MODULEPATH=$EESSI_MODULEPATH
 # Extensions are too many, let's not print them by default (requires Lmod 8.4.12)
 export LMOD_AVAIL_EXTENSIONS=no
 
+# add location of commands provided by compat layer to $PATH;
+# see https://github.com/EESSI/software-layer/issues/52
+export PATH=$EPREFIX/usr/bin:$EPREFIX/bin:$PATH
+
 # init Lmod
 source $EESSI_EPREFIX/usr/share/Lmod/init/bash
 

init/bash

@@ -15,6 +15,10 @@ if [ $? -eq 0 ]; then
 
     export PS1="[EESSI pilot $EESSI_PILOT_VERSION] $ "
 
+    # add location of commands provided by compat layer to $PATH;
+    # see https://github.com/EESSI/software-layer/issues/52
+    export PATH=$EPREFIX/usr/bin:$EPREFIX/bin:$PATH
+
     # init Lmod
     echo "Initializing Lmod..." >> $output
     source $EESSI_EPREFIX/usr/share/Lmod/init/bash

init/eessi_environment_variables

@@ -1,3 +1,7 @@
+# this script is *sourced*, not executed, so can't rely on $0 to determine path to self
+# $BASH_SOURCE points to correct path, see also http://mywiki.wooledge.org/BashFAQ/028
+EESSI_INIT_DIR_PATH=$(dirname $(realpath $BASH_SOURCE))
+
 # Allow for a silent mode
 if [[ -v EESSI_SILENT ]]; then
   # EESSI_SILENT set
@@ -11,40 +15,26 @@ function error() {
     false
 }
 
-export EESSI_PILOT_VERSION="2021.03"
-export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/$EESSI_PILOT_VERSION
+# set up minimal environment: $EESSI_PREFIX, $EESSI_PILOT_VERSION, $EESSI_OS_TYPE, $EESSI_CPU_FAMILY, $EPREFIX
+source $EESSI_INIT_DIR_PATH/minimal_eessi_env
 
 if [ -d $EESSI_PREFIX ]; then
   echo "Found EESSI pilot repo @ $EESSI_PREFIX!" >> $output
 
-  if [[ $(uname -s) == 'Linux' ]]; then
-      export EESSI_OS_TYPE='linux'
-  else
-      export EESSI_OS_TYPE='macos'
-  fi
-
-  # aarch64 (Arm 64-bit), ppc64le (POWER 64-bit), x86_64 (x86 64-bit)
-  export EESSI_CPU_FAMILY=$(uname -m)
-
-  # Set EPREFIX since that is basically a standard in Gentoo Prefix
-  export EPREFIX=$EESSI_PREFIX/compat/$EESSI_OS_TYPE/$EESSI_CPU_FAMILY
-
-  # add location of commands provided by compat layer to $PATH;
-  # see https://github.com/EESSI/software-layer/issues/52
-  export PATH=$EPREFIX/usr/bin:$EPREFIX/bin:$PATH
-
   export EESSI_EPREFIX=$EPREFIX
   if [ -d $EESSI_EPREFIX ]; then
 
     # determine subdirectory in software layer
     # note: eessi_software_subdir_for_host.py will pick up value from $EESSI_SOFTWARE_SUBDIR_OVERRIDE if it's defined!
     export EESSI_EPREFIX_PYTHON=$EESSI_EPREFIX/usr/bin/python3
-    export EESSI_SOFTWARE_SUBDIR=$($EESSI_EPREFIX_PYTHON ${EESSI_PREFIX}/init/eessi_software_subdir_for_host.py $EESSI_PREFIX)
+    export EESSI_SOFTWARE_SUBDIR=$($EESSI_EPREFIX_PYTHON ${EESSI_INIT_DIR_PATH}/eessi_software_subdir_for_host.py $EESSI_PREFIX)
     if [ ! -z $EESSI_SOFTWARE_SUBDIR ]; then
 
         echo "Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory." >> $output
         export EESSI_SOFTWARE_PATH=$EESSI_PREFIX/software/$EESSI_OS_TYPE/$EESSI_SOFTWARE_SUBDIR
-        if [ -d $EESSI_SOFTWARE_PATH ]; then
+        if [ ! -z $EESSI_BASIC_ENV ]; then
+          echo "Only setting up basic environment, so we're done" >> $output
+        elif [ -d $EESSI_SOFTWARE_PATH ]; then
           # Allow for the use of a custom MNS
           if [ -z ${EESSI_CUSTOM_MODULEPATH+x} ]; then
             # EESSI_CUSTOM_MODULEPATH not set so we use our defaults

init/eessi_software_subdir_for_host.py

@@ -49,7 +49,7 @@ def find_best_target(eessi_prefix):
         'Linux': 'linux',
     }[os_type]
 
-    eessi_software_layer_path = os.path.join(eessi_prefix, 'software', 'linux')
+    eessi_software_layer_path = os.path.join(eessi_prefix, 'software', eessi_os_type)
     if not os.path.exists(eessi_software_layer_path):
         error('Specified prefix "%s" does not exist!' % eessi_software_layer_path)