diff --git a/EESSI-extend-easybuild.eb b/EESSI-extend-easybuild.eb
index 0abb6ce9..eaf59540 100644
--- a/EESSI-extend-easybuild.eb
+++ b/EESSI-extend-easybuild.eb
@@ -212,7 +212,11 @@ easybuild_version = os.getenv("EBVERSIONEASYBUILD") or easybuild_version
 eessi_version = os.getenv("EESSI_VERSION") or "2023.06"
 
 -- Set environment variables that are EasyBuild version specific
-if convertToCanonical(easybuild_version) > convertToCanonical("4") then
+-- Do unload unconditionally, so that even if EB versions were switched in the meantime, this gets unset
+-- This avoids issues where EESSI-extend is first loaded with EB => 5.1 (which set these vars)
+-- but then EB is swapped for a version < 5.1 and then EESSI-extend is unloaded (which would not unset
+-- these vars if we did it conditional on the EB version)
+if convertToCanonical(easybuild_version) >= convertToCanonical("5.1") or mode() == "unload" then
   setenv ("EASYBUILD_STRICT_RPATH_SANITY_CHECK", "1")
   setenv ("EASYBUILD_CUDA_SANITY_CHECK_ERROR_ON_FAILED_CHECKS", "1")
   setenv ("EASYBUILD_FAIL_ON_MOD_FILES_GCCCORE", "1")
diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index 48e03f94..d462cad2 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -247,29 +247,6 @@ if [ ! -f ${_lmod_sitepackage_file} ]; then
     python3 ${TOPDIR}/create_lmodsitepackage.py ${_eessi_software_path}
 fi
 
-# Install full CUDA SDK and cu* libraries in host_injections
-# (This is done *before* configuring EasyBuild as it may rely on an older EB version)
-# Hardcode this for now, see if it works
-# TODO: We should make a nice yaml and loop over all CUDA versions in that yaml to figure out what to install
-# Allow skipping CUDA SDK install in e.g. CI environments
-echo "Going to install full CUDA SDK and cu* libraries under host_injections if necessary"
-temp_install_storage=${TMPDIR}/temp_install_storage
-mkdir -p ${temp_install_storage}
-if [ -z "${skip_cuda_install}" ] || [ ! "${skip_cuda_install}" ]; then
-    ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh \
-        -t ${temp_install_storage} \
-        --accept-cuda-eula \
-        --accept-cudnn-eula
-else
-    echo "Skipping installation of CUDA SDK and cu* libraries in host_injections, since the --skip-cuda-install flag was passed"
-fi
-
-# Install NVIDIA drivers in host_injections (if they exist)
-if nvidia_gpu_available; then
-    echo "Installing NVIDIA drivers for use in prefix shell..."
-    ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh
-fi
-
 echo ">> Configuring EasyBuild..."
 
 # Make sure EESSI-extend is not loaded, and configure location variables for a
@@ -316,6 +293,30 @@ echo "DEBUG: before loading EESSI-extend // EASYBUILD_INSTALLPATH='${EASYBUILD_I
 source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION}
 echo "DEBUG: after loading EESSI-extend //  EASYBUILD_INSTALLPATH='${EASYBUILD_INSTALLPATH}'"
 
+# Install full CUDA SDK and cu* libraries in host_injections
+# (This is done *before* configuring EasyBuild as it may rely on an older EB version)
+# Hardcode this for now, see if it works
+# TODO: We should make a nice yaml and loop over all CUDA versions in that yaml to figure out what to install
+# Allow skipping CUDA SDK install in e.g. CI environments
+echo "Going to install full CUDA SDK and cu* libraries under host_injections if necessary"
+temp_install_storage=${TMPDIR}/temp_install_storage
+mkdir -p ${temp_install_storage}
+if [ -z "${skip_cuda_install}" ] || [ ! "${skip_cuda_install}" ]; then
+    ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh \
+        -t ${temp_install_storage} \
+        --accept-cuda-eula \
+        --accept-cudnn-eula
+else
+    echo "Skipping installation of CUDA SDK and cu* libraries in host_injections, since the --skip-cuda-install flag was passed"
+fi
+
+# Install NVIDIA drivers in host_injections (if they exist)
+if nvidia_gpu_available; then
+    echo "Installing NVIDIA drivers for use in prefix shell..."
+    ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh
+fi
+
+
 if [ ! -z "${shared_fs_path}" ]; then
     shared_eb_sourcepath=${shared_fs_path}/easybuild/sources
     echo ">> Using ${shared_eb_sourcepath} as shared EasyBuild source path"
diff --git a/bot/build.sh b/bot/build.sh
index a1e103a4..eb7c68c5 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -271,7 +271,11 @@ source $software_layer_dir/init/eessi_defaults
 # append the project (subdirectory) name to the end tarball name. This is information
 # then used at the ingestion stage. If ${EESSI_DEV_PROJECT} is not defined, nothing is
 # appended
-export TGZ=$(printf "eessi-%s-software-%s-%s-%s-%b%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_ACCELERATOR_TARGET_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
+if [[ -z ${EESSI_ACCELERATOR_TARGET_OVERRIDE} ]]; then
+    export TGZ=$(printf "eessi-%s-software-%s-%s-%b%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
+else
+    export TGZ=$(printf "eessi-%s-software-%s-%s-%s-%b%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_ACCELERATOR_TARGET_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
+fi
 
 # Export EESSI_DEV_PROJECT to use it (if needed) when making tarball
 echo "bot/build.sh: EESSI_DEV_PROJECT='${EESSI_DEV_PROJECT}'"