toolchain: filter env vars when writing toolchain.env

fixes #709, but only for new builds
cp2k · Oct 22, 2020 · e64447f · e64447f
1 parent fe04ab9
commit e64447f
Show file tree

Hide file tree

Showing 33 changed files with 53 additions and 66 deletions.
diff --git a/tools/toolchain/install_cp2k_toolchain.sh b/tools/toolchain/install_cp2k_toolchain.sh
@@ -895,8 +895,7 @@ case $GPUVER in
         "--gpu-ver currently only supports K20X, K40, K80, P100, V100 as options"
 esac
 
-# write toolchain environment
-export -p > "${INSTALLDIR}"/toolchain.env
+write_toolchain_env "${INSTALLDIR}"
 
 # write toolchain config
 echo "tool_list=\"${tool_list}\"" > "${INSTALLDIR}"/toolchain.conf

diff --git a/tools/toolchain/scripts/install_acml.sh b/tools/toolchain/scripts/install_acml.sh
@@ -59,9 +59,8 @@ export FAST_MATH_LIBS="\${FAST_MATH_LIBS} ${ACML_LIBS}"
 EOF
 fi
 
-# update toolchain environment
 load "${BUILDDIR}/setup_acml"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 cd "${ROOTDIR}"
 report_timing "acml"
diff --git a/tools/toolchain/scripts/install_cmake.sh b/tools/toolchain/scripts/install_cmake.sh
@@ -69,9 +69,8 @@ EOF
     fi
 fi
 
-# update toolchain environment
 load "${BUILDDIR}/setup_cmake"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 cd "${ROOTDIR}"
 report_timing "cmake"
diff --git a/tools/toolchain/scripts/install_cosma.sh b/tools/toolchain/scripts/install_cosma.sh
@@ -159,9 +159,8 @@ EOF
     cat "${BUILDDIR}/setup_cosma" >> $SETUPFILE
 fi
 
-# update toolchain environment
 load "${BUILDDIR}/setup_cosma"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 cd "${ROOTDIR}"
 report_timing "cosma"
diff --git a/tools/toolchain/scripts/install_elpa.sh b/tools/toolchain/scripts/install_elpa.sh
@@ -176,9 +176,8 @@ export ELPAVERSION="${elpa_ver}"
 EOF
 fi
 
-# update toolchain environment
 load "${BUILDDIR}/setup_elpa"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 cd "${ROOTDIR}"
 report_timing "elpa"
diff --git a/tools/toolchain/scripts/install_fftw.sh b/tools/toolchain/scripts/install_fftw.sh
@@ -117,8 +117,7 @@ cat <<EOF >> ${INSTALLDIR}/valgrind.supp
 }
 EOF
 
-# update toolchain environment
 load "${BUILDDIR}/setup_fftw"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 report_timing "fftw"
diff --git a/tools/toolchain/scripts/install_gcc.sh b/tools/toolchain/scripts/install_gcc.sh
@@ -193,8 +193,7 @@ export LSAN_OPTIONS=suppressions=${INSTALLDIR}/lsan.supp
 export TSAN_OPTIONS=suppressions=${INSTALLDIR}/tsan.supp
 EOF
 
-# update toolchain environment
 load "${BUILDDIR}/setup_gcc"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 report_timing "gcc"
diff --git a/tools/toolchain/scripts/install_gsl.sh b/tools/toolchain/scripts/install_gsl.sh
@@ -102,9 +102,8 @@ EOF
     cat "${BUILDDIR}/setup_gsl" >> $SETUPFILE
 fi
 
-# update toolchain environment
 load "${BUILDDIR}/setup_gsl"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 cd "${ROOTDIR}"
 report_timing "gsl"
diff --git a/tools/toolchain/scripts/install_hdf5.sh b/tools/toolchain/scripts/install_hdf5.sh
@@ -92,9 +92,8 @@ EOF
     cat "${BUILDDIR}/setup_hdf5" >> $SETUPFILE
 fi
 
-# update toolchain environment
 load "${BUILDDIR}/setup_hdf5"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 cd "${ROOTDIR}"
 report_timing "hdf5"
diff --git a/tools/toolchain/scripts/install_intelmpi.sh b/tools/toolchain/scripts/install_intelmpi.sh
@@ -75,9 +75,8 @@ export CP_LIBS="\${CP_LIBS} IF_MPI(${INTELMPI_LIBS}|)"
 EOF
 fi
 
-# update toolchain environment
 load "${BUILDDIR}/setup_intelmpi"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 cd "${ROOTDIR}"
 report_timing "intelmpi"
diff --git a/tools/toolchain/scripts/install_libint.sh b/tools/toolchain/scripts/install_libint.sh
@@ -128,9 +128,8 @@ export CP_LIBS="${LIBINT_LIBS} \${CP_LIBS}"
 EOF
 fi
 
-# update toolchain environment
 load "${BUILDDIR}/setup_libint"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 cd "${ROOTDIR}"
 report_timing "libint"
diff --git a/tools/toolchain/scripts/install_libsmm.sh b/tools/toolchain/scripts/install_libsmm.sh
@@ -120,9 +120,8 @@ export CP_LIBS="IF_VALGRIND(|${LIBSMM_LIBS}) \${CP_LIBS}"
 EOF
 fi
 
-# update toolchain environment
 load "${BUILDDIR}/setup_libsmm"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 cd "${ROOTDIR}"
 report_timing "libsmm"
diff --git a/tools/toolchain/scripts/install_libvdwxc.sh b/tools/toolchain/scripts/install_libvdwxc.sh
@@ -130,9 +130,8 @@ EOF
         cat "${BUILDDIR}/setup_libvdwxc" >> $SETUPFILE
 fi
 
-# update toolchain environment
 load "${BUILDDIR}/setup_libvdwxc"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 cd "${ROOTDIR}"
 report_timing "libvdwxc"
diff --git a/tools/toolchain/scripts/install_libxc.sh b/tools/toolchain/scripts/install_libxc.sh
@@ -86,9 +86,8 @@ export LIBXCROOT="$pkg_install_dir"
 EOF
 fi
 
-# update toolchain environment
 load "${BUILDDIR}/setup_libxc"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 cd "${ROOTDIR}"
 report_timing "libxc"
diff --git a/tools/toolchain/scripts/install_libxsmm.sh b/tools/toolchain/scripts/install_libxsmm.sh
@@ -122,8 +122,7 @@ EOF
 fi
 cd "${ROOTDIR}"
 
-# update toolchain environment
 load "${BUILDDIR}/setup_libxsmm"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 report_timing "libxsmm"
diff --git a/tools/toolchain/scripts/install_mathlibs.sh b/tools/toolchain/scripts/install_mathlibs.sh
@@ -18,8 +18,7 @@ export FAST_MATH_CFLAGS=''
 export FAST_MATH_LDFLAGS=''
 export FAST_MATH_LIBS=''
 
-# update toolchain environment
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 "${SCRIPTDIR}"/install_reflapack.sh "${with_reflapack}"; load "${BUILDDIR}/setup_reflapack"
 
@@ -63,7 +62,6 @@ export CP_CFLAGS="${CP_CFLAGS} IF_DEBUG(${REF_MATH_CFLAGS}|IF_VALGRIND(${REF_MAT
 export CP_LDFLAGS="${CP_LDFLAGS} IF_DEBUG(${REF_MATH_LDFLAGS}|IF_VALGRIND(${REF_MATH_LDFLAGS}|${FAST_MATH_LDFLAGS}))"
 export CP_LIBS="${CP_LIBS} IF_DEBUG(${REF_MATH_LIBS}|IF_VALGRIND(${REF_MATH_LIBS}|${FAST_MATH_LIBS}))"
 
-# update toolchain environment
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 #EOF
diff --git a/tools/toolchain/scripts/install_mkl.sh b/tools/toolchain/scripts/install_mkl.sh
@@ -119,9 +119,8 @@ EOF
     fi
 fi
 
-# update toolchain environment
 load "${BUILDDIR}/setup_mkl"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 cd "${ROOTDIR}"
 report_timing "mkl"
diff --git a/tools/toolchain/scripts/install_mpich.sh b/tools/toolchain/scripts/install_mpich.sh
@@ -117,9 +117,8 @@ export CP_LIBS="\${CP_LIBS} IF_MPI(${MPICH_LIBS}|)"
 EOF
 fi
 
-# update toolchain environment
 load "${BUILDDIR}/setup_mpich"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 cd "${ROOTDIR}"
 report_timing "mpich"
diff --git a/tools/toolchain/scripts/install_openblas.sh b/tools/toolchain/scripts/install_openblas.sh
@@ -135,9 +135,8 @@ export FAST_MATH_LIBS="\${FAST_MATH_LIBS} ${OPENBLAS_LIBS}"
 EOF
 fi
 
-# update toolchain environment
 load "${BUILDDIR}/setup_openblas"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 cd "${ROOTDIR}"
 report_timing "openblas"
diff --git a/tools/toolchain/scripts/install_openmpi.sh b/tools/toolchain/scripts/install_openmpi.sh
@@ -193,8 +193,7 @@ leak:progress_engine
 leak:__GI___strdup
 EOF
 
-# update toolchain environment
 load "${BUILDDIR}/setup_openmpi"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 report_timing "openmpi"
diff --git a/tools/toolchain/scripts/install_pexsi.sh b/tools/toolchain/scripts/install_pexsi.sh
@@ -124,9 +124,8 @@ export CP_LIBS="IF_MPI(${PEXSI_LIBS}|) \${CP_LIBS}"
 EOF
 fi
 
-# update toolchain environment
 load "${BUILDDIR}/setup_pexsi"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 cd "${ROOTDIR}"
 report_timing "pexsi"
diff --git a/tools/toolchain/scripts/install_plumed.sh b/tools/toolchain/scripts/install_plumed.sh
@@ -85,9 +85,8 @@ export CP_LIBS="${PLUMED_LIBS} \${CP_LIBS}"
 EOF
 fi
 
-# update toolchain environment
 load "${BUILDDIR}/setup_plumed"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 cd "${ROOTDIR}"
 report_timing "plumed"
diff --git a/tools/toolchain/scripts/install_ptscotch.sh b/tools/toolchain/scripts/install_ptscotch.sh
@@ -104,9 +104,8 @@ export CP_LIBS="IF_MPI(${SCOTCH_LIBS}|) \${CP_LIBS}"
 EOF
 fi
 
-# update toolchain environment
 load "${BUILDDIR}/setup_ptscotch"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 cd "${ROOTDIR}"
 report_timing "ptscotch"
diff --git a/tools/toolchain/scripts/install_quip.sh b/tools/toolchain/scripts/install_quip.sh
@@ -153,9 +153,8 @@ export CP_LIBS="${QUIP_LIBS} \${CP_LIBS}"
 EOF
 fi
 
-# update toolchain environment
 load "${BUILDDIR}/setup_quip"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 cd "${ROOTDIR}"
 report_timing "quip"
diff --git a/tools/toolchain/scripts/install_reflapack.sh b/tools/toolchain/scripts/install_reflapack.sh
@@ -108,9 +108,8 @@ EOF
     fi
 fi
 
-# update toolchain environment
 load "${BUILDDIR}/setup_reflapack"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 cd "${ROOTDIR}"
 report_timing "reflapack"
diff --git a/tools/toolchain/scripts/install_scalapack.sh b/tools/toolchain/scripts/install_scalapack.sh
@@ -106,8 +106,7 @@ cat <<EOF >> ${INSTALLDIR}/lsan.supp
 leak:pdpotrf_
 EOF
 
-# update toolchain environment
 load "${BUILDDIR}/setup_scalapack"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 report_timing "scalapack"
diff --git a/tools/toolchain/scripts/install_sirius.sh b/tools/toolchain/scripts/install_sirius.sh
@@ -250,9 +250,8 @@ export CP_LIBS="IF_MPI("\${SIRIUS_LIBS}"|) \${CP_LIBS}"
 EOF
 fi
 
-# update toolchain environment
 load "${BUILDDIR}/setup_sirius"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 cd "${ROOTDIR}"
 report_timing "sirius"
diff --git a/tools/toolchain/scripts/install_spfft.sh b/tools/toolchain/scripts/install_spfft.sh
@@ -111,9 +111,8 @@ EOF
     cat "${BUILDDIR}/setup_spfft" >> $SETUPFILE
 fi
 
-# update toolchain environment
 load "${BUILDDIR}/setup_spfft"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 cd "${ROOTDIR}"
 report_timing "spfft"
diff --git a/tools/toolchain/scripts/install_spglib.sh b/tools/toolchain/scripts/install_spglib.sh
@@ -94,9 +94,8 @@ EOF
         cat "${BUILDDIR}/setup_spglib" >> $SETUPFILE
 fi
 
-# update toolchain environment
 load "${BUILDDIR}/setup_spglib"
-export -p > "${INSTALLDIR}"/toolchain.env
+write_toolchain_env "${INSTALLDIR}"
 
 cd "${ROOTDIR}"
 report_timing "spglib"
diff --git a/tools/toolchain/scripts/install_superlu.sh b/tools/toolchain/scripts/install_superlu.sh
@@ -94,9 +94,8 @@ export CP_LIBS="IF_MPI(${SUPERLU_LIBS}|) \${CP_LIBS}"
 EOF
 fi
 
-# update toolchain environment
 load "${BUILDDIR}/setup_superlu"
-export -p > "${INSTALLDIR}"/toolchain.env
+write_toolchain_env "${INSTALLDIR}"
 
 cd "${ROOTDIR}"
 report_timing "superlu"
diff --git a/tools/toolchain/scripts/install_valgrind.sh b/tools/toolchain/scripts/install_valgrind.sh
@@ -94,8 +94,7 @@ cat <<EOF >> ${SETUPFILE}
 export VALGRIND_OPTIONS="--suppressions=${INSTALLDIR}/valgrind.supp --max-stackframe=2168152 --error-exitcode=42"
 EOF
 
-# update toolchain environment
 load "${BUILDDIR}/setup_valgrind"
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 report_timing "valgrind"
diff --git a/tools/toolchain/scripts/setup_buildtools.sh b/tools/toolchain/scripts/setup_buildtools.sh
@@ -37,7 +37,6 @@ export LDFLAGS="${TSANFLAGS}"
 # get system arch information using OpenBLAS prebuild
 "${SCRIPTDIR}"/get_openblas_arch.sh; load "${BUILDDIR}/openblas_arch"
 
-# update toolchain environment
-export -p > "${INSTALLDIR}/toolchain.env"
+write_toolchain_env "${INSTALLDIR}"
 
 #EOF
diff --git a/tools/toolchain/scripts/tool_kit.sh b/tools/toolchain/scripts/tool_kit.sh
@@ -674,3 +674,23 @@ write_checksums() {
 
    ${__shasum_command} "${VERSION_FILE}" "$@" >"${__checksum_file}"
 }
+
+# generate a filtered toolchain.env
+write_toolchain_env() {
+    local __installdir=$1
+
+    # we do not need to achieve complete filtering, it is sufficient to
+    # remove problematic variables (TERM/TERMCAP/COLORTERM) which may trigger
+    # 'too many arguments' (since the environment vars are stored in the same memory block as command line arguments)
+    # or which may not be valid anymore the next time the user runs the toolchain scripts,
+    # like the proxy vars which may affect fetching tarballs
+    export -p | sed \
+        -e '/^\(declare -[rx]\+ \)\?\(COLORTERM\|DISPLAY\|EDITOR\|LESS\|LESSOPEN\|LOGNAME\|LS_COLORS\|PAGER\)=/d' \
+        -e '/^\(declare -[rx]\+ \)\?\(TERM\|TERMCAP\|USER\)=/d' \
+        -e '/^\(declare -[rx]\+ \)\?\(ftp\|http\|no\)_proxy=/d' \
+        -e '/^\(declare -[rx]\+ \)\?\(GPG_AGENT_INFO\|SSH_AGENT_PID\|SSH_AUTH_SOCK\)=/d' \
+        -e '/^\(declare -[rx]\+ \)\?LS_\(COLORS\|OPTIONS\)=/d' \
+        -e '/^\(declare -[rx]\+ \)\?\(STY\|WINDOW\|XAUTHORITY\)=/d' \
+        -e '/^\(declare -[rx]\+ \)\?XDG_/d' \
+        > "${__installdir}/toolchain.env"
+}