diff --git a/toolchain/bootstrap/modules.sh b/toolchain/bootstrap/modules.sh index bd4c85f621..fe7fed5918 100644 --- a/toolchain/bootstrap/modules.sh +++ b/toolchain/bootstrap/modules.sh @@ -24,8 +24,8 @@ if [ -v $u_c ]; then log "$Y""Gatech$W: Phoenix (p)" log "$R""Caltech$W: Richardson (r)" log "$BR""Brown$W: Oscar (o)" - log "$B""DoD$W: Carpenter (c) | Nautilus (n)" - log_n "($G""a$W/$G""f$W/$G""s$W/$G""w$W/$C""b$W/$C""e$CR/$C""d/$C""dai$CR/$Y""p$CR/$R""r$CR/$B""c$CR/$B""n$CR/$BR""o"$CR"): " + log "$B""DoD$W: Carpenter Cray (cc) | Carpenter GNU (c) | Nautilus (n)" + log_n "($G""a$W/$G""f$W/$G""s$W/$G""w$W/$C""b$W/$C""e$CR/$C""d/$C""dai$CR/$Y""p$CR/$R""r$CR/$B""cc$CR/$B""c$CR/$B""n$CR/$BR""o"$CR"): " read u_c log fi @@ -68,7 +68,7 @@ fi log "Loading modules (& env variables) for $M$COMPUTER$CR on $M$CG$CR"'s:' # Reset modules to default system configuration (unless Carpenter) -if [ "$u_c" != 'c' ]; then +if [ "$u_c" '!=' 'cc' ] && [ "$u_c" '!=' 'c' ]; then module reset > /dev/null 2>&1 code="$?" @@ -97,7 +97,7 @@ if [ $(echo "$VARIABLES" | grep = | wc -c) -gt 0 ]; then fi # Don't check for Cray paths on Carpenter, otherwise do check if they exist -if [ ! -z ${CRAY_LD_LIBRARY_PATH+x} ] && [ "$u_c" != 'c' ]; then +if [ ! -z ${CRAY_LD_LIBRARY_PATH+x} ] && [ "$u_c" '!=' 'c' ]; then ok "Found $M\$CRAY_LD_LIBRARY_PATH$CR. Prepending to $M\$LD_LIBRARY_PATH$CR." export LD_LIBRARY_PATH="$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH" fi diff --git a/toolchain/modules b/toolchain/modules index 429beb46e4..43ae2f7b6c 100644 --- a/toolchain/modules +++ b/toolchain/modules @@ -70,11 +70,15 @@ dai-all python cmake nvhpc-openmpi3/24.3 cuda dai-all CC=nvc CXX=nvc++ FC=nvfortran dai-gpu MFC_CUDA_CC=89,90 -c DoD Carpenter +c DoD Carpenter (GNU) c-all python/3.12.1 c-cpu compiler-rt/2024.2.0 ifort/2024.2.0 icc/2023.1.0 mpi/latest cmake/3.28.1-intel-2023.0.0 c-cpu CC=gcc CXX=g++ FC=gfortran +cc DoD Carpenter (Cray) +cc-all cray-python/3.11.7 craype-x86-rome +cc-cpu PrgEnv-cray/8.5.0 cray-fftw/3.3.10.8 cray-hdf5/1.14.3.1 cray-pals/1.4.0 cray-libpals/1.4.0 + n DoD Nautilus n-all slurm n-cpu penguin/openmpi/4.1.5/gcc-8.5.0 diff --git a/toolchain/templates/carpenter-cray.mako b/toolchain/templates/carpenter-cray.mako new file mode 100644 index 0000000000..bfaad0b427 --- /dev/null +++ b/toolchain/templates/carpenter-cray.mako @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +<%namespace name="helpers" file="helpers.mako"/> + +% if engine == 'batch': +#PBS -l select=${nodes}:ncpus=192:mpiprocs=${tasks_per_node} +#PBS -N "${name}" +#PBS -l walltime=${walltime} +% if partition: +#PBS -q ${partition} +% endif +% if account: +#PBS -A ${account} +% endif +% if email: +#PBS -M ${email} +#PBS -m abe +% endif +#PBS -o "${name}.out" +#PBS -e "${name}.err" +#PBS -V +% endif + +${helpers.template_prologue()} + +ok ":) Loading modules:\n" +cd "${MFC_ROOT_DIR}" +. ./mfc.sh load -c cc -m ${'g' if gpu else 'c'} +cd - > /dev/null +echo + +% for target in targets: + ${helpers.run_prologue(target)} + + % if not mpi: + (set -x; ${profiler} "${target.get_install_binpath(case)}") + % else: + (set -x; ${profiler} \ + mpirun -np ${nodes*tasks_per_node} \ + "${target.get_install_binpath(case)}") + % endif + + ${helpers.run_epilogue(target)} + + echo +% endfor + +${helpers.template_epilogue()}