From f01cca6606aebcc765328e79f8a63433bee4c4e1 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Wed, 14 Jun 2023 10:37:01 -0400 Subject: [PATCH 01/14] Gaea modulefile update, fix files location update on Jet,Hera,Orion,Gaea,Cheyenne --- modulefiles/gsi_cheyenne.intel.lua | 4 +- modulefiles/gsi_gaea | 62 ------------------------------ modulefiles/gsi_gaea.lua | 31 +++++++++++++++ modulefiles/gsi_hera.intel.lua | 2 +- modulefiles/gsi_jet.lua | 2 +- modulefiles/gsi_orion.lua | 2 +- ush/module-setup.sh | 26 +------------ 7 files changed, 37 insertions(+), 92 deletions(-) delete mode 100644 modulefiles/gsi_gaea create mode 100644 modulefiles/gsi_gaea.lua diff --git a/modulefiles/gsi_cheyenne.intel.lua b/modulefiles/gsi_cheyenne.intel.lua index cafc1973be..d02a0c3885 100644 --- a/modulefiles/gsi_cheyenne.intel.lua +++ b/modulefiles/gsi_cheyenne.intel.lua @@ -17,8 +17,8 @@ load("mkl/2022.1") load("gsi_common") -local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" -load(pathJoin("prod_util", prod_util_ver)) +load(pathJoin("prod_util", os.getenv("prod_util_ver") or "1.2.2")) +pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_data/fix") pushenv("CFLAGS", "-xHOST") pushenv("FFLAGS", "-xHOST") diff --git a/modulefiles/gsi_gaea b/modulefiles/gsi_gaea deleted file mode 100644 index 641f3d0fcf..0000000000 --- a/modulefiles/gsi_gaea +++ /dev/null @@ -1,62 +0,0 @@ -#%Module1.0 -###################################################################### -## NOAA-EMC/GSI -##_____________________________________________________ -proc ModulesHelp { } { -puts stderr "Set environment variables for NOAA-EMC/GSI" -puts stderr "This module initializes the environment " -puts stderr "for the Intel Compiler Suite $version\n" -} -module-whatis " NOAA-EMC/GSI whatis description" - -set COMPILER intel - -setenv FFLAGS_COM "-fp-model strict" -setenv LDFLAGS_COM " " - -#set WRF_SHARED_VER v1.1.0 -#set WRF_SHARED_ROOT /gpfs/hps/nco/ops/nwprod/wrf_shared -#set WRF_SHARED_ROOT /lustre/f1/pdata/ncep_shared/NCEPLIBS/lib/EXTERNAL/wrf_shared -#setenv WRF_SHARED_PATH ${WRF_SHARED_ROOT}.${WRF_SHARED_VER} - -setenv NCEPLIBS /lustre/f1/pdata/ncep_shared/NCEPLIBS/lib - -# Loading ncep environment -##module load ncep/1.0 -module use /opt/cray/pe/craype/2.5.5/modulefiles - -# Loading Intel Compiler Suite -module load PrgEnv-intel - -# Loading pe environment -module load cray-mpich -module load cray-libsci -module unload craype-broadwell -module load craype-haswell - -module use /sw/gaea/modulefiles -module load cmake - -# Loading nceplibs modules -module use /lustre/f1/pdata/ncep_shared/NCEPLIBS/lib/modulefiles -#module load HDF5-serial-intel-haswell/1.8.9 -#module load NetCDF-intel-haswell/4.2 -module load cray-hdf5 -module load cray-netcdf - -#module use /gpfs/hps/nco/ops/nwprod/lib/modulefiles -module load bufr-intel-sandybridge/11.0.1 -module load nemsio-intel-sandybridge/2.2.2 -module load sfcio-intel-sandybridge/1.0.0 -module load sigio-intel-sandybridge/2.0.1 -module load sp-intel-sandybridge/2.0.2 -module load w3nco-intel-sandybridge/2.0.6 -module load w3emc-intel-sandybridge/2.2.0 -module load bacio-intel-sandybridge/2.0.2 -setenv CRAYOS_VERSION $::env(CRAYPE_VERSION) -#setenv CRAYOS_VERSION ${CRAYPE_VERSION} - -# Compiler flags specific to this platform -setenv CFLAGS "-xCORE-AVX2" -setenv FFLAGS "-xCORE-AVX2" - diff --git a/modulefiles/gsi_gaea.lua b/modulefiles/gsi_gaea.lua new file mode 100644 index 0000000000..9ea375d72e --- /dev/null +++ b/modulefiles/gsi_gaea.lua @@ -0,0 +1,31 @@ +help([[ +]]) + +load("cmake/3.20.1") + +prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2/modulefiles/stack") +load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) + +load(pathJoin("intel-classic", os.getenv("intel_classic_ver") or "2022.0.2")) +load(pathJoin("cray-mpich", os.getenv("cray_mpich_ver") or "7.7.20")) +load(pathJoin("hpc-intel-classic", os.getenv("hpc_intel_classic_ver") or "2022.0.2")) +load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "7.7.20")) + +load("gsi_common") + +local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" +load(pathJoin("prod_util", prod_util_ver)) + +local MKLROOT="/opt/intel/oneapi/mkl/2022.0.2/" +prepend_path("LD_LIBRARY_PATH",pathJoin(MKLROOT,"lib/intel64")) +pushenv("MKLROOT", MKLROOT) + +pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix") +setenv("CC","cc") +setenv("FC","ftn") +setenv("CXX","CC") +unsetenv("CRAYPE_LINK_TYPE") +pushenv("CRAYPE_LINK_TYPE","dynamic") + +whatis("Description: GSI environment on Gaea with Intel Compilers") + diff --git a/modulefiles/gsi_hera.intel.lua b/modulefiles/gsi_hera.intel.lua index 7c98d48ad9..866af02d50 100644 --- a/modulefiles/gsi_hera.intel.lua +++ b/modulefiles/gsi_hera.intel.lua @@ -25,6 +25,6 @@ load(pathJoin("prod_util", prod_util_ver)) pushenv("CFLAGS", "-xHOST") pushenv("FFLAGS", "-xHOST") -pushenv("GSI_BINARY_SOURCE_DIR", "/scratch1/NCEPDEV/global/glopara/fix/gsi/20221128") +pushenv("GSI_BINARY_SOURCE_DIR", "/scratch1/NCEPDEV/global/glopara/fix/gsi/20230601") whatis("Description: GSI environment on Hera with Intel Compilers") diff --git a/modulefiles/gsi_jet.lua b/modulefiles/gsi_jet.lua index 2f8cacff8a..e2ea2ef1d0 100644 --- a/modulefiles/gsi_jet.lua +++ b/modulefiles/gsi_jet.lua @@ -26,6 +26,6 @@ pushenv("CFLAGS", "-axSSE4.2,AVX,CORE-AVX2") pushenv("FFLAGS", "-axSSE4.2,AVX,CORE-AVX2") -pushenv("GSI_BINARY_SOURCE_DIR", "/mnt/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/gsi/20221128") +pushenv("GSI_BINARY_SOURCE_DIR", "/mnt/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/gsi/20230601") whatis("Description: GSI environment on Jet with Intel Compilers") diff --git a/modulefiles/gsi_orion.lua b/modulefiles/gsi_orion.lua index 1f0787be50..43276b7cbe 100644 --- a/modulefiles/gsi_orion.lua +++ b/modulefiles/gsi_orion.lua @@ -25,6 +25,6 @@ load(pathJoin("prod_util", prod_util_ver)) pushenv("CFLAGS", "-xHOST") pushenv("FFLAGS", "-xHOST") -pushenv("GSI_BINARY_SOURCE_DIR", "/work/noaa/global/glopara/fix/gsi/20221128") +pushenv("GSI_BINARY_SOURCE_DIR", "/work/noaa/global/glopara/fix/gsi/20230601") whatis("Description: GSI environment on Orion with Intel Compilers") diff --git a/ush/module-setup.sh b/ush/module-setup.sh index 469fd4a3c5..207dbba925 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -57,32 +57,8 @@ elif [[ $MACHINE_ID = gaea* ]] ; then # /etc/profile here. source /etc/profile __ms_source_etc_profile=yes - else - __ms_source_etc_profile=no - fi - module purge - # clean up after purge - unset _LMFILES_ - unset _LMFILES_000 - unset _LMFILES_001 - unset LOADEDMODULES - module load modules - if [[ -d /opt/cray/ari/modulefiles ]] ; then - module use -a /opt/cray/ari/modulefiles - fi - if [[ -d /opt/cray/pe/ari/modulefiles ]] ; then - module use -a /opt/cray/pe/ari/modulefiles - fi - if [[ -d /opt/cray/pe/craype/default/modulefiles ]] ; then - module use -a /opt/cray/pe/craype/default/modulefiles - fi - if [[ -s /etc/opt/cray/pe/admin-pe/site-config ]] ; then - source /etc/opt/cray/pe/admin-pe/site-config - fi - if [[ "$__ms_source_etc_profile" == yes ]] ; then - source /etc/profile - unset __ms_source_etc_profile fi + source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh elif [[ $MACHINE_ID = expanse* ]]; then # We are on SDSC Expanse From aace46e7f47630027a49da1588d70724b9913698 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Wed, 14 Jun 2023 10:56:04 -0400 Subject: [PATCH 02/14] Gaea modulefile update --- modulefiles/gsi_gaea.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/modulefiles/gsi_gaea.lua b/modulefiles/gsi_gaea.lua index 9ea375d72e..ea1826fde9 100644 --- a/modulefiles/gsi_gaea.lua +++ b/modulefiles/gsi_gaea.lua @@ -24,7 +24,6 @@ pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix" setenv("CC","cc") setenv("FC","ftn") setenv("CXX","CC") -unsetenv("CRAYPE_LINK_TYPE") pushenv("CRAYPE_LINK_TYPE","dynamic") whatis("Description: GSI environment on Gaea with Intel Compilers") From 3bf5e013c3bead65286e41b2b613af315659ed93 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Fri, 23 Jun 2023 08:17:51 -0400 Subject: [PATCH 03/14] Modulefile for Gaea and regression tests configuration --- modulefiles/gsi_common.lua | 2 +- modulefiles/gsi_gaea | 62 ------------ modulefiles/gsi_gaea.lua | 33 +++++++ regression/regression_driver.sh | 3 +- regression/regression_param.sh | 37 +++++++ regression/regression_var.sh | 14 +++ ush/build.sh | 7 +- ush/module-setup.sh | 26 +---- ush/sub_gaea | 166 ++++++++++++++++++++++++++++++++ 9 files changed, 258 insertions(+), 92 deletions(-) delete mode 100644 modulefiles/gsi_gaea create mode 100644 modulefiles/gsi_gaea.lua create mode 100755 ush/sub_gaea diff --git a/modulefiles/gsi_common.lua b/modulefiles/gsi_common.lua index b2929bcdf8..ba2eea3241 100644 --- a/modulefiles/gsi_common.lua +++ b/modulefiles/gsi_common.lua @@ -16,7 +16,7 @@ local nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" local wrf_io_ver=os.getenv("wrf_io_ver") or "1.2.0" local ncio_ver=os.getenv("ncio_ver") or "1.1.2" local crtm_ver=os.getenv("crtm_ver") or "2.4.0" -local ncdiag_ver=os.getenv("ncdiag_ver") or "1.1.0" +local ncdiag_ver=os.getenv("ncdiag_ver") or "1.1.1" load(pathJoin("netcdf", netcdf_ver)) diff --git a/modulefiles/gsi_gaea b/modulefiles/gsi_gaea deleted file mode 100644 index 641f3d0fcf..0000000000 --- a/modulefiles/gsi_gaea +++ /dev/null @@ -1,62 +0,0 @@ -#%Module1.0 -###################################################################### -## NOAA-EMC/GSI -##_____________________________________________________ -proc ModulesHelp { } { -puts stderr "Set environment variables for NOAA-EMC/GSI" -puts stderr "This module initializes the environment " -puts stderr "for the Intel Compiler Suite $version\n" -} -module-whatis " NOAA-EMC/GSI whatis description" - -set COMPILER intel - -setenv FFLAGS_COM "-fp-model strict" -setenv LDFLAGS_COM " " - -#set WRF_SHARED_VER v1.1.0 -#set WRF_SHARED_ROOT /gpfs/hps/nco/ops/nwprod/wrf_shared -#set WRF_SHARED_ROOT /lustre/f1/pdata/ncep_shared/NCEPLIBS/lib/EXTERNAL/wrf_shared -#setenv WRF_SHARED_PATH ${WRF_SHARED_ROOT}.${WRF_SHARED_VER} - -setenv NCEPLIBS /lustre/f1/pdata/ncep_shared/NCEPLIBS/lib - -# Loading ncep environment -##module load ncep/1.0 -module use /opt/cray/pe/craype/2.5.5/modulefiles - -# Loading Intel Compiler Suite -module load PrgEnv-intel - -# Loading pe environment -module load cray-mpich -module load cray-libsci -module unload craype-broadwell -module load craype-haswell - -module use /sw/gaea/modulefiles -module load cmake - -# Loading nceplibs modules -module use /lustre/f1/pdata/ncep_shared/NCEPLIBS/lib/modulefiles -#module load HDF5-serial-intel-haswell/1.8.9 -#module load NetCDF-intel-haswell/4.2 -module load cray-hdf5 -module load cray-netcdf - -#module use /gpfs/hps/nco/ops/nwprod/lib/modulefiles -module load bufr-intel-sandybridge/11.0.1 -module load nemsio-intel-sandybridge/2.2.2 -module load sfcio-intel-sandybridge/1.0.0 -module load sigio-intel-sandybridge/2.0.1 -module load sp-intel-sandybridge/2.0.2 -module load w3nco-intel-sandybridge/2.0.6 -module load w3emc-intel-sandybridge/2.2.0 -module load bacio-intel-sandybridge/2.0.2 -setenv CRAYOS_VERSION $::env(CRAYPE_VERSION) -#setenv CRAYOS_VERSION ${CRAYPE_VERSION} - -# Compiler flags specific to this platform -setenv CFLAGS "-xCORE-AVX2" -setenv FFLAGS "-xCORE-AVX2" - diff --git a/modulefiles/gsi_gaea.lua b/modulefiles/gsi_gaea.lua new file mode 100644 index 0000000000..23fa6edf7d --- /dev/null +++ b/modulefiles/gsi_gaea.lua @@ -0,0 +1,33 @@ +help([[ +]]) + +load("cmake/3.20.1") + +prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2/modulefiles/stack") +load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) + +load(pathJoin("intel-classic", os.getenv("intel_classic_ver") or "2022.0.2")) +load(pathJoin("cray-mpich", os.getenv("cray_mpich_ver") or "7.7.20")) +load(pathJoin("hpc-intel-classic", os.getenv("hpc_intel_classic_ver") or "2022.0.2")) +load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "7.7.20")) + +load("gsi_common") + +local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" +load(pathJoin("prod_util", prod_util_ver)) + +-- Needed at runtime: +load("alps") + +local MKLROOT="/opt/intel/oneapi/mkl/2022.0.2/" +prepend_path("LD_LIBRARY_PATH",pathJoin(MKLROOT,"lib/intel64")) +pushenv("MKLROOT", MKLROOT) + +pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix/20230601") +setenv("CC","cc") +setenv("FC","ftn") +setenv("CXX","CC") +pushenv("CRAYPE_LINK_TYPE","dynamic") + +whatis("Description: GSI environment on Gaea with Intel Compilers") + diff --git a/regression/regression_driver.sh b/regression/regression_driver.sh index e1d3b18dc7..58ebf64e85 100755 --- a/regression/regression_driver.sh +++ b/regression/regression_driver.sh @@ -10,6 +10,7 @@ if [ -d "$config_path" ]; then source $config_path/local_vars.sh fi + # source the necessary files to setup if [ "$#" -eq 2 ]; then export regdir=$2 @@ -41,7 +42,7 @@ for jn in `seq ${RSTART} ${REND}`; do fi rm -f ${job[$jn]}.out - /bin/sh $ush/$sub_cmd -q $queue -j ${job[$jn]} -t ${topts[$jn]} -p ${popts[$jn]} -r ${ropts[$jn]} $scripts/${regtest}.sh + /bin/sh $ush/$sub_cmd -q $queue -j ${job[$jn]} -t ${topts[$jn]} -p ${popts[$jn]} -r ${ropts[$jn]} $scripts/${regtest}.sh >& $ush/sub_cmd.${job[$jn]}.out if [ $debug == ".true." ]; then break; fi $scripts/regression_wait.sh ${job[$jn]} ${rcname} $check_resource diff --git a/regression/regression_param.sh b/regression/regression_param.sh index 6024dbdb54..ea597fe0ae 100755 --- a/regression/regression_param.sh +++ b/regression/regression_param.sh @@ -18,6 +18,11 @@ case $machine in sub_cmd="sub_jet" memnode=96 numcore=40 + ;; + Gaea) + sub_cmd="sub_gaea" + memnode=64 + numcore=36 ;; wcoss2) sub_cmd="sub_wcoss2" @@ -58,6 +63,9 @@ case $regtest in elif [[ "$machine" = "Cheyenne" ]]; then topts[1]="0:30:00" ; popts[1]="16/2/" ; ropts[1]="/1" topts[2]="0:30:00" ; popts[2]="16/4/" ; ropts[2]="/2" + elif [[ "$machine" = "Gaea" ]]; then + topts[1]="0:30:00" ; popts[1]="18/2/" ; ropts[1]="/1" + topts[2]="0:30:00" ; popts[2]="18/4/" ; ropts[2]="/2" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:15:00" ; popts[1]="12/5/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="12/9/" ; ropts[2]="/2" @@ -88,6 +96,9 @@ case $regtest in elif [[ "$machine" = "Cheyenne" ]]; then topts[1]="0:35:00" ; popts[1]="16/2/" ; ropts[1]="/1" topts[2]="0:25:00" ; popts[2]="16/4/" ; ropts[2]="/2" + elif [[ "$machine" = "Gaea" ]]; then + topts[1]="0:35:00" ; popts[1]="18/2/" ; ropts[1]="/1" + topts[2]="0:25:00" ; popts[2]="18/4/" ; ropts[2]="/2" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:15:00" ; popts[1]="28/2/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="28/4/" ; ropts[2]="/2" @@ -104,6 +115,8 @@ case $regtest in popts[1]="12/5/" elif [[ "$machine" = "Jet" ]]; then popts[1]="12/5/" + elif [[ "$machine" = "Gaea" ]]; then + popts[1]="18/5/" elif [[ "$machine" = "wcoss2" ]]; then popts[1]="28/4/" topts[1]="3:00:00" @@ -131,6 +144,9 @@ case $regtest in elif [[ "$machine" = "Cheyenne" ]]; then topts[1]="1:59:00" ; popts[1]="6/8/" ; ropts[1]="/1" topts[2]="0:35:00" ; popts[2]="6/10/" ; ropts[2]="/2" + elif [[ "$machine" = "Gaea" ]]; then + topts[1]="0:10:00" ; popts[1]="18/8/" ; ropts[1]="/1" + topts[2]="0:10:00" ; popts[2]="18/10/" ; ropts[2]="/2" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2" @@ -155,6 +171,9 @@ case $regtest in elif [[ "$machine" = "Jet" ]]; then topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" + elif [[ "$machine" = "Gaea" ]]; then + topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1" @@ -179,6 +198,9 @@ case $regtest in elif [[ "$machine" = "Jet" ]]; then topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1" + elif [[ "$machine" = "Gaea" ]]; then + topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:15:00" ; popts[1]="28/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="28/2/" ; ropts[2]="/1" @@ -206,6 +228,9 @@ case $regtest in elif [[ "$machine" = "Cheyenne" ]]; then topts[1]="0:15:00" ; popts[1]="8/6/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="8/8/" ; ropts[2]="/1" + elif [[ "$machine" = "Gaea" ]]; then + topts[1]="0:30:00" ; popts[1]="8/6/" ; ropts[1]="/1" + topts[2]="0:30:00" ; popts[2]="8/8/" ; ropts[2]="/1" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:30:00" ; popts[1]="14/8/" ; ropts[1]="/1" topts[2]="0:30:00" ; popts[2]="14/14/" ; ropts[2]="/2" @@ -233,6 +258,9 @@ case $regtest in elif [[ "$machine" = "Cheyenne" ]]; then topts[1]="0:20:00" ; popts[1]="6/6/" ; ropts[1]="/1" topts[2]="0:20:00" ; popts[2]="8/8/" ; ropts[2]="/1" + elif [[ "$machine" = "Gaea" ]]; then + topts[1]="0:20:00" ; popts[1]="6/6/" ; ropts[1]="/1" + topts[2]="0:20:00" ; popts[2]="8/8/" ; ropts[2]="/1" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:15:00" ; popts[1]="10/10/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="14/14/" ; ropts[2]="/2" @@ -260,6 +288,9 @@ case $regtest in elif [[ "$machine" = "Cheyenne" ]]; then topts[1]="0:15:00" ; popts[1]="16/2/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="16/4/" ; ropts[2]="/2" + elif [[ "$machine" = "Gaea" ]]; then + topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1" + topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:10:00" ; popts[1]="16/2/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="16/4/" ; ropts[2]="/2" @@ -317,6 +348,12 @@ elif [[ "$machine" = "Jet" ]]; then export MPI_BUFS_PER_HOST=256 export MPI_GROUP_MAX=256 export APRUN="srun" +elif [[ "$machine" = "Gaea" ]]; then + export OMP_STACKSIZE=1024M + export MPI_BUFS_PER_PROC=256 + export MPI_BUFS_PER_HOST=256 + export MPI_GROUP_MAX=256 + export APRUN="srun --export=ALL --mpi=pmi2 -n \$size" elif [[ "$machine" = "Cheyenne" ]]; then export OMP_STACKSIZE=1024M export MPI_BUFS_PER_PROC=256 diff --git a/regression/regression_var.sh b/regression/regression_var.sh index 05b5563ef1..e883a5b1c8 100755 --- a/regression/regression_var.sh +++ b/regression/regression_var.sh @@ -49,6 +49,20 @@ fi echo "Running Regression Tests on '$machine'"; case $machine in + Gaea) + export queue="batch" + export noscrub="/lustre/f2/scratch/$LOGNAME/gsi_tmp/noscrub" + export ptmp="/lustre/f2/scratch/$LOGNAME/gsi_tmp/ptmp" + export casesdir="/lustre/f2/dev/role.epic/contrib/GSI_data/CASES/regtest" + + export group="global" + if [[ "$cmaketest" = "false" ]]; then + export basedir="/lustre/f2/dev/$LOGNAME/sandbox/GSI" + fi + + export check_resource="no" + export accnt="nggps_emc" + ;; Cheyenne) export queue="economy" export noscrub="/glade/scratch/$LOGNAME" diff --git a/ush/build.sh b/ush/build.sh index 71674c4f4c..f706f413c1 100755 --- a/ush/build.sh +++ b/ush/build.sh @@ -30,12 +30,13 @@ set -x # Set CONTROLPATH variable to user develop installation CONTROLPATH="$DIR_ROOT/../develop/install/bin" - +CMAKELIBS=".so .a" +CMAKE_OPTS+=" -DCMAKE_FIND_LIBRARY_SUFFIXES=${CMAKELIBS}" # Collect BUILD Options CMAKE_OPTS+=" -DCMAKE_BUILD_TYPE=$BUILD_TYPE" # Install destination for built executables, libraries, CMake Package config -CMAKE_OPTS+=" -DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX" +CMAKE_OPTS+=" -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}" # Configure for GSI and EnKF CMAKE_OPTS+=" -DGSI_MODE=$GSI_MODE -DENKF_MODE=${ENKF_MODE}" @@ -44,7 +45,7 @@ CMAKE_OPTS+=" -DGSI_MODE=$GSI_MODE -DENKF_MODE=${ENKF_MODE}" [[ ${REGRESSION_TESTS} =~ [yYtT] ]] && CMAKE_OPTS+=" -DBUILD_REG_TESTING=ON -DCONTROLPATH=${CONTROLPATH:-}" # Re-use or create a new BUILD_DIR (Default: create new BUILD_DIR) -[[ ${BUILD_CLEAN:-"YES"} =~ [yYtT] ]] && rm -rf $BUILD_DIR +[[ ${BUILD_CLEAN:-"YES"} =~ [yYtT] ]] && rm -rf $BUILD_DIR && echo "Removing $BUILD_DIR" mkdir -p $BUILD_DIR && cd $BUILD_DIR # Configure, build, install diff --git a/ush/module-setup.sh b/ush/module-setup.sh index 469fd4a3c5..207dbba925 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -57,32 +57,8 @@ elif [[ $MACHINE_ID = gaea* ]] ; then # /etc/profile here. source /etc/profile __ms_source_etc_profile=yes - else - __ms_source_etc_profile=no - fi - module purge - # clean up after purge - unset _LMFILES_ - unset _LMFILES_000 - unset _LMFILES_001 - unset LOADEDMODULES - module load modules - if [[ -d /opt/cray/ari/modulefiles ]] ; then - module use -a /opt/cray/ari/modulefiles - fi - if [[ -d /opt/cray/pe/ari/modulefiles ]] ; then - module use -a /opt/cray/pe/ari/modulefiles - fi - if [[ -d /opt/cray/pe/craype/default/modulefiles ]] ; then - module use -a /opt/cray/pe/craype/default/modulefiles - fi - if [[ -s /etc/opt/cray/pe/admin-pe/site-config ]] ; then - source /etc/opt/cray/pe/admin-pe/site-config - fi - if [[ "$__ms_source_etc_profile" == yes ]] ; then - source /etc/profile - unset __ms_source_etc_profile fi + source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh elif [[ $MACHINE_ID = expanse* ]]; then # We are on SDSC Expanse diff --git a/ush/sub_gaea b/ush/sub_gaea new file mode 100755 index 0000000000..5c755e6930 --- /dev/null +++ b/ush/sub_gaea @@ -0,0 +1,166 @@ +#!/bin/sh --login +set -x +usage="\ +Usage: $0 [options] executable [args] + where the options are: + -a account account (default: none) + -b binding run smt binding or not (default:NO) + -d dirin initial directory (default: cwd) + -e envars copy comma-separated environment variables + -g group group name + -i append standard input to command file + -j jobname specify jobname (default: executable basename) + -m machine machine on which to run (default: current) + -n write command file to stdout rather than submitting it + -o output specify output file (default: jobname.out) + -p procs[/nodes[/ppreq] + number of MPI tasks and optional nodes or Bblocking and + ppreq option (N or S) (defaults: serial, Bunlimited, S) + -q queue[/qpreq] queue name and optional requirement, e.g. dev/P + (defaults: 1 if serial or dev if parallel and none) + (queue 3 or 4 is dev or prod with twice tasks over ip) + (options: P=parallel, B=bigmem, b=batch) + -r rmem[/rcpu] resources memory and cpus/task (default: '1024 mb', 1) + -t timew wall time limit in [[hh:]mm:]ss format (default: 900) + -u userid userid to run under (default: self) + -v verbose mode + -w when when to run, in yyyymmddhh[mm], +hh[mm], thh[mm], or + Thh[mm] (full, incremental, today or tomorrow) format + (default: now) +Function: This command submits a job to the batch queue." +subcmd="$*" +stdin=NO +nosub=NO +account="" +binding="NO" +dirin="" +envars="" +group="" +jobname="" +machine="" +output="" +procs=0 +nodes="" +ppreq="" +queue="" +qpreq="" +rmem="1024" +rcpu="1" +timew="900" +userid="" +verbose=NO +when="" +while getopts a:b:d:e:g:ij:m:no:p:q:r:t:u:vw: opt;do + case $opt in + a) account="$OPTARG";; + b) binding="$OPTARG";; + d) dirin="$OPTARG";; + e) envars="$OPTARG";; + g) group="$OPTARG";; + i) stdin=YES;; + j) jobname=$OPTARG;; + m) machine="$OPTARG";; + n) nosub=YES;; + o) output=$OPTARG;; + p) procs=$(echo $OPTARG/|cut -d/ -f1);nodes=$(echo $OPTARG/|cut -d/ -f2);ppreq=$(echo $OPTARG/|cut -d/ -f3);; + q) queue=$(echo $OPTARG/|cut -d/ -f1);qpreq=$(echo $OPTARG/|cut -d/ -f2);; + r) rmem=$(echo $OPTARG/|cut -d/ -f1);rcpu=$(echo $OPTARG/|cut -d/ -f2);; + t) timew=$OPTARG;; + u) userid=$OPTARG;; + v) verbose=YES;; + w) when=$OPTARG;; + \?) echo $0: invalid option >&2;echo "$usage" >&2;exit 1;; + esac +done +shift $(($OPTIND-1)) +if [[ $# -eq 0 ]];then + echo $0: missing executable name >&2;echo "$usage" >&2;exit 1 +fi +exec=$1 +if [[ ! -s $exec ]]&&which $exec >/dev/null 2>&1;then + exec=$(which $exec) +fi +shift +args="$*" +bn=$(basename $exec) +export jobname=${jobname:-$bn} +output=${output:-$jobname.out} +myuser=$LOGNAME +myhost=$(hostname) + +if [ -d /lustre/f2/scratch/$LOGNAME ]; then + DATA=/lustre/f2/scratch/$LOGNAME/tmp +fi +DATA=${DATA:-$ptmp/tmp} + +mkdir -p $DATA + +queue=${queue:-batch} +timew=${timew:-01:20:00} +task_node=${task_node:-$procs} +export size=$((nodes*task_node)) +echo "In sub_gaea: task_node, nodes, size=",$task_node,$nodes,$size +envars=$envars +threads=${rcpu:-1} + +export TZ=GMT +cfile=$DATA/sub$$ +> $cfile +echo "#!/bin/bash -l" >> $cfile +echo "" >> $cfile +echo "#SBATCH --output=$output" >> $cfile +echo "#SBATCH --job-name=$jobname" >> $cfile +echo "#SBATCH --qos=normal" >> $cfile +echo "#SBATCH --clusters=c4" >> $cfile +echo "#SBATCH --time=$timew" >> $cfile +echo "#SBATCH --nodes=$nodes --ntasks-per-node=$procs --cpus-per-task=$threads" >> $cfile +echo "#SBATCH --account=nggps_emc" >> $cfile +echo "#SBATCH --mem=0" >> $cfile + +echo "" >>$cfile +echo "export OMP_NUM_THREADS=$threads" >> $cfile +echo "" >>$cfile +echo ". "$(awk '{ print $1, $2, $3, $4, $5, $6, $7, $8, $9 }' $regdir/regression_var.out) >>$cfile +echo "" >>$cfile + +echo "source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh" >> $cfile +echo "module use $gsisrc/modulefiles" >> $cfile +echo "module load gsi_gaea" >> $cfile +echo "module list" >> $cfile +echo "" >>$cfile + +cat $exec >> $cfile + +if [[ $nosub = YES ]];then + cat $cfile + exit +elif [[ $verbose = YES ]];then + set -x + cat $cfile +fi + +if [[ $stdin = YES ]];then + cat +fi >>$cfile +if [[ $nosub = YES ]];then + cat $cfile + exit +elif [[ $verbose = YES ]];then + set -x + cat $cfile +fi +sbatch=${sbatch:-sbatch} + +ofile=$DATA/subout$$ +>$ofile +chmod 777 $ofile +$sbatch $cfile >$ofile +rc=$? +cat $ofile +if [[ -w $SUBLOG ]];then + jobn=$(grep -i submitted $ofile|head -n1|cut -d\" -f2) + date -u +"%Y%m%d%H%M%S : $subcmd : $jobn" >>$SUBLOG +fi +#rm $cfile $ofile +#[[ $MKDATA = YES ]] && rmdir $DATA +exit $rc From fe42c85e7e32785429bf69fcbc53fd1e3b70645e Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Mon, 26 Jun 2023 12:19:22 -0400 Subject: [PATCH 04/14] Intel 2022 for Gaea,update tests scripts for Gaea, Cheyenne, Orion, Jet --- regression/regression_driver.sh | 2 +- regression/regression_param.sh | 46 +++++---- regression/regression_var.sh | 20 ++-- ush/build.sh | 4 +- ush/sub_cheyenne | 169 ++++++++++++++++++++++++++++++++ ush/sub_gaea | 16 +-- ush/sub_jet | 52 ++-------- 7 files changed, 224 insertions(+), 85 deletions(-) create mode 100644 ush/sub_cheyenne diff --git a/regression/regression_driver.sh b/regression/regression_driver.sh index 58ebf64e85..96200c0363 100755 --- a/regression/regression_driver.sh +++ b/regression/regression_driver.sh @@ -42,7 +42,7 @@ for jn in `seq ${RSTART} ${REND}`; do fi rm -f ${job[$jn]}.out - /bin/sh $ush/$sub_cmd -q $queue -j ${job[$jn]} -t ${topts[$jn]} -p ${popts[$jn]} -r ${ropts[$jn]} $scripts/${regtest}.sh >& $ush/sub_cmd.${job[$jn]}.out + /bin/sh $ush/$sub_cmd -q $queue -j ${job[$jn]} -t ${topts[$jn]} -p ${popts[$jn]} -r ${ropts[$jn]} $scripts/${regtest}.sh if [ $debug == ".true." ]; then break; fi $scripts/regression_wait.sh ${job[$jn]} ${rcname} $check_resource diff --git a/regression/regression_param.sh b/regression/regression_param.sh index ea597fe0ae..79a91bde8e 100755 --- a/regression/regression_param.sh +++ b/regression/regression_param.sh @@ -33,7 +33,9 @@ case $machine in sub_cmd="sub_discover" ;; Cheyenne) - sub_cmd="sub_ncar -a p48503002 -q economy -d $PWD" + sub_cmd="sub_cheyenne" + memnode=128 + numcore=36 ;; *) # EXIT out for unresolved machine echo "unknown $machine" @@ -61,11 +63,11 @@ case $regtest in topts[1]="0:15:00" ; popts[1]="12/5/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="12/9/" ; ropts[2]="/2" elif [[ "$machine" = "Cheyenne" ]]; then - topts[1]="0:30:00" ; popts[1]="16/2/" ; ropts[1]="/1" - topts[2]="0:30:00" ; popts[2]="16/4/" ; ropts[2]="/2" + topts[1]="0:15:00" ; popts[1]="16/2/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="16/4/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then - topts[1]="0:30:00" ; popts[1]="18/2/" ; ropts[1]="/1" - topts[2]="0:30:00" ; popts[2]="18/4/" ; ropts[2]="/2" + topts[1]="0:15:00" ; popts[1]="18/2/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="18/4/" ; ropts[2]="/2" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:15:00" ; popts[1]="12/5/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="12/9/" ; ropts[2]="/2" @@ -94,11 +96,11 @@ case $regtest in topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2" elif [[ "$machine" = "Cheyenne" ]]; then - topts[1]="0:35:00" ; popts[1]="16/2/" ; ropts[1]="/1" - topts[2]="0:25:00" ; popts[2]="16/4/" ; ropts[2]="/2" + topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then - topts[1]="0:35:00" ; popts[1]="18/2/" ; ropts[1]="/1" - topts[2]="0:25:00" ; popts[2]="18/4/" ; ropts[2]="/2" + topts[1]="0:15:00" ; popts[1]="18/2/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="18/4/" ; ropts[2]="/2" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:15:00" ; popts[1]="28/2/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="28/4/" ; ropts[2]="/2" @@ -142,8 +144,8 @@ case $regtest in topts[1]="0:30:00" ; popts[1]="48/2" ; ropts[1]="/1" topts[2]="0:30:00" ; popts[2]="60/3" ; ropts[2]="/2" elif [[ "$machine" = "Cheyenne" ]]; then - topts[1]="1:59:00" ; popts[1]="6/8/" ; ropts[1]="/1" - topts[2]="0:35:00" ; popts[2]="6/10/" ; ropts[2]="/2" + topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1" + topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then topts[1]="0:10:00" ; popts[1]="18/8/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="18/10/" ; ropts[2]="/2" @@ -171,6 +173,9 @@ case $regtest in elif [[ "$machine" = "Jet" ]]; then topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" + elif [[ "$machine" = "Cheyenne" ]]; then + topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1" @@ -198,6 +203,9 @@ case $regtest in elif [[ "$machine" = "Jet" ]]; then topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1" + elif [[ "$machine" = "Cheyenne" ]]; then + topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1" @@ -226,8 +234,8 @@ case $regtest in topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1" topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1" elif [[ "$machine" = "Cheyenne" ]]; then - topts[1]="0:15:00" ; popts[1]="8/6/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="8/8/" ; ropts[2]="/1" + topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1" + topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1" elif [[ "$machine" = "Gaea" ]]; then topts[1]="0:30:00" ; popts[1]="8/6/" ; ropts[1]="/1" topts[2]="0:30:00" ; popts[2]="8/8/" ; ropts[2]="/1" @@ -286,8 +294,8 @@ case $regtest in topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2" elif [[ "$machine" = "Cheyenne" ]]; then - topts[1]="0:15:00" ; popts[1]="16/2/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="16/4/" ; ropts[2]="/2" + topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2" @@ -341,25 +349,25 @@ if [[ "$machine" = "Hera" ]]; then export APRUN="srun" elif [[ "$machine" = "Orion" ]]; then export OMP_STACKSIZE=2048M - export APRUN="srun -n \$ntasks" + export APRUN="srun --label -n \$ntasks" elif [[ "$machine" = "Jet" ]]; then export OMP_STACKSIZE=1024M export MPI_BUFS_PER_PROC=256 export MPI_BUFS_PER_HOST=256 export MPI_GROUP_MAX=256 - export APRUN="srun" + export APRUN="srun --label -n \$ntasks --cpus-per-task=\$threads" elif [[ "$machine" = "Gaea" ]]; then export OMP_STACKSIZE=1024M export MPI_BUFS_PER_PROC=256 export MPI_BUFS_PER_HOST=256 export MPI_GROUP_MAX=256 - export APRUN="srun --export=ALL --mpi=pmi2 -n \$size" + export APRUN="srun --export=ALL --mpi=pmi2 -n \$ntasks" elif [[ "$machine" = "Cheyenne" ]]; then export OMP_STACKSIZE=1024M export MPI_BUFS_PER_PROC=256 export MPI_BUFS_PER_HOST=256 export MPI_GROUP_MAX=256 - export APRUN="mpirun -v -np \$NCPUS" + export APRUN="mpirun -v -np \$ntasks" elif [[ "$machine" = "wcoss2" ]]; then export OMP_PLACES=cores export OMP_STACKSIZE=2G diff --git a/regression/regression_var.sh b/regression/regression_var.sh index e883a5b1c8..ed80739ae6 100755 --- a/regression/regression_var.sh +++ b/regression/regression_var.sh @@ -64,18 +64,18 @@ case $machine in export accnt="nggps_emc" ;; Cheyenne) - export queue="economy" - export noscrub="/glade/scratch/$LOGNAME" + export queue="regular" + export noscrub="/glade/scratch/$LOGNAME/tmp_gsi/noscrub" export group="global" if [[ "$cmaketest" = "false" ]]; then - export basedir="/glade/scratch/$LOGNAME/gsi" + export basedir="/glade/scratch/$LOGNAME/tmp_gsi" fi - export ptmp="/glade/scratch/$LOGNAME/$ptmpName" + export ptmp="/glade/scratch/$LOGNAME/tmp_gsi/ptmp" - export casesdir="/glade/p/ral/jntp/tools/CASES" + export casesdir="/glade/work/epicufsrt/contrib/GSI_data/CASES/regtest" export check_resource="no" - export accnt="p48503002" + export accnt="NRAL0032" ;; wcoss2) export local_or_default="${local_or_default:-/lfs/h2/emc/da/noscrub/$LOGNAME}" @@ -98,7 +98,7 @@ case $machine in export accnt="${accnt:-GFS-DEV}" ;; Orion) - export local_or_default="${local_or_default:-/work/noaa/da/$LOGNAME}" + export local_or_default="${local_or_default:-/work/noaa/epic-ps/$LOGNAME}" if [ -d $local_or_default ]; then export noscrub="$local_or_default/noscrub" elif [ -d /work/noaa/global/$LOGNAME ]; then @@ -108,14 +108,14 @@ case $machine in export queue="${queue:-batch}" export group="${group:-global}" if [[ "$cmaketest" = "false" ]]; then - export basedir="/work/noaa/da/$LOGNAME/gsi" + export basedir="/work/noaa/epic-ps/$LOGNAME/gsi" fi - export ptmp="${ptmp:-/work/noaa/stmp/$LOGNAME/$ptmpName}" + export ptmp="${ptmp:-/work/noaa/epic-ps/$LOGNAME/ptmp}" export casesdir="/work/noaa/da/rtreadon/CASES/regtest" export check_resource="no" - export accnt="${accnt:-da-cpu}" + export accnt="${accnt:-epic}" ;; Hera) diff --git a/ush/build.sh b/ush/build.sh index f706f413c1..c7f27b61e2 100755 --- a/ush/build.sh +++ b/ush/build.sh @@ -30,8 +30,6 @@ set -x # Set CONTROLPATH variable to user develop installation CONTROLPATH="$DIR_ROOT/../develop/install/bin" -CMAKELIBS=".so .a" -CMAKE_OPTS+=" -DCMAKE_FIND_LIBRARY_SUFFIXES=${CMAKELIBS}" # Collect BUILD Options CMAKE_OPTS+=" -DCMAKE_BUILD_TYPE=$BUILD_TYPE" @@ -45,7 +43,7 @@ CMAKE_OPTS+=" -DGSI_MODE=$GSI_MODE -DENKF_MODE=${ENKF_MODE}" [[ ${REGRESSION_TESTS} =~ [yYtT] ]] && CMAKE_OPTS+=" -DBUILD_REG_TESTING=ON -DCONTROLPATH=${CONTROLPATH:-}" # Re-use or create a new BUILD_DIR (Default: create new BUILD_DIR) -[[ ${BUILD_CLEAN:-"YES"} =~ [yYtT] ]] && rm -rf $BUILD_DIR && echo "Removing $BUILD_DIR" +[[ ${BUILD_CLEAN:-"YES"} =~ [yYtT] ]] && rm -rf $BUILD_DIR mkdir -p $BUILD_DIR && cd $BUILD_DIR # Configure, build, install diff --git a/ush/sub_cheyenne b/ush/sub_cheyenne new file mode 100644 index 0000000000..cb671ce8c1 --- /dev/null +++ b/ush/sub_cheyenne @@ -0,0 +1,169 @@ +#!/bin/sh --login +set -x +echo "starting sub_cheyenne" +usage="\ +Usage: $0 [options] executable [args] + where the options are: + -a account account (default: none) + -b binding run smt binding or not (default:NO) + -d dirin initial directory (default: cwd) + -e envars copy comma-separated environment variables + -g group group name + -i append standard input to command file + -j jobname specify jobname (default: executable basename) + -m machine machine on which to run (default: current) + -n write command file to stdout rather than submitting it + -o output specify output file (default: jobname.out) + -p procs[/nodes[/ppreq] + number of MPI tasks and optional nodes or Bblocking and + ppreq option (N or S) (defaults: serial, Bunlimited, S) + -q queue[/qpreq] queue name and optional requirement, e.g. dev/P + (defaults: 1 if serial or dev if parallel and none) + (queue 3 or 4 is dev or prod with twice tasks over ip) + (options: P=parallel, B=bigmem, b=batch) + -r rmem[/rcpu] resources memory and cpus/task (default: '1024 mb', 1) + -t timew wall time limit in [[hh:]mm:]ss format (default: 900) + -u userid userid to run under (default: self) + -v verbose mode + -w when when to run, in yyyymmddhh[mm], +hh[mm], thh[mm], or + Thh[mm] (full, incremental, today or tomorrow) format + (default: now) +Function: This command submits a job to the batch queue." +subcmd="$*" +stdin=NO +nosub=NO +account="" +binding="NO" +dirin="" +envars="" +group="" +jobname="" +machine="" +output="" +procs=0 +nodes="" +ppreq="" +queue="" +qpreq="" +rmem="1024" +rcpu="1" +timew="900" +userid="" +verbose=NO +when="" +while getopts a:b:d:e:g:ij:m:no:p:q:r:t:u:vw: opt;do + case $opt in + a) account="$OPTARG";; + b) binding="$OPTARG";; + d) dirin="$OPTARG";; + e) envars="$OPTARG";; + g) group="$OPTARG";; + i) stdin=YES;; + j) jobname=$OPTARG;; + m) machine="$OPTARG";; + n) nosub=YES;; + o) output=$OPTARG;; + p) procs=$(echo $OPTARG/|cut -d/ -f1);nodes=$(echo $OPTARG/|cut -d/ -f2);ppreq=$(echo $OPTARG/|cut -d/ -f3);; + q) queue=$(echo $OPTARG/|cut -d/ -f1);qpreq=$(echo $OPTARG/|cut -d/ -f2);; + r) rmem=$(echo $OPTARG/|cut -d/ -f1);rcpu=$(echo $OPTARG/|cut -d/ -f2);; + t) timew=$OPTARG;; + u) userid=$OPTARG;; + v) verbose=YES;; + w) when=$OPTARG;; + \?) echo $0: invalid option >&2;echo "$usage" >&2;exit 1;; + esac +done +shift $(($OPTIND-1)) +if [[ $# -eq 0 ]];then + echo $0: missing executable name >&2;echo "$usage" >&2;exit 1 +fi +exec=$1 +if [[ ! -s $exec ]]&&which $exec >/dev/null 2>&1;then + exec=$(which $exec) +fi +shift +args="$*" +bn=$(basename $exec) +export jobname=${jobname:-$bn} +output=${output:-$jobname.out} +myuser=$LOGNAME +myhost=$(hostname) + +DATA=/glade/scratch/$LOGNAME/tmp +mkdir -p $DATA + +timew=${timew:-01:20:00} +task_node=${task_node:-$procs} +size=$((nodes*task_node)) +envars=$envars +threads=${rcpu:-1} + +export TZ=GMT +cfile=$DATA/sub$$ +> $cfile +echo "#!/bin/sh --login" >> $cfile +echo "" >> $cfile +echo "#PBS -o $output" >> $cfile +echo "#PBS -N $jobname" >> $cfile +echo "#PBS -q $queue" >> $cfile +echo "#PBS -l walltime=$timew" >> $cfile +echo "#PBS -l select=$nodes:ncpus=$procs:mpiprocs=$procs" >> $cfile +echo "#PBS -j oe" >> $cfile +echo "#PBS -A $accnt" >> $cfile +echo "#PBS -V" >> $cfile + +echo "" >>$cfile +echo "export ntasks=$(( $nodes * $procs ))" >> $cfile +echo "export ppn=$procs" >> $cfile +echo "export threads=$threads" >> $cfile +echo "export OMP_NUM_THREADS=$threads" >> $cfile +echo "ulimit -s unlimited" >> $cfile +echo "" >>$cfile +echo ". "$(awk '{ print $1, $2, $3, $4, $5, $6, $7, $8, $9 }' $regdir/regression_var.out) >>$cfile +echo "" >>$cfile + +echo "cfile = $cfile" +echo "source /glade/u/apps/ch/modulefiles/default/localinit/localinit.sh >> $cfile" +echo "module purge" >> $cfile +echo "module use $gsisrc/modulefiles" >> $cfile +echo "module load gsi_cheyenne.intel" >> $cfile +echo "module list" >> $cfile + +cat $exec >> $cfile + +if [[ $nosub = YES ]];then + cat $cfile + exit +elif [[ $verbose = YES ]];then + set -x + cat $cfile +fi + + +if [[ $stdin = YES ]];then + cat +fi >>$cfile +if [[ $nosub = YES ]];then + cat $cfile + exit +elif [[ $verbose = YES ]];then + set -x + cat $cfile +fi +qsub=${qsub:-qsub} + +ofile=$DATA/subout$$ +>$ofile +chmod 777 $ofile +$qsub $cfile >$ofile +rc=$? +cat $ofile +if [[ -w $SUBLOG ]];then + jobn=$(grep -i submitted $ofile|head -n1|cut -d\" -f2) + date -u +"%Y%m%d%H%M%S : $subcmd : $jobn" >>$SUBLOG +fi +rm $cfile $ofile +[[ $MKDATA = YES ]] && rmdir $DATA +echo "ending sub_cheyenne" +exit $rc + diff --git a/ush/sub_gaea b/ush/sub_gaea index 5c755e6930..3a9526b0d1 100755 --- a/ush/sub_gaea +++ b/ush/sub_gaea @@ -98,8 +98,7 @@ mkdir -p $DATA queue=${queue:-batch} timew=${timew:-01:20:00} task_node=${task_node:-$procs} -export size=$((nodes*task_node)) -echo "In sub_gaea: task_node, nodes, size=",$task_node,$nodes,$size +size=$((nodes*task_node)) envars=$envars threads=${rcpu:-1} @@ -114,11 +113,16 @@ echo "#SBATCH --qos=normal" echo "#SBATCH --clusters=c4" >> $cfile echo "#SBATCH --time=$timew" >> $cfile echo "#SBATCH --nodes=$nodes --ntasks-per-node=$procs --cpus-per-task=$threads" >> $cfile -echo "#SBATCH --account=nggps_emc" >> $cfile +echo "#SBATCH --account=$accnt" >> $cfile echo "#SBATCH --mem=0" >> $cfile echo "" >>$cfile +echo "export ntasks=$(( $nodes * $procs ))" >> $cfile +echo "export ppn=$procs" >> $cfile +echo "export threads=$threads" >> $cfile echo "export OMP_NUM_THREADS=$threads" >> $cfile +echo "ulimit -s unlimited" >> $cfile + echo "" >>$cfile echo ". "$(awk '{ print $1, $2, $3, $4, $5, $6, $7, $8, $9 }' $regdir/regression_var.out) >>$cfile echo "" >>$cfile @@ -154,13 +158,13 @@ sbatch=${sbatch:-sbatch} ofile=$DATA/subout$$ >$ofile chmod 777 $ofile -$sbatch $cfile >$ofile +$sbatch --export=ALL $cfile >$ofile rc=$? cat $ofile if [[ -w $SUBLOG ]];then jobn=$(grep -i submitted $ofile|head -n1|cut -d\" -f2) date -u +"%Y%m%d%H%M%S : $subcmd : $jobn" >>$SUBLOG fi -#rm $cfile $ofile -#[[ $MKDATA = YES ]] && rmdir $DATA +rm $cfile $ofile +[[ $MKDATA = YES ]] && rmdir $DATA exit $rc diff --git a/ush/sub_jet b/ush/sub_jet index e11be1280c..e394a0c09e 100755 --- a/ush/sub_jet +++ b/ush/sub_jet @@ -98,16 +98,10 @@ task_node=${task_node:-$procs} size=$((nodes*task_node)) envars=$envars threads=${rcpu:-1} -#envars=$envars,mpi_tasks=$procs -#Options -###PBS -l partition=c1ms,size=0528,walltime=01:20:00 -##PBS -l partition=$queue,size=$size,walltime=$timew -##PBS -S /bin/sh export TZ=GMT cfile=$DATA/sub$$ > $cfile -#echo "#PBS -S /bin/sh" >> $cfile echo "#!/bin/sh --login" >> $cfile echo "" >> $cfile echo "#SBATCH --output=$output" >> $cfile @@ -115,17 +109,17 @@ echo "#SBATCH --job-name=$jobname" echo "#SBATCH --qos=$queue" >> $cfile echo "#SBATCH --time=$timew" >> $cfile echo "#SBATCH --nodes=$nodes --ntasks-per-node=$procs --cpus-per-task=$threads" >> $cfile -#echo "#SBATCH -j oe" >> $cfile echo "#SBATCH --account=$accnt" >> $cfile echo "#SBATCH --mem=0" >> $cfile -echo "#SBATCH --partition=kjet" >> $cfile -#echo "#SBATCH -V" >> $cfile -#echo "#PBS -d" >> $cfile -#. $exec >> $cfile -#echo "/bin/sh -x $exec" >> $cfile +echo "#SBATCH --partition=xjet" >> $cfile echo "" >>$cfile +echo "export ntasks=$(( $nodes * $procs ))" >> $cfile +echo "export ppn=$procs" >> $cfile +echo "export threads=$threads" >> $cfile echo "export OMP_NUM_THREADS=$threads" >> $cfile +echo "ulimit -s unlimited" >> $cfile + echo "" >>$cfile echo ". "$(awk '{ print $1, $2, $3, $4, $5, $6, $7, $8, $9 }' $regdir/regression_var.out) >>$cfile echo "" >>$cfile @@ -146,40 +140,6 @@ elif [[ $verbose = YES ]];then set -x cat $cfile fi -#msub -I partition=$partition,size=$procs,walltime=$walltime $cfile - -#if [[ -n $when ]];then -# whena=$when -# if [[ $when = +* ]];then -# hr=$(echo $when|cut -c2-3) -# mn=$(echo $when|cut -c4-5) -# [[ -n $mn ]] || mn=00 -# now=$(date -u +"%Y%m%d%H%M") -# ((mn+=$(echo $now|cut -c11-12))) -# [[ $mn -ge 60 ]] && ((hr+=1)) && ((mn-=60)) -# [[ $mn -lt 10 ]] && mn=0$mn -# whena=$(/nwprod/util/exec/ndate +$hr $(echo $now|cut -c1-10))$mn -# elif [[ $when = t* ]];then -# hr=$(echo $when|cut -c2-3) -# mn=$(echo $when|cut -c4-5) -# [[ -n $mn ]] || mn=00 -# now=$(date -u +"%Y%m%d") -# whena=$now$hr$mn -# elif [[ $when = T* ]];then -# hr=$(echo $when|cut -c2-3) -# mn=$(echo $when|cut -c4-5) -# [[ -n $mn ]] || mn=00 -# now=$(date -u +"%Y%m%d%H") -# whena=$(/nwprod/util/exec/ndate +24 $now|cut -c1-8)$hr$mn -# fi -# yr=$(echo $whena|cut -c1-4) -# mo=$(echo $whena|cut -c5-6) -# dy=$(echo $whena|cut -c7-8) -# hr=$(echo $whena|cut -c9-10) -# mn=$(echo $whena|cut -c11-12) -# [[ -n $mn ]] || mn=00 -# echo "#@ startdate = $mo/$dy/$yr $hr:$mn" -#fi >>$cfile if [[ $stdin = YES ]];then From 3baf6c795ce776f9bf3b2a73a9be66e30fba0928 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Mon, 26 Jun 2023 12:51:23 -0400 Subject: [PATCH 05/14] update Cheyenne modulefile --- modulefiles/gsi_cheyenne.intel.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modulefiles/gsi_cheyenne.intel.lua b/modulefiles/gsi_cheyenne.intel.lua index d02a0c3885..4a3525bca1 100644 --- a/modulefiles/gsi_cheyenne.intel.lua +++ b/modulefiles/gsi_cheyenne.intel.lua @@ -8,7 +8,7 @@ load("intel/2022.1") load("mpt/2.25") load("ncarcompilers/0.5.0") -prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/hpc-stack/intel2022.1") +prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/hpc-stack/intel2022.1/modulefiles/stack") load("hpc/1.2.0") load("hpc-intel/2022.1") @@ -18,7 +18,7 @@ load("mkl/2022.1") load("gsi_common") load(pathJoin("prod_util", os.getenv("prod_util_ver") or "1.2.2")) -pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_data/fix") +pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_data/fix/20230601") pushenv("CFLAGS", "-xHOST") pushenv("FFLAGS", "-xHOST") From d3fe9a273f75990bc9514c701bed4883d35a2328 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Mon, 26 Jun 2023 13:45:24 -0400 Subject: [PATCH 06/14] updatesto scripts to address comments --- regression/regression_var.sh | 14 +++++++------- ush/sub_jet | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/regression/regression_var.sh b/regression/regression_var.sh index ed80739ae6..a3fe9e4e9d 100755 --- a/regression/regression_var.sh +++ b/regression/regression_var.sh @@ -65,12 +65,12 @@ case $machine in ;; Cheyenne) export queue="regular" - export noscrub="/glade/scratch/$LOGNAME/tmp_gsi/noscrub" + export noscrub="/glade/scratch/$LOGNAME/noscrub" export group="global" if [[ "$cmaketest" = "false" ]]; then - export basedir="/glade/scratch/$LOGNAME/tmp_gsi" + export basedir="/glade/scratch/$LOGNAME" fi - export ptmp="/glade/scratch/$LOGNAME/tmp_gsi/ptmp" + export ptmp="/glade/scratch/$LOGNAME/$ptmpName" export casesdir="/glade/work/epicufsrt/contrib/GSI_data/CASES/regtest" @@ -98,7 +98,7 @@ case $machine in export accnt="${accnt:-GFS-DEV}" ;; Orion) - export local_or_default="${local_or_default:-/work/noaa/epic-ps/$LOGNAME}" + export local_or_default="${local_or_default:-/work/noaa/da/$LOGNAME}" if [ -d $local_or_default ]; then export noscrub="$local_or_default/noscrub" elif [ -d /work/noaa/global/$LOGNAME ]; then @@ -108,14 +108,14 @@ case $machine in export queue="${queue:-batch}" export group="${group:-global}" if [[ "$cmaketest" = "false" ]]; then - export basedir="/work/noaa/epic-ps/$LOGNAME/gsi" + export basedir="/work/noaa/da/$LOGNAME/gsi" fi - export ptmp="${ptmp:-/work/noaa/epic-ps/$LOGNAME/ptmp}" + export ptmp="${ptmp:-/work/noaa/stmp/$LOGNAME/$ptmpName}" export casesdir="/work/noaa/da/rtreadon/CASES/regtest" export check_resource="no" - export accnt="${accnt:-epic}" + export accnt="${accnt:-da-cpu}" ;; Hera) diff --git a/ush/sub_jet b/ush/sub_jet index e394a0c09e..e45dcc0b3c 100755 --- a/ush/sub_jet +++ b/ush/sub_jet @@ -111,7 +111,7 @@ echo "#SBATCH --time=$timew" echo "#SBATCH --nodes=$nodes --ntasks-per-node=$procs --cpus-per-task=$threads" >> $cfile echo "#SBATCH --account=$accnt" >> $cfile echo "#SBATCH --mem=0" >> $cfile -echo "#SBATCH --partition=xjet" >> $cfile +echo "#SBATCH --partition=kjet" >> $cfile echo "" >>$cfile echo "export ntasks=$(( $nodes * $procs ))" >> $cfile From 7bd5f95f278deceea3c7b7118c5293c591a059fe Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Mon, 26 Jun 2023 13:48:09 -0400 Subject: [PATCH 07/14] update regression/regression_driver.sh --- regression/regression_driver.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression/regression_driver.sh b/regression/regression_driver.sh index 96200c0363..b818089cad 100755 --- a/regression/regression_driver.sh +++ b/regression/regression_driver.sh @@ -42,7 +42,7 @@ for jn in `seq ${RSTART} ${REND}`; do fi rm -f ${job[$jn]}.out - /bin/sh $ush/$sub_cmd -q $queue -j ${job[$jn]} -t ${topts[$jn]} -p ${popts[$jn]} -r ${ropts[$jn]} $scripts/${regtest}.sh + /bin/sh $ush/$sub_cmd -q $queue -j ${job[$jn]} -t ${topts[$jn]} -p ${popts[$jn]} -r ${ropts[$jn]} $scripts/${regtest}.sh if [ $debug == ".true." ]; then break; fi $scripts/regression_wait.sh ${job[$jn]} ${rcname} $check_resource From e50cf96f927ead5ff5cc58f6186e26a6a9dec904 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Mon, 26 Jun 2023 13:49:22 -0400 Subject: [PATCH 08/14] update regression/regression_driver.sh --- regression/regression_driver.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/regression/regression_driver.sh b/regression/regression_driver.sh index b818089cad..e1d3b18dc7 100755 --- a/regression/regression_driver.sh +++ b/regression/regression_driver.sh @@ -10,7 +10,6 @@ if [ -d "$config_path" ]; then source $config_path/local_vars.sh fi - # source the necessary files to setup if [ "$#" -eq 2 ]; then export regdir=$2 From a0e48e2183bf08d5622eb3bbaf7b5e17c5c9eb26 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Mon, 26 Jun 2023 13:52:49 -0400 Subject: [PATCH 09/14] update ush/build.sh --- ush/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/build.sh b/ush/build.sh index c7f27b61e2..9a280c4e55 100755 --- a/ush/build.sh +++ b/ush/build.sh @@ -34,7 +34,7 @@ CONTROLPATH="$DIR_ROOT/../develop/install/bin" CMAKE_OPTS+=" -DCMAKE_BUILD_TYPE=$BUILD_TYPE" # Install destination for built executables, libraries, CMake Package config -CMAKE_OPTS+=" -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}" +CMAKE_OPTS+=" -DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX" # Configure for GSI and EnKF CMAKE_OPTS+=" -DGSI_MODE=$GSI_MODE -DENKF_MODE=${ENKF_MODE}" From 1d8beed1b8d55cb88c723267a58df191a812a141 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Tue, 27 Jun 2023 10:23:01 -0400 Subject: [PATCH 10/14] fixed typ inmodulefile for gaea --- modulefiles/gsi_gaea.lua | 3 --- 1 file changed, 3 deletions(-) diff --git a/modulefiles/gsi_gaea.lua b/modulefiles/gsi_gaea.lua index 4744e55c73..f76c8f3ad9 100644 --- a/modulefiles/gsi_gaea.lua +++ b/modulefiles/gsi_gaea.lua @@ -31,7 +31,4 @@ setenv("CXX","CC") pushenv("CRAYPE_LINK_TYPE","dynamic") whatis("Description: GSI environment on Gaea with Intel Compilers") -<<<<<<< HEAD -======= ->>>>>>> ming/intel2022 From e2398c7d542b5014311bf5792a0be6c25b8690b5 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Wed, 28 Jun 2023 00:12:12 -0400 Subject: [PATCH 11/14] ncdiag/1.1.1 for spack build --- ci/spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/spack.yaml b/ci/spack.yaml index 4374d8dd85..5d97fd48a0 100644 --- a/ci/spack.yaml +++ b/ci/spack.yaml @@ -19,7 +19,7 @@ spack: - wrf-io@1.2.0 - ncio@1.1.2 - crtm@2.4.0 - - gsi-ncdiag@1.1.0 + - gsi-ncdiag@1.1.1 view: true concretizer: unify: true From 164126b779a9e6af300af8110aa2e81152bd0106 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Wed, 28 Jun 2023 08:24:59 -0400 Subject: [PATCH 12/14] update ncdiag/1.1.1 --- modulefiles/gsi_gaea.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulefiles/gsi_gaea.lua b/modulefiles/gsi_gaea.lua index f76c8f3ad9..25c765cab1 100644 --- a/modulefiles/gsi_gaea.lua +++ b/modulefiles/gsi_gaea.lua @@ -3,7 +3,7 @@ help([[ load("cmake/3.20.1") -prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2/modulefiles/stack") +prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2_ncdf492/modulefiles/stack") load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) load(pathJoin("intel-classic", os.getenv("intel_classic_ver") or "2022.0.2")) From 8d6009b15b3bac7129a0c9402c37e382f9b0cb49 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Wed, 28 Jun 2023 09:39:20 -0400 Subject: [PATCH 13/14] gaea modulefile update --- modulefiles/gsi_gaea.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulefiles/gsi_gaea.lua b/modulefiles/gsi_gaea.lua index 25c765cab1..f76c8f3ad9 100644 --- a/modulefiles/gsi_gaea.lua +++ b/modulefiles/gsi_gaea.lua @@ -3,7 +3,7 @@ help([[ load("cmake/3.20.1") -prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2_ncdf492/modulefiles/stack") +prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2/modulefiles/stack") load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) load(pathJoin("intel-classic", os.getenv("intel_classic_ver") or "2022.0.2")) From 29a1bb66d4c6de186e2a9031c789214963df4a71 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Wed, 28 Jun 2023 09:45:31 -0400 Subject: [PATCH 14/14] updated test scripts for gaea --- regression/regression_var.sh | 2 +- ush/sub_gaea | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/regression/regression_var.sh b/regression/regression_var.sh index a3fe9e4e9d..d94e53a04d 100755 --- a/regression/regression_var.sh +++ b/regression/regression_var.sh @@ -50,7 +50,7 @@ echo "Running Regression Tests on '$machine'"; case $machine in Gaea) - export queue="batch" + export queue="normal" export noscrub="/lustre/f2/scratch/$LOGNAME/gsi_tmp/noscrub" export ptmp="/lustre/f2/scratch/$LOGNAME/gsi_tmp/ptmp" export casesdir="/lustre/f2/dev/role.epic/contrib/GSI_data/CASES/regtest" diff --git a/ush/sub_gaea b/ush/sub_gaea index 3a9526b0d1..b00e547a6a 100755 --- a/ush/sub_gaea +++ b/ush/sub_gaea @@ -109,7 +109,7 @@ echo "#!/bin/bash -l" echo "" >> $cfile echo "#SBATCH --output=$output" >> $cfile echo "#SBATCH --job-name=$jobname" >> $cfile -echo "#SBATCH --qos=normal" >> $cfile +echo "#SBATCH --qos=$queue" >> $cfile echo "#SBATCH --clusters=c4" >> $cfile echo "#SBATCH --time=$timew" >> $cfile echo "#SBATCH --nodes=$nodes --ntasks-per-node=$procs --cpus-per-task=$threads" >> $cfile