Merge pull request NOAA-EMC#11 from natalie-perlin/intel2022_gaea

Gaea intel 2022 update, reg. tests updates for Jet,Hera,Orion,Cheyenne,Gaea
RussTreadon-NOAA · Jun 29, 2023 · c077a1c · c077a1c
2 parents 06772e5 + 29a1bb6
commit c077a1c
Show file tree

Hide file tree

Showing 11 changed files with 433 additions and 70 deletions.
diff --git a/ci/spack.yaml b/ci/spack.yaml
@@ -19,7 +19,7 @@ spack:
   - wrf-io@1.2.0
   - ncio@1.1.2
   - crtm@2.4.0
-  - gsi-ncdiag@1.1.0
+  - gsi-ncdiag@1.1.1
   view: true
   concretizer:
     unify: true
diff --git a/modulefiles/gsi_cheyenne.intel.lua b/modulefiles/gsi_cheyenne.intel.lua
@@ -8,7 +8,7 @@ load("intel/2022.1")
 load("mpt/2.25")
 load("ncarcompilers/0.5.0")
 
-prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/hpc-stack/intel2022.1")
+prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/hpc-stack/intel2022.1/modulefiles/stack")
 
 load("hpc/1.2.0")
 load("hpc-intel/2022.1")
@@ -18,7 +18,7 @@ load("mkl/2022.1")
 load("gsi_common")
 
 load(pathJoin("prod_util", os.getenv("prod_util_ver") or "1.2.2"))
-pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_data/fix")
+pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_data/fix/20230601")
 
 pushenv("CFLAGS", "-xHOST")
 pushenv("FFLAGS", "-xHOST")

diff --git a/modulefiles/gsi_common.lua b/modulefiles/gsi_common.lua
@@ -15,7 +15,7 @@ local nemsio_ver=os.getenv("nemsio_ver") or "2.5.4"
 local wrf_io_ver=os.getenv("wrf_io_ver") or "1.2.0"
 local ncio_ver=os.getenv("ncio_ver") or "1.1.2"
 local crtm_ver=os.getenv("crtm_ver") or "2.4.0"
-local ncdiag_ver=os.getenv("ncdiag_ver") or "1.1.0"
+local ncdiag_ver=os.getenv("ncdiag_ver") or "1.1.1"
 
 load(pathJoin("netcdf", netcdf_ver))
 

diff --git a/modulefiles/gsi_gaea.lua b/modulefiles/gsi_gaea.lua
@@ -16,14 +16,19 @@ load("gsi_common")
 local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2"
 load(pathJoin("prod_util", prod_util_ver))
 
+-- Needed at runtime:
+load("alps")
+
 local MKLROOT="/opt/intel/oneapi/mkl/2022.0.2/"
 prepend_path("LD_LIBRARY_PATH",pathJoin(MKLROOT,"lib/intel64"))
 pushenv("MKLROOT", MKLROOT)
 
-pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix")
+pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix/20230601")
+
 setenv("CC","cc")
 setenv("FC","ftn")
 setenv("CXX","CC")
 pushenv("CRAYPE_LINK_TYPE","dynamic")
 
 whatis("Description: GSI environment on Gaea with Intel Compilers")
+
diff --git a/regression/regression_param.sh b/regression/regression_param.sh
@@ -18,6 +18,11 @@ case $machine in
 	   sub_cmd="sub_jet"
            memnode=96
            numcore=40
+    ;;
+	Gaea)
+	   sub_cmd="sub_gaea"
+           memnode=64
+           numcore=36
     ;;
 	wcoss2)
 	   sub_cmd="sub_wcoss2"
@@ -28,7 +33,9 @@ case $machine in
 	   sub_cmd="sub_discover"
     ;;
 	Cheyenne)
-	   sub_cmd="sub_ncar -a p48503002 -q economy -d $PWD"
+           sub_cmd="sub_cheyenne"
+           memnode=128
+           numcore=36
     ;;
     *) # EXIT out for unresolved machine
         echo "unknown $machine"
@@ -56,8 +63,11 @@ case $regtest in
            topts[1]="0:15:00" ; popts[1]="12/5/" ; ropts[1]="/1"
            topts[2]="0:15:00" ; popts[2]="12/9/" ; ropts[2]="/2"
         elif [[ "$machine" = "Cheyenne" ]]; then
-           topts[1]="0:30:00" ; popts[1]="16/2/" ; ropts[1]="/1"
-           topts[2]="0:30:00" ; popts[2]="16/4/" ; ropts[2]="/2"
+           topts[1]="0:15:00" ; popts[1]="16/2/" ; ropts[1]="/1"
+           topts[2]="0:15:00" ; popts[2]="16/4/" ; ropts[2]="/2"
+        elif [[ "$machine" = "Gaea" ]]; then
+           topts[1]="0:15:00" ; popts[1]="18/2/" ; ropts[1]="/1"
+           topts[2]="0:15:00" ; popts[2]="18/4/" ; ropts[2]="/2"
         elif [[ "$machine" = "wcoss2" ]]; then
            topts[1]="0:15:00" ; popts[1]="12/5/" ; ropts[1]="/1"
            topts[2]="0:15:00" ; popts[2]="12/9/" ; ropts[2]="/2"
@@ -86,8 +96,11 @@ case $regtest in
             topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1"
             topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2"
         elif [[ "$machine" = "Cheyenne" ]]; then
-            topts[1]="0:35:00" ; popts[1]="16/2/" ; ropts[1]="/1"
-            topts[2]="0:25:00" ; popts[2]="16/4/" ; ropts[2]="/2"
+            topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1"
+            topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2"
+        elif [[ "$machine" = "Gaea" ]]; then
+            topts[1]="0:15:00" ; popts[1]="18/2/" ; ropts[1]="/1"
+            topts[2]="0:15:00" ; popts[2]="18/4/" ; ropts[2]="/2"
         elif [[ "$machine" = "wcoss2" ]]; then
            topts[1]="0:15:00" ; popts[1]="28/2/" ; ropts[1]="/1"
            topts[2]="0:15:00" ; popts[2]="28/4/" ; ropts[2]="/2"
@@ -104,6 +117,8 @@ case $regtest in
               popts[1]="12/5/"
            elif [[ "$machine" = "Jet" ]]; then
               popts[1]="12/5/"
+           elif [[ "$machine" = "Gaea" ]]; then
+              popts[1]="18/5/"
            elif [[ "$machine" = "wcoss2" ]]; then
               popts[1]="28/4/"
               topts[1]="3:00:00"
@@ -129,8 +144,11 @@ case $regtest in
            topts[1]="0:30:00" ; popts[1]="48/2"  ; ropts[1]="/1"
            topts[2]="0:30:00" ; popts[2]="60/3"  ; ropts[2]="/2"
         elif [[ "$machine" = "Cheyenne" ]]; then
-           topts[1]="1:59:00" ; popts[1]="6/8/" ; ropts[1]="/1"
-           topts[2]="0:35:00" ; popts[2]="6/10/" ; ropts[2]="/2"
+           topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1"
+           topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2"
+        elif [[ "$machine" = "Gaea" ]]; then
+           topts[1]="0:10:00" ; popts[1]="18/8/" ; ropts[1]="/1"
+           topts[2]="0:10:00" ; popts[2]="18/10/" ; ropts[2]="/2"
         elif [[ "$machine" = "wcoss2" ]]; then
            topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1"
            topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2"
@@ -155,6 +173,12 @@ case $regtest in
         elif [[ "$machine" = "Jet" ]]; then
             topts[1]="0:15:00" ; popts[1]="20/1/"  ; ropts[1]="/1"
             topts[2]="0:15:00" ; popts[2]="20/2/"  ; ropts[2]="/1"
+        elif [[ "$machine" = "Cheyenne" ]]; then
+           topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1"
+           topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2"
+        elif [[ "$machine" = "Gaea" ]]; then
+            topts[1]="0:15:00" ; popts[1]="18/1/"  ; ropts[1]="/1"
+            topts[2]="0:15:00" ; popts[2]="18/2/"  ; ropts[2]="/1"
         elif [[ "$machine" = "wcoss2" ]]; then
            topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1"
            topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1"
@@ -179,6 +203,12 @@ case $regtest in
         elif [[ "$machine" = "Jet" ]]; then
             topts[1]="0:15:00" ; popts[1]="4/4/"  ; ropts[1]="/1"
             topts[2]="0:15:00" ; popts[2]="6/6/"  ; ropts[2]="/1"
+        elif [[ "$machine" = "Cheyenne" ]]; then
+           topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1"
+           topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/2"
+        elif [[ "$machine" = "Gaea" ]]; then
+            topts[1]="0:15:00" ; popts[1]="4/4/"  ; ropts[1]="/1"
+            topts[2]="0:15:00" ; popts[2]="6/6/"  ; ropts[2]="/1"
         elif [[ "$machine" = "wcoss2" ]]; then
            topts[1]="0:15:00" ; popts[1]="28/1/" ; ropts[1]="/1"
            topts[2]="0:15:00" ; popts[2]="28/2/" ; ropts[2]="/1"
@@ -204,8 +234,11 @@ case $regtest in
             topts[1]="0:30:00" ; popts[1]="6/12/"  ; ropts[1]="/1"
             topts[2]="0:30:00" ; popts[2]="8/12/"  ; ropts[2]="/1"
         elif [[ "$machine" = "Cheyenne" ]]; then
-            topts[1]="0:15:00" ; popts[1]="8/6/"  ; ropts[1]="/1"
-            topts[2]="0:15:00" ; popts[2]="8/8/"  ; ropts[2]="/1"
+            topts[1]="0:30:00" ; popts[1]="6/12/"  ; ropts[1]="/1"
+            topts[2]="0:30:00" ; popts[2]="8/12/"  ; ropts[2]="/1"
+        elif [[ "$machine" = "Gaea" ]]; then
+            topts[1]="0:30:00" ; popts[1]="8/6/"  ; ropts[1]="/1"
+            topts[2]="0:30:00" ; popts[2]="8/8/"  ; ropts[2]="/1"
         elif [[ "$machine" = "wcoss2" ]]; then
            topts[1]="0:30:00" ; popts[1]="14/8/" ; ropts[1]="/1"
            topts[2]="0:30:00" ; popts[2]="14/14/" ; ropts[2]="/2"
@@ -233,6 +266,9 @@ case $regtest in
         elif [[ "$machine" = "Cheyenne" ]]; then
             topts[1]="0:20:00" ; popts[1]="6/6/"  ; ropts[1]="/1"
             topts[2]="0:20:00" ; popts[2]="8/8/"  ; ropts[2]="/1"
+        elif [[ "$machine" = "Gaea" ]]; then
+            topts[1]="0:20:00" ; popts[1]="6/6/"  ; ropts[1]="/1"
+            topts[2]="0:20:00" ; popts[2]="8/8/"  ; ropts[2]="/1"
         elif [[ "$machine" = "wcoss2" ]]; then
            topts[1]="0:15:00" ; popts[1]="10/10/" ; ropts[1]="/1"
            topts[2]="0:15:00" ; popts[2]="14/14/" ; ropts[2]="/2"
@@ -258,8 +294,11 @@ case $regtest in
             topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1"
             topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2"
         elif [[ "$machine" = "Cheyenne" ]]; then
-            topts[1]="0:15:00" ; popts[1]="16/2/" ; ropts[1]="/1"
-            topts[2]="0:15:00" ; popts[2]="16/4/" ; ropts[2]="/2"
+            topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1"
+            topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2"
+        elif [[ "$machine" = "Gaea" ]]; then
+            topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1"
+            topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2"
         elif [[ "$machine" = "wcoss2" ]]; then
            topts[1]="0:10:00" ; popts[1]="16/2/" ; ropts[1]="/1"
            topts[2]="0:10:00" ; popts[2]="16/4/" ; ropts[2]="/2"
@@ -310,19 +349,25 @@ if [[ "$machine" = "Hera" ]]; then
    export APRUN="srun"
 elif [[ "$machine" = "Orion" ]]; then
    export OMP_STACKSIZE=2048M
-   export APRUN="srun -n \$ntasks"
+   export APRUN="srun --label -n \$ntasks"
 elif [[ "$machine" = "Jet" ]]; then
    export OMP_STACKSIZE=1024M
    export MPI_BUFS_PER_PROC=256
    export MPI_BUFS_PER_HOST=256
    export MPI_GROUP_MAX=256
-   export APRUN="srun"
+   export APRUN="srun --label -n \$ntasks --cpus-per-task=\$threads"
+elif [[ "$machine" = "Gaea" ]]; then
+   export OMP_STACKSIZE=1024M
+   export MPI_BUFS_PER_PROC=256
+   export MPI_BUFS_PER_HOST=256
+   export MPI_GROUP_MAX=256
+   export APRUN="srun --export=ALL --mpi=pmi2 -n \$ntasks"
 elif [[ "$machine" = "Cheyenne" ]]; then
    export OMP_STACKSIZE=1024M
    export MPI_BUFS_PER_PROC=256
    export MPI_BUFS_PER_HOST=256
    export MPI_GROUP_MAX=256
-   export APRUN="mpirun -v -np \$NCPUS"
+   export APRUN="mpirun -v -np \$ntasks"
 elif [[ "$machine" = "wcoss2" ]]; then
    export OMP_PLACES=cores
    export OMP_STACKSIZE=2G

diff --git a/regression/regression_var.sh b/regression/regression_var.sh
@@ -49,19 +49,33 @@ fi
 echo "Running Regression Tests on '$machine'";
 
 case $machine in
+  Gaea)
+    export queue="normal"
+    export noscrub="/lustre/f2/scratch/$LOGNAME/gsi_tmp/noscrub"
+    export ptmp="/lustre/f2/scratch/$LOGNAME/gsi_tmp/ptmp"
+    export casesdir="/lustre/f2/dev/role.epic/contrib/GSI_data/CASES/regtest"
+
+    export group="global"
+    if [[ "$cmaketest" = "false" ]]; then
+      export basedir="/lustre/f2/dev/$LOGNAME/sandbox/GSI"
+    fi
+
+    export check_resource="no"
+    export accnt="nggps_emc"
+  ;;
   Cheyenne)
-    export queue="economy"
-    export noscrub="/glade/scratch/$LOGNAME"
+    export queue="regular"
+    export noscrub="/glade/scratch/$LOGNAME/noscrub"
     export group="global"
     if [[ "$cmaketest" = "false" ]]; then
-      export basedir="/glade/scratch/$LOGNAME/gsi"
+      export basedir="/glade/scratch/$LOGNAME"
     fi
     export ptmp="/glade/scratch/$LOGNAME/$ptmpName"
 
-    export casesdir="/glade/p/ral/jntp/tools/CASES"
+    export casesdir="/glade/work/epicufsrt/contrib/GSI_data/CASES/regtest"
 
     export check_resource="no"
-    export accnt="p48503002"
+    export accnt="NRAL0032"
   ;;
   wcoss2)
       export local_or_default="${local_or_default:-/lfs/h2/emc/da/noscrub/$LOGNAME}"

diff --git a/ush/build.sh b/ush/build.sh
@@ -30,7 +30,6 @@ set -x
 
 # Set CONTROLPATH variable to user develop installation
 CONTROLPATH="$DIR_ROOT/../develop/install/bin"
-
 # Collect BUILD Options
 CMAKE_OPTS+=" -DCMAKE_BUILD_TYPE=$BUILD_TYPE"
 

diff --git a/ush/module-setup.sh b/ush/module-setup.sh
@@ -58,6 +58,7 @@ elif [[ $MACHINE_ID = gaea* ]] ; then
         source /etc/profile
         __ms_source_etc_profile=yes
     fi
+    source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh
 
     source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh