Merge pull request #232 from DavidHuber-NOAA/port_master_2_jet

GitHub Issue #215. Update Jet support, port monitors to Jet and S4.
NOAA-EMC · Oct 15, 2021 · 2f28fbf · 2f28fbf
2 parents 6cecd8b + 0af2a32
commit 2f28fbf
Show file tree

Hide file tree

Showing 26 changed files with 577 additions and 41 deletions.
diff --git a/cmake/Modules/FindGSICONTROL.cmake b/cmake/Modules/FindGSICONTROL.cmake
@@ -21,6 +21,7 @@ find_file( CONTROL_EXE
         /gpfs/hps3/emc/da/noscrub/Michael.Lueken/svn1/build/bin
         /gpfs/hps3/emc/da/noscrub/Michael.Lueken/svn1/src
         /scratch1/NCEPDEV/da/Michael.Lueken/svn1/build/bin
+        /lfs1/NESDIS/nesdis-rdo2/David.Huber/save/gsi_control/exec
 
     ${NO_DEFAULT_PATH})
 
@@ -45,6 +46,7 @@ find_file( ENKF_CONTROL_EXE
         /gpfs/hps3/emc/da/noscrub/Michael.Lueken/svn1/build/bin
         /gpfs/hps3/emc/da/noscrub/Michael.Lueken/svn1/src/enkf
         /scratch1/NCEPDEV/da/Michael.Lueken/svn1/build/bin
+        /lfs1/NESDIS/nesdis-rdo2/David.Huber/save/gsi_control/exec
 
     ${NO_DEFAULT_PATH})
 

diff --git a/cmake/Modules/platforms/Jet.cmake b/cmake/Modules/platforms/Jet.cmake
@@ -1,10 +1,42 @@
 macro (setJet)
   message("Setting paths for Jet")
-  option(FIND_HDF5 "Try to Find HDF5 libraries" OFF)
+  option(FIND_HDF5 "Try to Find HDF5 libraries" ON)
   option(FIND_HDF5_HL "Try to Find HDF5 libraries" OFF)
   set(HOST_FLAG "-axSSE4.2,AVX,CORE-AVX2" CACHE INTERNAL "Host Flag")
   set(MKL_FLAG "-mkl"  CACHE INTERNAL "MKL Flag")
-  set(GSI_Intel_Platform_FLAGS "${HOST_FLAG} -DPOUND_FOR_STRINGIFY -O3 -fp-model source -assume byterecl -convert big_endian -g -traceback -D_REAL8_ ${OpenMP_Fortran_FLAGS} ${MPI_Fortran_COMPILE_FLAGS}" CACHE INTERNAL "GSI Fortran Flags")
+  set(GSI_Intel_Platform_FLAGS "-DPOUND_FOR_STRINGIFY -O3 -fp-model source -assume byterecl -convert big_endian -g -traceback -D_REAL8_ ${OpenMP_Fortran_FLAGS} ${MPI_Fortran_COMPILE_FLAGS}" CACHE INTERNAL "GSI Fortran Flags")
   set(ENKF_Platform_FLAGS "-O3 ${HOST_FLAG} -warn all -implicitnone -traceback -fp-model strict -convert big_endian -DGFS -D_REAL8_  ${OpenMP_Fortran_FLAGS}" CACHE INTERNAL "ENKF Fortran Flags")
+  set(HDF5_USE_STATIC_LIBRARIES "ON")
+
+  if( NOT DEFINED ENV{NETCDF_VER} )
+    set(NETCDF_VER "3.6.3" )
+  endif()
+  if( NOT DEFINED ENV{BACIO_VER} )
+    set(BACIO_VER "2.0.1" )
+  endif()
+  if( NOT DEFINED ENV{BUFR_VER} )
+    set(BUFR_VER "10.2.5" )
+  endif()
+  if( NOT DEFINED ENV{CRTM_VER} )
+    set(CRTM_VER "2.2.3" )
+  endif()
+  if( NOT DEFINED ENV{NEMSIO_VER} )
+    set(NEMSIO_VER "2.2.1" )
+  endif()
+  if( NOT DEFINED ENV{SFCIO_VER} )
+    set(SFCIO_VER "1.0.0" )
+  endif()
+  if( NOT DEFINED ENV{SIGIO_VER} )
+    set(SIGIO_VER "2.0.1" )
+  endif()
+  if( NOT DEFINED ENV{SP_VER} )
+    set(SP_VER "2.0.2" )
+  endif()
+  if( NOT DEFINED ENV{W3EMC_VER} )
+    set(W3EMC_VER "2.0.5" )
+  endif()
+  if( NOT DEFINED ENV{W3NCO_VER} )
+    set(W3NCO_VER "2.0.6" )
+  endif()
 endmacro()
 
diff --git a/cmake/Modules/setHOST.cmake b/cmake/Modules/setHOST.cmake
@@ -42,7 +42,7 @@ macro( setHOST )
     set( HOST-Generic "TRUE"  )
     setGeneric()
   elseif(HOST-Jet)
-    option(BUILD_CORELIBS "Build the Core libraries " ON)
+    option(BUILD_CORELIBS "Build the Core libraries " OFF)
     set( host "Jet"  )
     set( HOST-Jet "TRUE"  )
     setJet()

diff --git a/modulefiles/modulefile.ProdGSI.jet b/modulefiles/modulefile.ProdGSI.jet
@@ -1,22 +1,23 @@
- #%Module######################################################################
-## NOAA-EMC/GSI
-##_____________________________________________________
+#%Module######################################################################
+# NOAA-EMC/GSI
+#_____________________________________________________
 #proc ModulesHelp { } {
 #puts stderr "Set environment variables for NOAA-EMC/GSI"
 #puts stderr "This module initializes the environment "
 #puts stderr "for the Intel Compiler Suite $version\n"
-#}
+##}
 #module-whatis " NOAA-EMC/GSI whatis description"
 #
 #
 
-set CRTM_FIX /lfs4/HFIP/hfv3gfs/gwv/l0530/lib/crtm/v2.3.0/fix
+setenv CRTM_FIX /lfs4/HFIP/hfv3gfs/glopara/crtm_v2.3.0
 
 # Load cmake
 module load cmake/3.16.1
 
 # Load python
-module load intelpython/3.6.5
+module use /contrib/anaconda/modulefiles
+module load anaconda/5.3.1
 
 # Load hpc-stack
 module use /lfs4/HFIP/hfv3gfs/nwprod/hpc-stack/libs/modulefiles/stack

diff --git a/regression/regression_param.sh b/regression/regression_param.sh
@@ -6,6 +6,11 @@ case $machine in
 	   sub_cmd="sub_hera"
            memnode=96
            numcore=40
+    ;;
+	Jet)
+	   sub_cmd="sub_jet"
+           memnode=96
+           numcore=40
     ;;
 	WCOSS)
 	   sub_cmd="sub_wcoss -a GDAS-T2O -d $PWD"
@@ -52,6 +57,9 @@ case $regtest in
         if [[ "$machine" = "Hera" ]]; then
            topts[1]="0:50:00" ; popts[1]="12/3/" ; ropts[1]="/1"
            topts[2]="0:50:00" ; popts[2]="12/9/" ; ropts[2]="/2"
+        elif [[ "$machine" = "Jet" ]]; then
+           topts[1]="0:50:00" ; popts[1]="12/3/" ; ropts[1]="/1"
+           topts[2]="0:50:00" ; popts[2]="12/9/" ; ropts[2]="/2"
         elif [[ "$machine" = "Cheyenne" ]]; then
            topts[1]="0:30:00" ; popts[1]="16/2/" ; ropts[1]="/1"
            topts[2]="0:30:00" ; popts[2]="16/4/" ; ropts[2]="/2"
@@ -82,6 +90,9 @@ case $regtest in
         if [[ "$machine" = "Hera" ]]; then
             topts[1]="0:15:00" ; popts[1]="12/1/" ; ropts[1]="/1"
             topts[2]="0:15:00" ; popts[2]="12/2/" ; ropts[2]="/2"
+        elif [[ "$machine" = "Jet" ]]; then
+            topts[1]="0:15:00" ; popts[1]="12/1/" ; ropts[1]="/1"
+            topts[2]="0:15:00" ; popts[2]="12/2/" ; ropts[2]="/2"
         elif [[ "$machine" = "Cheyenne" ]]; then
             topts[1]="0:15:00" ; popts[1]="16/1/" ; ropts[1]="/1"
             topts[2]="0:15:00" ; popts[2]="16/2/" ; ropts[2]="/2"
@@ -112,6 +123,9 @@ case $regtest in
         if [[ "$machine" = "Hera" ]]; then
             topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1"
             topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2"
+        elif [[ "$machine" = "Jet" ]]; then
+            topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1"
+            topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2"
         elif [[ "$machine" = "WCOSS" ]]; then
             topts[1]="0:35:00" ; popts[1]="16/2/" ; ropts[1]="/1"
             topts[2]="0:25:00" ; popts[2]="16/4/" ; ropts[2]="/2"
@@ -133,6 +147,8 @@ case $regtest in
            topts[1]="0:45:00"
            if [[ "$machine" = "Hera" ]]; then
               popts[1]="12/5/"
+           elif [[ "$machine" = "Jet" ]]; then
+              popts[1]="12/5/"
            elif [[ "$machine" = "WCOSS" ]]; then
               popts[1]="16/4/"
            elif [[ "$machine" = "WCOSS_C" ]]; then
@@ -153,6 +169,9 @@ case $regtest in
         if [[ "$machine" = "Hera" ]]; then
            topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1"
            topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2"
+        elif [[ "$machine" = "Jet" ]]; then
+           topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1"
+           topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2"
         elif [[ "$machine" = "WCOSS" ]]; then
            topts[1]="0:15:00" ; popts[1]="16/2/" ; ropts[1]="/1"
            topts[2]="0:15:00" ; popts[2]="16/4/" ; ropts[2]="/2"
@@ -183,6 +202,9 @@ case $regtest in
         if [[ "$machine" = "Hera" ]]; then
            topts[1]="0:15:00" ; popts[1]="6/8/" ; ropts[1]="/1"
            topts[2]="0:15:00" ; popts[2]="6/10/" ; ropts[2]="/2"
+        elif [[ "$machine" = "Jet" ]]; then
+           topts[1]="0:15:00" ; popts[1]="6/8/" ; ropts[1]="/1"
+           topts[2]="0:15:00" ; popts[2]="6/10/" ; ropts[2]="/2"
         elif [[ "$machine" = "WCOSS" ]]; then
            topts[1]="1:59:00" ; popts[1]="6/8/" ; ropts[1]="/1"
            topts[2]="0:35:00" ; popts[2]="6/10/" ; ropts[2]="/2"
@@ -213,6 +235,9 @@ case $regtest in
         if [[ "$machine" = "Hera" ]]; then
            topts[1]="0:35:00" ; popts[1]="6/8/" ; ropts[1]="/1"
            topts[2]="0:35:00" ; popts[2]="6/10/" ; ropts[2]="/2"
+        elif [[ "$machine" = "Jet" ]]; then
+           topts[1]="0:35:00" ; popts[1]="6/8/" ; ropts[1]="/1"
+           topts[2]="0:35:00" ; popts[2]="6/10/" ; ropts[2]="/2"
         elif [[ "$machine" = "WCOSS" ]]; then
            topts[1]="1:59:00" ; popts[1]="6/8/" ; ropts[1]="/1"
            topts[2]="0:35:00" ; popts[2]="6/10/" ; ropts[2]="/2"
@@ -243,6 +268,9 @@ case $regtest in
         if [[ "$machine" = "Hera" ]]; then
            topts[1]="0:35:00" ; popts[1]="6/8/" ; ropts[1]="/1"
            topts[2]="0:35:00" ; popts[2]="6/10/" ; ropts[2]="/2"
+        elif [[ "$machine" = "Jet" ]]; then
+           topts[1]="0:35:00" ; popts[1]="6/8/" ; ropts[1]="/1"
+           topts[2]="0:35:00" ; popts[2]="6/10/" ; ropts[2]="/2"
         elif [[ "$machine" = "WCOSS" ]]; then
            topts[1]="1:59:00" ; popts[1]="6/8/" ; ropts[1]="/1"
            topts[2]="0:35:00" ; popts[2]="6/10/" ; ropts[2]="/2"
@@ -273,6 +301,9 @@ case $regtest in
         if [[ "$machine" = "Hera" ]]; then
            topts[1]="0:30:00" ; popts[1]="12/3/" ; ropts[1]="/1"
            topts[2]="0:30:00" ; popts[2]="12/5/" ; ropts[2]="/2"
+        elif [[ "$machine" = "Jet" ]]; then
+           topts[1]="0:30:00" ; popts[1]="12/3/" ; ropts[1]="/1"
+           topts[2]="0:30:00" ; popts[2]="12/5/" ; ropts[2]="/2"
         elif [[ "$machine" = "WCOSS" ]]; then
            topts[1]="0:20:00" ; popts[1]="16/2/" ; ropts[1]="/1"
            topts[2]="0:20:00" ; popts[2]="16/4/" ; ropts[2]="/2"
@@ -303,6 +334,9 @@ case $regtest in
         if [[ "$machine" = "Hera" ]]; then
            topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1"
            topts[2]="0:15:00" ; popts[2]="12/9/" ; ropts[2]="/2"
+        elif [[ "$machine" = "Jet" ]]; then
+           topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1"
+           topts[2]="0:15:00" ; popts[2]="12/9/" ; ropts[2]="/2"
         elif [[ "$machine" = "WCOSS" ]]; then
            topts[1]="0:15:00" ; popts[1]="16/2/" ; ropts[1]="/1"
            topts[2]="0:15:00" ; popts[2]="16/4/" ; ropts[2]="/2"
@@ -333,6 +367,9 @@ case $regtest in
         if [[ "$machine" = "Hera" ]]; then
             topts[1]="0:15:00" ; popts[1]="4/4/"  ; ropts[1]="/1"
             topts[2]="0:15:00" ; popts[2]="6/6/"  ; ropts[2]="/1"
+        elif [[ "$machine" = "Jet" ]]; then
+            topts[1]="0:15:00" ; popts[1]="4/4/"  ; ropts[1]="/1"
+            topts[2]="0:15:00" ; popts[2]="6/6/"  ; ropts[2]="/1"
         elif [[ "$machine" = "WCOSS" ]]; then
             topts[1]="0:15:00" ; popts[1]="16/1/" ; ropts[1]="/1"
             topts[2]="0:15:00" ; popts[2]="16/2/" ; ropts[2]="/1"
@@ -363,6 +400,9 @@ case $regtest in
         if [[ "$machine" = "Hera" ]]; then
             topts[1]="0:15:00" ; popts[1]="4/4/"  ; ropts[1]="/1"
             topts[2]="0:15:00" ; popts[2]="6/6/"  ; ropts[2]="/1"
+        elif [[ "$machine" = "Jet" ]]; then
+            topts[1]="0:15:00" ; popts[1]="4/4/"  ; ropts[1]="/1"
+            topts[2]="0:15:00" ; popts[2]="6/6/"  ; ropts[2]="/1"
         elif [[ "$machine" = "WCOSS" ]]; then
             topts[1]="0:15:00" ; popts[1]="16/1/" ; ropts[1]="/1"
             topts[2]="0:15:00" ; popts[2]="16/2/" ; ropts[2]="/1"
@@ -387,6 +427,9 @@ case $regtest in
         if [[ "$machine" = "Hera" ]]; then
             topts[1]="0:30:00" ; popts[1]="6/6/"  ; ropts[1]="/1"
             topts[2]="0:30:00" ; popts[2]="8/8/"  ; ropts[2]="/1"
+        elif [[ "$machine" = "Jet" ]]; then
+            topts[1]="0:30:00" ; popts[1]="6/6/"  ; ropts[1]="/1"
+            topts[2]="0:30:00" ; popts[2]="8/8/"  ; ropts[2]="/1"
         elif [[ "$machine" = "WCOSS" ]]; then
             topts[1]="0:30:00" ; popts[1]="7/12/" ; ropts[1]="/1"
             topts[2]="0:30:00" ; popts[2]="9/12/" ; ropts[2]="/2"
@@ -417,6 +460,9 @@ case $regtest in
         if [[ "$machine" = "Hera" ]]; then
             topts[1]="0:15:00" ; popts[1]="4/2/"  ; ropts[1]="/1"
             topts[2]="0:15:00" ; popts[2]="4/4/"  ; ropts[2]="/1"
+        elif [[ "$machine" = "Jet" ]]; then
+            topts[1]="0:15:00" ; popts[1]="4/2/"  ; ropts[1]="/1"
+            topts[2]="0:15:00" ; popts[2]="4/4/"  ; ropts[2]="/1"
         elif [[ "$machine" = "WCOSS" ]]; then
             topts[1]="0:15:00" ; popts[1]="8/1/"  ; ropts[1]="/1"
             topts[2]="0:15:00" ; popts[2]="16/1/" ; ropts[2]="/2"
@@ -447,6 +493,9 @@ case $regtest in
         if [[ "$machine" = "Hera" ]]; then
             topts[1]="0:30:00" ; popts[1]="7/10/"  ; ropts[1]="/1"
             topts[2]="0:30:00" ; popts[2]="9/10/"  ; ropts[2]="/1"
+        elif [[ "$machine" = "Jet" ]]; then
+            topts[1]="0:30:00" ; popts[1]="7/10/"  ; ropts[1]="/1"
+            topts[2]="0:30:00" ; popts[2]="9/10/"  ; ropts[2]="/1"
         elif [[ "$machine" = "WCOSS" ]]; then
             topts[1]="0:30:00" ; popts[1]="7/10/" ; ropts[1]="/1"
             topts[2]="0:30:00" ; popts[2]="9/10/" ; ropts[2]="/2"
@@ -474,6 +523,9 @@ case $regtest in
         if [[ "$machine" = "Hera" ]]; then
             topts[1]="0:30:00" ; popts[1]="6/12/"  ; ropts[1]="/1"
             topts[2]="0:30:00" ; popts[2]="8/12/"  ; ropts[2]="/1"
+        elif [[ "$machine" = "Jet" ]]; then
+            topts[1]="0:30:00" ; popts[1]="6/12/"  ; ropts[1]="/1"
+            topts[2]="0:30:00" ; popts[2]="8/12/"  ; ropts[2]="/1"
         elif [[ "$machine" = "WCOSS" ]]; then
             topts[1]="0:30:00" ; popts[1]="8/10/"  ; ropts[1]="/1"
             topts[2]="0:30:00" ; popts[2]="8/12/"  ; ropts[2]="/1"
@@ -504,6 +556,9 @@ case $regtest in
         if [[ "$machine" = "Hera" ]]; then
             topts[1]="0:20:00" ; popts[1]="6/6/"  ; ropts[1]="/1"
             topts[2]="0:20:00" ; popts[2]="8/8/"  ; ropts[2]="/1"
+        elif [[ "$machine" = "Jet" ]]; then
+            topts[1]="0:20:00" ; popts[1]="6/6/"  ; ropts[1]="/1"
+            topts[2]="0:20:00" ; popts[2]="8/8/"  ; ropts[2]="/1"
         elif [[ "$machine" = "WCOSS" ]]; then
             topts[1]="0:20:00" ; popts[1]="6/6/"  ; ropts[1]="/1"
             topts[2]="0:20:00" ; popts[2]="8/8/"  ; ropts[2]="/1"
@@ -531,6 +586,9 @@ case $regtest in
         if [[ "$machine" = "Hera" ]]; then
             topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1"
             topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2"
+        elif [[ "$machine" = "Jet" ]]; then
+            topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1"
+            topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2"
         elif [[ "$machine" = "WCOSS" ]]; then
             topts[1]="0:15:00" ; popts[1]="16/2/" ; ropts[1]="/1"
             topts[2]="0:15:00" ; popts[2]="16/4/" ; ropts[2]="/2"
@@ -558,6 +616,9 @@ case $regtest in
         if [[ "$machine" = "Hera" ]]; then
             topts[1]="0:30:00" ; popts[1]="12/3/" ; ropts[1]="/1"
             topts[2]="0:30:00" ; popts[2]="12/9/" ; ropts[2]="/2"
+        elif [[ "$machine" = "Jet" ]]; then
+            topts[1]="0:30:00" ; popts[1]="12/3/" ; ropts[1]="/1"
+            topts[2]="0:30:00" ; popts[2]="12/9/" ; ropts[2]="/2"
         elif [[ "$machine" = "Cheyenne" ]]; then
             topts[1]="0:15:00" ; popts[1]="16/2/" ; ropts[1]="/1"
             topts[2]="0:15:00" ; popts[2]="16/4/" ; ropts[2]="/2"
@@ -588,6 +649,9 @@ case $regtest in
         if [[ "$machine" = "Hera" ]]; then
             topts[1]="0:20:00" ; popts[1]="16/4/" ; ropts[1]="/1"
             topts[2]="0:20:00" ; popts[2]="16/8/" ; ropts[2]="/2"
+        elif [[ "$machine" = "Jet" ]]; then
+            topts[1]="0:20:00" ; popts[1]="16/4/" ; ropts[1]="/1"
+            topts[2]="0:20:00" ; popts[2]="16/8/" ; ropts[2]="/2"
         elif [[ "$machine" = "Cheyenne" ]]; then
             topts[1]="0:15:00" ; popts[1]="16/2/" ; ropts[1]="/1"
             topts[2]="0:15:00" ; popts[2]="16/4/" ; ropts[2]="/2"
@@ -648,6 +712,12 @@ if [[ "$machine" = "Hera" ]]; then
    export MPI_BUFS_PER_HOST=256
    export MPI_GROUP_MAX=256
    export APRUN="srun"
+elif [[ "$machine" = "Jet" ]]; then
+   export OMP_STACKSIZE=1024M
+   export MPI_BUFS_PER_PROC=256
+   export MPI_BUFS_PER_HOST=256
+   export MPI_GROUP_MAX=256
+   export APRUN="srun"
 elif [[ "$machine" = "Cheyenne" ]]; then
    export OMP_STACKSIZE=1024M
    export MPI_BUFS_PER_PROC=256

diff --git a/regression/regression_var.sh b/regression/regression_var.sh
@@ -38,6 +38,8 @@ if [ -z ${machine+x} ]; then
    export machine="Cheyenne"
   elif [ -d /scratch1/NCEPDEV/da ]; then # Hera
    export machine="Hera"
+  elif [ -d /lfs1/NCEPDEV ]; then # Jet
+   export machine="Jet"
   elif [ -d /gpfs/hps/ptmp ]; then # LUNA or SURGE
    export machine="WCOSS_C"
   elif [ -d /gpfs/dell1/ptmp ]; then # venus or mars
@@ -131,6 +133,30 @@ case $machine in
    #  After completion of regression tests, will remove the regression test subdirecories
    export clean=".true."
    ;;
+   Jet)
+
+   set -x
+   export noscrub=/lfs1/NESDIS/nesdis-rdo2/$LOGNAME/noscrub
+   export ptmp=/lfs1/NESDIS/nesdis-rdo2/$LOGNAME/ptmp
+   export fixcrtm="/lfs1/NESDIS/nesdis-rdo2/David.Huber/save/CRTM_REL-2.2.3/crtm_v2.2.3/fix_update"
+   export casesdir="/lfs1/NESDIS/nesdis-rdo2/David.Huber/save/CASES"
+   export ndate=$NDATE
+   export check_resource="no"
+   export accnt="nesdis-rdo2"
+
+   export group="global"
+   export queue="batch"
+   if [[ "$cmaketest" = "false" ]]; then
+     export basedir="/lfs1/NESDIS/nesdis-rdo2/$LOGNAME/gsi"
+   fi 
+
+   export ptmp="/lfs1/NESDIS/nesdis-rdo2/$LOGNAME/ptmp/$ptmpName"
+
+   #  On Jet, there are no scrubbers to remove old contents from stmp* directories.
+   #  After completion of regression tests, will remove the regression test subdirecories
+   export clean=".true."
+   set +x
+   ;;
    WCOSS_C)
    if [ -d /gpfs/hps3/emc/global/noscrub/$LOGNAME ]; then
       export noscrub="/gpfs/hps3/emc/global/noscrub/$LOGNAME"

diff --git a/ush/build_all_cmake.sh b/ush/build_all_cmake.sh
@@ -82,6 +82,10 @@ elif [ $target = hera -o $target = orion -o $target = s4 ]; then
     module purge
     module use $dir_modules
     module load modulefile.ProdGSI.$target
+elif [ $target = jet ]; then
+    module purge
+    module use $dir_modules
+    module load modulefile.ProdGSI.$target
 elif [ $target = cheyenne ]; then
     module purge
     source $dir_modules/modulefile.ProdGSI.$target

diff --git a/ush/calcanl_gfs.py b/ush/calcanl_gfs.py
@@ -229,6 +229,13 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix,
             with open(CalcAnlDir+'/hosts', 'w') as hostfile:
                 hostfile.write(hosts[ihost]+'\n')
                 if launcher == 'srun': # need to write host per task not per node for slurm
+                    # For xjet, each instance of chgres_inc must run on two nodes each
+                    if os.getenv('SLURM_JOB_PARTITION','') == 'xjet':
+                        for a in range(0,4):
+                            hostfile.write(hosts[ihost]+'\n')
+                        ihost+=1
+                        for a in range(0,5):
+                            hostfile.write(hosts[ihost]+'\n')
                     for a in range(0,9): # need 9 more of the same host for the 10 tasks for chgres_inc
                         hostfile.write(hosts[ihost]+'\n')
             if launcher == 'srun':