Merge branch 'NOAA-EMC:develop' into feature/gwdev_issue_2129

kayeekayee · Feb 16, 2024 · eb2ed53 · eb2ed53
2 parents 9929277 + a23b7f2
commit eb2ed53
Show file tree

Hide file tree

Showing 29 changed files with 200 additions and 64 deletions.
diff --git a/ci/cases/pr/C48C48_ufs_hybatmDA.yaml b/ci/cases/pr/C48C48_ufs_hybatmDA.yaml
@@ -0,0 +1,22 @@
+experiment:
+  system: gfs
+  mode: cycled
+
+arguments:
+  pslot: {{ 'pslot' | getenv }}
+  app: ATM
+  resdetatmos: 48
+  resensatmos: 48
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
+  expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
+  idate: 2021032312
+  edate: 2021032400
+  nens: 2
+  gfs_cyc: 1
+  start: warm
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml
+
+skip_ci_on_hosts:
+  - hera
+  - orion
+  - hercules
diff --git a/ci/cases/pr/C48_S2SWA_gefs.yaml b/ci/cases/pr/C48_S2SWA_gefs.yaml
@@ -16,3 +16,8 @@ arguments:
   idate: 2021032312
   edate: 2021032312
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_ci_defaults.yaml
+
+skip_ci_on_hosts:
+  - hera
+  - orion
+  - hercules
diff --git a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml
@@ -17,7 +17,7 @@ arguments:
   start: warm
   yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml
 
-skip_ci_on_host:
+skip_ci_on_hosts:
   - orion
   - hera
   - hercules
diff --git a/ci/cases/yamls/build.yaml b/ci/cases/yamls/build.yaml
@@ -1,3 +1,3 @@
 builds:
  - gefs: './build_all.sh'
- - gfs: './build_all.sh -gu'
+ - gfs: './build_all.sh -wgu'
diff --git a/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml b/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml
@@ -0,0 +1,14 @@
+defaults:
+  !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
+base:
+  DOIAU: "NO"
+  DO_JEDIATMVAR: "YES"
+  DO_JEDIATMENS: "YES"
+  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
+esfc:
+  DONST: "NO"
+nsst:
+  NST_MODEL: "1"
+sfcanl:
+  DONST: "NO"
+
diff --git a/ci/scripts/clone-build_ci.sh b/ci/scripts/clone-build_ci.sh
@@ -74,7 +74,7 @@ set +e
 source "${HOMEgfs}/ush/module-setup.sh"
 export BUILD_JOBS=8
 rm -rf log.build
-./build_all.sh -gu  >> log.build 2>&1
+./build_all.sh -guw  >> log.build 2>&1
 build_status=$?
 
 DATE=$(date +'%D %r')

diff --git a/docs/source/clone.rst b/docs/source/clone.rst
@@ -39,6 +39,13 @@ For coupled cycling (include new UFSDA) use the `-gu` options during build:
    ./build_all.sh -gu
 
 
+For building with PDLIB for the wave model, use the `-w` options during build:
+
+::
+
+   ./build_all.sh -w
+
+
 Build workflow components and link workflow artifacts such as executables, etc.
 
 ::

diff --git a/env/WCOSS2.env b/env/WCOSS2.env
@@ -176,6 +176,8 @@ elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
     if [[ "${step}" = "fcst" ]]; then
         export OMP_PLACES=cores
         export OMP_STACKSIZE=2048M
+        export MPICH_MPIIO_HINTS="*:romio_cb_write=disable"
+        export FI_OFI_RXM_SAR_LIMIT=3145728
     elif [[ "${step}" = "efcs" ]]; then
         export MPICH_MPIIO_HINTS="*:romio_cb_write=disable"
         export FI_OFI_RXM_SAR_LIMIT=3145728

diff --git a/parm/config/gfs/config.base.emc.dyn b/parm/config/gfs/config.base.emc.dyn
@@ -183,16 +183,16 @@ export ICERES="${OCNRES}"
 # These are the currently recommended grid-combinations
 case "${CASE}" in
     "C48")
-        export waveGRD='glo_500'
+        export waveGRD='uglo_100km'
         ;;
     "C96" | "C192")
-        export waveGRD='glo_200'
+        export waveGRD='uglo_100km'
         ;;
     "C384")
-        export waveGRD='glo_025'
+        export waveGRD='uglo_100km'
         ;;
     "C768" | "C1152")
-        export waveGRD='mx025'
+        export waveGRD='uglo_m1g16'
         ;;
     *)
         echo "FATAL ERROR: Unrecognized CASE ${CASE}, ABORT!"
@@ -397,4 +397,11 @@ export FITSARC="YES"
 export FHMAX_FITS=132
 [[ "${FHMAX_FITS}" -gt "${FHMAX_GFS}" ]] && export FHMAX_FITS=${FHMAX_GFS}
 
+# The monitor jobs are not yet supported for JEDIATMVAR
+if [[ ${DO_JEDIATMVAR} = "YES" ]]; then
+   export DO_VERFOZN="NO"     # Ozone data assimilation monitoring
+   export DO_VERFRAD="NO"     # Radiance data assimilation monitoring
+   export DO_VMINMON="NO"     # GSI minimization monitoring
+fi
+
 echo "END: config.base"
diff --git a/parm/config/gfs/config.esfc b/parm/config/gfs/config.esfc
@@ -16,4 +16,9 @@ if [ $DOIAU_ENKF = "YES" ]; then
    export DOSFCANL_ENKF="NO"
 fi
 
+# Turn off NST in JEDIATMENS
+if [[ "${DO_JEDIATMENS}" == "YES" ]]; then
+   export DONST="NO"
+fi
+
 echo "END: config.esfc"
diff --git a/parm/config/gfs/config.nsst b/parm/config/gfs/config.nsst
@@ -10,6 +10,11 @@ echo "BEGIN: config.nsst"
 # nstf_name(1) : NST_MODEL (NSST Model) : 0 = OFF, 1 = ON but uncoupled, 2 = ON and coupled
 export NST_MODEL=2
 
+# Set NST_MODEL for JEDIATMVAR or JEDIATMENS
+if [[ "${DO_JEDIATMVAR}" == "YES" || "${DO_JEDIATMENS}" == "YES" ]]; then
+   export NST_MODEL=1
+fi
+
 # nstf_name(2) : NST_SPINUP : 0 = OFF, 1 = ON,
 export NST_SPINUP=0
 cdate="${PDY}${cyc}"

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
@@ -91,7 +91,7 @@ case ${step} in
     ;;
 
   "prepatmiodaobs")
-    export wtime_prepatmiodaobs="00:10:00"
+    export wtime_prepatmiodaobs="00:30:00"
     export npe_prepatmiodaobs=1
     export nth_prepatmiodaobs=1
     export npe_node_prepatmiodaobs=$(( npe_node_max / nth_prepatmiodaobs ))
@@ -196,8 +196,8 @@ case ${step} in
 
   "atmanlinit")
     # make below case dependent later
-    export layout_x=1
-    export layout_y=1
+    export layout_x=8
+    export layout_y=8
 
     export layout_gsib_x=$(( layout_x * 3 ))
     export layout_gsib_y=$(( layout_y * 2 ))
@@ -212,15 +212,16 @@ case ${step} in
 
   "atmanlrun")
     # make below case dependent later
-    export layout_x=1
-    export layout_y=1
+    export layout_x=8
+    export layout_y=8
 
     export wtime_atmanlrun="00:30:00"
     export npe_atmanlrun=$(( layout_x * layout_y * 6 ))
     export npe_atmanlrun_gfs=$(( layout_x * layout_y * 6 ))
     export nth_atmanlrun=1
     export nth_atmanlrun_gfs=${nth_atmanlrun}
     export npe_node_atmanlrun=$(( npe_node_max / nth_atmanlrun ))
+    export memory_atmanlrun="96GB"
     export is_exclusive=True
     ;;
 
@@ -737,15 +738,15 @@ case ${step} in
     export npe_genesis=1
     export nth_genesis=1
     export npe_node_genesis=1
-    export memory_genesis="4G"
+    export memory_genesis="10G"
     ;;
 
   "genesis_fsu")
     export wtime_genesis_fsu="00:10:00"
     export npe_genesis_fsu=1
     export nth_genesis_fsu=1
     export npe_node_genesis_fsu=1
-    export memory_genesis_fsu="4G"
+    export memory_genesis_fsu="10G"
     ;;
 
   "fit2obs")
@@ -829,8 +830,8 @@ case ${step} in
 
   "atmensanlinit")
     # make below case dependent later
-    export layout_x=1
-    export layout_y=1
+    export layout_x=8
+    export layout_y=8
 
     export wtime_atmensanlinit="00:10:00"
     export npe_atmensanlinit=1
@@ -841,15 +842,16 @@ case ${step} in
 
   "atmensanlrun")
     # make below case dependent later
-    export layout_x=1
-    export layout_y=1
+    export layout_x=8
+    export layout_y=8
 
     export wtime_atmensanlrun="00:30:00"
     export npe_atmensanlrun=$(( layout_x * layout_y * 6 ))
     export npe_atmensanlrun_gfs=$(( layout_x * layout_y * 6 ))
     export nth_atmensanlrun=1
     export nth_atmensanlrun_gfs=${nth_atmensanlrun}
     export npe_node_atmensanlrun=$(( npe_node_max / nth_atmensanlrun ))
+    export memory_atmensanlrun="96GB"
     export is_exclusive=True
     ;;
 

diff --git a/parm/config/gfs/config.sfcanl b/parm/config/gfs/config.sfcanl
@@ -8,4 +8,9 @@ echo "BEGIN: config.sfcanl"
 # Get task specific resources
 . $EXPDIR/config.resources sfcanl
 
+# Turn off NST in JEDIATMVAR
+if [[ "${DO_JEDIATMVAR}" == "YES" ]]; then
+   export DONST="NO"
+fi
+
 echo "END: config.sfcanl"
diff --git a/parm/config/gfs/config.stage_ic b/parm/config/gfs/config.stage_ic
@@ -21,16 +21,16 @@ case "${CASE}" in
     export CPL_WAVIC=workflow_C384_refactored
     ;;
   "C768")
-    export CPL_ATMIC=HR2_refactored
-    export CPL_ICEIC=HR1_refactored
-    export CPL_OCNIC=HR1_refactored
-    export CPL_WAVIC=HR1_refactored
+    export CPL_ATMIC=HR3C768
+    export CPL_ICEIC=HR3marine
+    export CPL_OCNIC=HR3marine
+    export CPL_WAVIC=HR3marine
     ;;
   "C1152")
-    export CPL_ATMIC=HR2_C1152_refactored
-    export CPL_ICEIC=HR3_refactored
-    export CPL_OCNIC=HR3_refactored
-    export CPL_WAVIC=HR1_refactored
+    export CPL_ATMIC=HR3C1152
+    export CPL_ICEIC=HR3marine
+    export CPL_OCNIC=HR3marine
+    export CPL_WAVIC=HR3marine
     ;;
   *)
     echo "FATAL ERROR Unrecognized resolution: ${CASE}"

diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs
@@ -15,7 +15,7 @@ if (( $# <= 1 )); then
     echo "--fv3 C48|C96|C192|C384|C768|C1152|C3072"
     echo "--mom6 500|100|025"
     echo "--cice6 500|100|025"
-    echo "--ww3 gnh_10m;aoc_9km;gsh_15m|gwes_30m|glo_025|glo_200|glo_500|mx025"
+    echo "--ww3 gnh_10m;aoc_9km;gsh_15m|gwes_30m|glo_025|glo_200|glo_500|mx025|uglo_100km|uglo_m1g16"
     echo "--gocart"
 
     exit 1
@@ -416,6 +416,14 @@ if [[ "${skip_ww3}" == "false" ]]; then
     "mx025")
       ntasks_ww3=80
       ;;
+    "uglo_100km")
+      ntasks_ww3=40
+      nthreads_ww3=1
+      ;;
+    "uglo_m1g16")  
+      ntasks_ww3=1000
+      nthreads_ww3=1
+      ;;
     *)
       echo "FATAL ERROR: Unsupported WW3 resolution = ${ww3_res}, ABORT!"
       exit 1

diff --git a/parm/config/gfs/config.wave b/parm/config/gfs/config.wave
@@ -80,7 +80,19 @@ case "${waveGRD}" in
     export wavepostGRD='glo_500'
     export waveuoutpGRD=${waveGRD}
     ;;
-  *)
+  "uglo_100km") 
+    #unstructured 100km grid 
+    export waveinterpGRD='glo_200'
+    export wavepostGRD=''
+    export waveuoutpGRD=${waveGRD}
+    ;;
+  "uglo_m1g16")
+    #unstructured m1v16 grid 
+    export waveinterpGRD='glo_15mxt'
+    export wavepostGRD=''
+    export waveuoutpGRD=${waveGRD}
+    ;;
+   *)
     echo "FATAL ERROR: No grid specific wave config values exist for ${waveGRD}. Aborting."
     exit 1
     ;;

diff --git a/parm/wave/at_10m_interp.inp.tmpl b/parm/wave/at_10m_interp.inp.tmpl
@@ -5,7 +5,7 @@ $ Start Time DT NSteps
 $ Total number of grids 
  2
 $ Grid extensions 
-  'gnh_10m'
+  'uglo_m1g16'
   'at_10m'
 $
  0

diff --git a/parm/wave/ep_10m_interp.inp.tmpl b/parm/wave/ep_10m_interp.inp.tmpl
@@ -5,7 +5,7 @@ $ Start Time DT NSteps
 $ Total number of grids 
  2
 $ Grid extensions 
-  'gnh_10m'
+  'uglo_m1g16'
   'ep_10m'
 $
  0

diff --git a/parm/wave/glo_15mxt_interp.inp.tmpl b/parm/wave/glo_15mxt_interp.inp.tmpl
@@ -3,11 +3,9 @@ $------------------------------------------------
 $ Start Time DT NSteps
  TIME DT NSTEPS
 $ Total number of grids 
- 4
+ 2
 $ Grid extensions 
-  'gnh_10m'
-  'aoc_9km'
-  'gsh_15m' 
+  'uglo_m1g16'
   'glo_15mxt'
 $
  0

diff --git a/parm/wave/glo_200_interp.inp.tmpl b/parm/wave/glo_200_interp.inp.tmpl
@@ -0,0 +1,12 @@
+$ Input file for interpolation of GLO30m_ext Grid
+$------------------------------------------------
+$ Start Time DT NSteps
+ TIME DT NSTEPS
+$ Total number of grids 
+ 2
+$ Grid extensions 
+  'uglo_100km'
+  'glo_200'
+$
+ 0
+$
diff --git a/parm/wave/glo_30m_interp.inp.tmpl b/parm/wave/glo_30m_interp.inp.tmpl
@@ -3,11 +3,9 @@ $------------------------------------------------
 $ Start Time DT NSteps
  TIME DT NSTEPS
 $ Total number of grids 
- 4
+ 2
 $ Grid extensions 
-  'gnh_10m'
-  'aoc_9km'
-  'gsh_15m' 
+  'uglo_m1g16'
   'glo_30m'
 $
  0

diff --git a/parm/wave/wc_10m_interp.inp.tmpl b/parm/wave/wc_10m_interp.inp.tmpl
@@ -5,7 +5,7 @@ $ Start Time DT NSteps
 $ Total number of grids 
  2
 $ Grid extensions 
-  'gnh_10m'
+  'uglo_m1g16'
   'wc_10m'
 $
  0