Add unstructured grid for HR3/GFS (#2230)

This adds the capability to use unstructured grids in the global workflow,
which will be used in HR3.  There are new fix files for a low-resolution
100km grid and a grid closer to our targeted GFSv17 grid which has the
resolution combined from the older multi_1 and GFSv16 grids.

The fix file update is here: #2229 

Note: This now means that GFS tests need a new build option:  
`./build_all.sh -w` So that PDLIB=ON is turned on for compiling relevant
UFS and WW3 codes. 

Resolves #1547

ci/cases/pr/C48_S2SWA_gefs.yaml

@@ -16,3 +16,8 @@ arguments:
   idate: 2021032312
   edate: 2021032312
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_ci_defaults.yaml
+
+skip_ci_on_hosts:
+  - hera
+  - orion
+  - hercules

ci/cases/pr/C48mx500_3DVarAOWCDA.yaml

@@ -17,7 +17,7 @@ arguments:
   start: warm
   yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml
 
-skip_ci_on_host:
+skip_ci_on_hosts:
   - orion
   - hera
   - hercules

ci/cases/yamls/build.yaml

@@ -1,3 +1,3 @@
 builds:
  - gefs: './build_all.sh'
- - gfs: './build_all.sh -gu'
+ - gfs: './build_all.sh -wgu'

ci/scripts/clone-build_ci.sh

@@ -74,7 +74,7 @@ set +e
 source "${HOMEgfs}/ush/module-setup.sh"
 export BUILD_JOBS=8
 rm -rf log.build
-./build_all.sh -gu  >> log.build 2>&1
+./build_all.sh -guw  >> log.build 2>&1
 build_status=$?
 
 DATE=$(date +'%D %r')

docs/source/clone.rst

@@ -39,6 +39,13 @@ For coupled cycling (include new UFSDA) use the `-gu` options during build:
    ./build_all.sh -gu
 
 
+For building with PDLIB for the wave model, use the `-w` options during build:
+
+::
+
+   ./build_all.sh -w
+
+
 Build workflow components and link workflow artifacts such as executables, etc.
 
 ::

env/WCOSS2.env

@@ -176,6 +176,8 @@ elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
     if [[ "${step}" = "fcst" ]]; then
         export OMP_PLACES=cores
         export OMP_STACKSIZE=2048M
+        export MPICH_MPIIO_HINTS="*:romio_cb_write=disable"
+        export FI_OFI_RXM_SAR_LIMIT=3145728
     elif [[ "${step}" = "efcs" ]]; then
         export MPICH_MPIIO_HINTS="*:romio_cb_write=disable"
         export FI_OFI_RXM_SAR_LIMIT=3145728

parm/config/gfs/config.base.emc.dyn

@@ -183,16 +183,16 @@ export ICERES="${OCNRES}"
 # These are the currently recommended grid-combinations
 case "${CASE}" in
     "C48")
-        export waveGRD='glo_500'
+        export waveGRD='uglo_100km'
         ;;
     "C96" | "C192")
-        export waveGRD='glo_200'
+        export waveGRD='uglo_100km'
         ;;
     "C384")
-        export waveGRD='glo_025'
+        export waveGRD='uglo_100km'
         ;;
     "C768" | "C1152")
-        export waveGRD='mx025'
+        export waveGRD='uglo_m1g16'
         ;;
     *)
         echo "FATAL ERROR: Unrecognized CASE ${CASE}, ABORT!"

parm/config/gfs/config.resources

@@ -737,15 +737,15 @@ case ${step} in
     export npe_genesis=1
     export nth_genesis=1
     export npe_node_genesis=1
-    export memory_genesis="4G"
+    export memory_genesis="10G"
     ;;
 
   "genesis_fsu")
     export wtime_genesis_fsu="00:10:00"
     export npe_genesis_fsu=1
     export nth_genesis_fsu=1
     export npe_node_genesis_fsu=1
-    export memory_genesis_fsu="4G"
+    export memory_genesis_fsu="10G"
     ;;
 
   "fit2obs")

parm/config/gfs/config.stage_ic

@@ -21,16 +21,16 @@ case "${CASE}" in
     export CPL_WAVIC=workflow_C384_refactored
     ;;
   "C768")
-    export CPL_ATMIC=HR2_refactored
-    export CPL_ICEIC=HR1_refactored
-    export CPL_OCNIC=HR1_refactored
-    export CPL_WAVIC=HR1_refactored
+    export CPL_ATMIC=HR3C768
+    export CPL_ICEIC=HR3marine
+    export CPL_OCNIC=HR3marine
+    export CPL_WAVIC=HR3marine
     ;;
   "C1152")
-    export CPL_ATMIC=HR2_C1152_refactored
-    export CPL_ICEIC=HR3_refactored
-    export CPL_OCNIC=HR3_refactored
-    export CPL_WAVIC=HR1_refactored
+    export CPL_ATMIC=HR3C1152
+    export CPL_ICEIC=HR3marine
+    export CPL_OCNIC=HR3marine
+    export CPL_WAVIC=HR3marine
     ;;
   *)
     echo "FATAL ERROR Unrecognized resolution: ${CASE}"

parm/config/gfs/config.ufs

@@ -15,7 +15,7 @@ if (( $# <= 1 )); then
     echo "--fv3 C48|C96|C192|C384|C768|C1152|C3072"
     echo "--mom6 500|100|025"
     echo "--cice6 500|100|025"
-    echo "--ww3 gnh_10m;aoc_9km;gsh_15m|gwes_30m|glo_025|glo_200|glo_500|mx025"
+    echo "--ww3 gnh_10m;aoc_9km;gsh_15m|gwes_30m|glo_025|glo_200|glo_500|mx025|uglo_100km|uglo_m1g16"
     echo "--gocart"
 
     exit 1
@@ -416,6 +416,14 @@ if [[ "${skip_ww3}" == "false" ]]; then
     "mx025")
       ntasks_ww3=80
       ;;
+    "uglo_100km")
+      ntasks_ww3=40
+      nthreads_ww3=1
+      ;;
+    "uglo_m1g16")  
+      ntasks_ww3=1000
+      nthreads_ww3=1
+      ;;
     *)
       echo "FATAL ERROR: Unsupported WW3 resolution = ${ww3_res}, ABORT!"
       exit 1

parm/config/gfs/config.wave

@@ -80,7 +80,19 @@ case "${waveGRD}" in
     export wavepostGRD='glo_500'
     export waveuoutpGRD=${waveGRD}
     ;;
-  *)
+  "uglo_100km") 
+    #unstructured 100km grid 
+    export waveinterpGRD='glo_200'
+    export wavepostGRD=''
+    export waveuoutpGRD=${waveGRD}
+    ;;
+  "uglo_m1g16")
+    #unstructured m1v16 grid 
+    export waveinterpGRD='glo_15mxt'
+    export wavepostGRD=''
+    export waveuoutpGRD=${waveGRD}
+    ;;
+   *)
     echo "FATAL ERROR: No grid specific wave config values exist for ${waveGRD}. Aborting."
     exit 1
     ;;

parm/wave/at_10m_interp.inp.tmpl

@@ -5,7 +5,7 @@ $ Start Time DT NSteps
 $ Total number of grids 
  2
 $ Grid extensions 
-  'gnh_10m'
+  'uglo_m1g16'
   'at_10m'
 $
  0

parm/wave/ep_10m_interp.inp.tmpl

@@ -5,7 +5,7 @@ $ Start Time DT NSteps
 $ Total number of grids 
  2
 $ Grid extensions 
-  'gnh_10m'
+  'uglo_m1g16'
   'ep_10m'
 $
  0

parm/wave/glo_15mxt_interp.inp.tmpl

@@ -3,11 +3,9 @@ $------------------------------------------------
 $ Start Time DT NSteps
  TIME DT NSTEPS
 $ Total number of grids 
- 4
+ 2
 $ Grid extensions 
-  'gnh_10m'
-  'aoc_9km'
-  'gsh_15m' 
+  'uglo_m1g16'
   'glo_15mxt'
 $
  0

parm/wave/glo_200_interp.inp.tmpl

@@ -0,0 +1,12 @@
+$ Input file for interpolation of GLO30m_ext Grid
+$------------------------------------------------
+$ Start Time DT NSteps
+ TIME DT NSTEPS
+$ Total number of grids 
+ 2
+$ Grid extensions 
+  'uglo_100km'
+  'glo_200'
+$
+ 0
+$

parm/wave/glo_30m_interp.inp.tmpl

@@ -3,11 +3,9 @@ $------------------------------------------------
 $ Start Time DT NSteps
  TIME DT NSTEPS
 $ Total number of grids 
- 4
+ 2
 $ Grid extensions 
-  'gnh_10m'
-  'aoc_9km'
-  'gsh_15m' 
+  'uglo_m1g16'
   'glo_30m'
 $
  0

parm/wave/wc_10m_interp.inp.tmpl

@@ -5,7 +5,7 @@ $ Start Time DT NSteps
 $ Total number of grids 
  2
 $ Grid extensions 
-  'gnh_10m'
+  'uglo_m1g16'
   'wc_10m'
 $
  0

scripts/exgfs_wave_init.sh

@@ -118,6 +118,13 @@ source "${HOMEgfs}/ush/preamble.sh"
         err=2;export err;${errchk}
       fi
 
+
+      if [ -f ${FIXwave}/${grdID}.msh ]
+      then
+        cp "${FIXwave}/${grdID}.msh" "${grdID}.msh"
+      fi
+      #TO DO: how do we say "it's unstructured, and therefore need to have error check here" 
+
       [[ ! -d "${COM_WAVE_PREP}" ]] && mkdir -m 775 -p "${COM_WAVE_PREP}"
       if [ ${CFP_MP:-"NO"} = "YES" ]; then
         echo "$nmoddef $USHwave/wave_grid_moddef.sh $grdID > $grdID.out 2>&1" >> cmdfile
@@ -166,7 +173,7 @@ source "${HOMEgfs}/ush/preamble.sh"
       exit=$?
     fi
 
-    if [ "$exit" != '0' ]
+    if [[ "$exit" != '0' ]]
     then
       set +x
       echo ' '
@@ -195,9 +202,9 @@ source "${HOMEgfs}/ush/preamble.sh"
       echo '********************************************** '
       echo '*** FATAL ERROR : NO MODEL DEFINITION FILE *** '
       echo '********************************************** '
-      echo "                                grdID = $grdID"
+      echo "                                grdID = ${grdID}"
       echo ' '
-      sed "s/^/$grdID.out : /g"  $grdID.out
+      sed "s/^/${grdID}.out : /g"  "${grdID}.out"
       set_trace
       err=3;export err;${errchk}
     fi

sorc/build_all.sh

@@ -16,7 +16,7 @@ function _usage() {
 Builds all of the global-workflow components by calling the individual build
   scripts in sequence.
 
-Usage: ${BASH_SOURCE[0]} [-a UFS_app][-c build_config][-h][-j n][-v]
+Usage: ${BASH_SOURCE[0]} [-a UFS_app][-c build_config][-h][-j n][-v][-w]
   -a UFS_app:
     Build a specific UFS app instead of the default
   -g:
@@ -29,6 +29,8 @@ Usage: ${BASH_SOURCE[0]} [-a UFS_app][-c build_config][-h][-j n][-v]
     Build UFS-DA
   -v:
     Execute all build scripts with -v option to turn on verbose where supported
+  -w: 
+    Use unstructured wave grid 
 EOF
   exit 1
 }
@@ -40,17 +42,19 @@ _build_ufs_opt=""
 _build_ufsda="NO"
 _build_gsi="NO"
 _verbose_opt=""
+_wave_unst=""
 _build_job_max=20
 # Reset option counter in case this script is sourced
 OPTIND=1
-while getopts ":a:ghj:uv" option; do
+while getopts ":a:ghj:uvw" option; do
   case "${option}" in
     a) _build_ufs_opt+="-a ${OPTARG} ";;
     g) _build_gsi="YES" ;;
     h) _usage;;
     j) _build_job_max="${OPTARG} ";;
     u) _build_ufsda="YES" ;;
     v) _verbose_opt="-v";;
+    w) _wave_unst="-w";;
     :)
       echo "[${BASH_SOURCE[0]}]: ${option} requires an argument"
       _usage
@@ -113,7 +117,7 @@ declare -A build_opts
 big_jobs=0
 build_jobs["ufs"]=8
 big_jobs=$((big_jobs+1))
-build_opts["ufs"]="${_verbose_opt} ${_build_ufs_opt}"
+build_opts["ufs"]="${_wave_unst} ${_verbose_opt} ${_build_ufs_opt}"
 
 build_jobs["upp"]=6     # The UPP is hardcoded to use 6 cores
 build_opts["upp"]=""
@@ -125,11 +129,11 @@ build_jobs["gfs_utils"]=1
 build_opts["gfs_utils"]="${_verbose_opt}"
 
 build_jobs["ww3prepost"]=3
-build_opts["ww3prepost"]="${_verbose_opt} ${_build_ufs_opt}"
+build_opts["ww3prepost"]="${_wave_unst} ${_verbose_opt} ${_build_ufs_opt}"
 
 # Optional DA builds
 if [[ "${_build_ufsda}" == "YES" ]]; then
-   if [[ "${MACHINE_ID}" != "orion" && "${MACHINE_ID}" != "hera.intel" && "${MACHINE_ID}" != "hercules" ]]; then
+   if [[ "${MACHINE_ID}" != "orion" && "${MACHINE_ID}" != "hera" && "${MACHINE_ID}" != "hercules" ]]; then
       echo "NOTE: The GDAS App is not supported on ${MACHINE_ID}.  Disabling build."
    else
       build_jobs["gdas"]=8

sorc/build_ufs.sh

@@ -7,12 +7,13 @@ cwd=$(pwd)
 APP="S2SWA"
 CCPP_SUITES="FV3_GFS_v17_p8_ugwpv1,FV3_GFS_v17_coupled_p8_ugwpv1"  # TODO: does the g-w need to build with all these CCPP_SUITES?
 
-while getopts ":da:j:v" option; do
+while getopts ":da:j:vw" option; do
   case "${option}" in
     d) BUILD_TYPE="DEBUG";;
     a) APP="${OPTARG}";;
     j) BUILD_JOBS="${OPTARG}";;
     v) export BUILD_VERBOSE="YES";;
+    w) PDLIB="ON";; 
     :)
       echo "[${BASH_SOURCE[0]}]: ${option} requires an argument"
       ;;
@@ -28,6 +29,7 @@ source "./tests/detect_machine.sh"
 source "./tests/module-setup.sh"
 
 MAKE_OPT="-DAPP=${APP} -D32BIT=ON -DCCPP_SUITES=${CCPP_SUITES}"
+[[ ${PDLIB:-"OFF"} = "ON" ]] && MAKE_OPT+=" -DPDLIB=ON"
 [[ ${BUILD_TYPE:-"Release"} = "DEBUG" ]] && MAKE_OPT+=" -DDEBUG=ON"
 COMPILE_NR=0
 CLEAN_BEFORE=YES