Add wave init to GEFS (#2089)

Adds the wave init job to GEFS. Since the output would be the same for
every member, the job is only run once for the control, then links are
created in each of the member directories pointing to that copy.

New ww3_multi and ww3_shell templates for GEFS are copied from the
GFS versions.

The GEFS restart frequency is changed to not be zero, and the comment
about not setting it to zero is copied from the gfs config.base. The restart
interval for the GEFS perturbed members is changed to equal that of the
control.

Also fixes a bug where `RUNwave` was not defined for ensemble forecasts,
and another bug in the GEFS stage_ic dependency.

Resolves #2017

ci/cases/pr/C48_S2SA_gefs.yaml → ci/cases/pr/C48_S2SWA_gefs.yaml

@@ -4,7 +4,7 @@ experiment:
 
 arguments:
   pslot: {{ 'pslot' | getenv }}
-  app: S2SA
+  app: S2SWA
   resdet: 48
   resens: 48
   nens: 2

jobs/JGDAS_ENKF_FCST

@@ -30,6 +30,9 @@ fi
 export ENSEND=$((NMEM_EFCSGRP * 10#${ENSGRP}))
 export ENSBEG=$((ENSEND - NMEM_EFCSGRP + 1))
 
+if [[ ${DO_WAVE} == "YES" ]]; then
+  declare -rx RUNwave="${RUN}wave"
+fi
 
 ###############################################################
 # Run relevant script

parm/config/gefs/config.base.emc.dyn

@@ -242,7 +242,10 @@ export FHMAX_ENKF=${FHMAX_GFS}
 export FHOUT_ENKF=${FHOUT_GFS}
 
 # GFS restart interval in hours
-export restart_interval_gfs=0
+export restart_interval_gfs=12
+# NOTE: Do not set this to zero.  Instead set it to $FHMAX_GFS
+# TODO: Remove this variable from config.base and reference from config.fcst
+# TODO: rework logic in config.wave and push it to parsing_nameslist_WW3.sh where it is actually used
 
 export QUILTING=".true."
 export OUTPUT_GRID="gaussian_grid"

parm/config/gefs/config.efcs

@@ -54,6 +54,6 @@ export SPPT_LSCALE=500000.
 export SPPT_LOGIT=".true."
 export SPPT_SFCLIMIT=".true."
 
-export restart_interval=${restart_interval:-6}
+export restart_interval=${restart_interval_gfs}
 
 echo "END: config.efcs"

parm/config/gefs/config.wave

@@ -0,0 +1,185 @@
+#! /usr/bin/env bash
+
+########## config.wave ##########
+# Wave steps specific
+
+echo "BEGIN: config.wave"
+
+# Parameters that are common to all wave model steps
+
+# System and version
+export wave_sys_ver=v1.0.0
+
+export EXECwave="${HOMEgfs}/exec"
+export FIXwave="${HOMEgfs}/fix/wave"
+export PARMwave="${HOMEgfs}/parm/wave"
+export USHwave="${HOMEgfs}/ush"
+
+# This config contains variables/parameters used in the fcst step
+# Some others are also used across the workflow in wave component scripts
+
+# General runtime labels
+export CDUMPwave="${RUN}wave"
+
+# In GFS/GDAS, restart files are generated/read from gdas runs
+export CDUMPRSTwave="gdas"
+
+# Grids for wave model
+export waveGRD=${waveGRD:-'mx025'}
+
+#grid dependent variable defaults
+export waveGRDN='1'                   # grid number for ww3_multi
+export waveGRDG='10'                  # grid group for ww3_multi
+export USE_WAV_RMP='NO'               # YES/NO rmp grid remapping pre-processed coefficients
+export waveMULTIGRID='.false.'        # .true./.false. for multi or shel
+export MESH_WAV="mesh.${waveGRD}.nc"  # Mesh grid for wave model for CMEPS
+export waveesmfGRD=' '                # input grid for multigrid
+
+#Grid dependent variables for various grids
+case "${waveGRD}" in
+  "gnh_10m;aoc_9km;gsh_15m")
+    #GFSv16 settings:
+    export waveGRDN='1 2 3'
+    export waveGRDG='10 20 30'
+    export USE_WAV_RMP='YES'
+    export waveMULTIGRID='.true.'
+    export IOSRV='3'
+    export MESH_WAV=' '
+    export waveesmfGRD='glox_10m'
+    export waveuoutpGRD='points'
+    export waveinterpGRD='glo_15mxt at_10m ep_10m wc_10m glo_30m'
+    export wavepostGRD='gnh_10m aoc_9km gsh_15m'
+    ;;
+  "gwes_30m")
+    #Grid used for P8
+    export waveinterpGRD=''
+    export wavepostGRD='gwes_30m'
+    export waveuoutpGRD=${waveGRD}
+    ;;
+  "mx025")
+    #Grid used for HR1 (tripolar 1/4 deg)
+    export waveinterpGRD='reg025'
+    export wavepostGRD=''
+    export waveuoutpGRD=${waveGRD}
+    ;;
+  "glo_025")
+    #GEFSv13 regular lat/lon 1/4 deg grid
+    export waveinterpGRD=''
+    export wavepostGRD='glo_025'
+    export waveuoutpGRD=${waveGRD}
+    ;;
+  "glo_200")
+    #Global regular lat/lon 2deg deg grid
+    export waveinterpGRD=''
+    export wavepostGRD='glo_200'
+    export waveuoutpGRD=${waveGRD}
+    ;;
+  "glo_500")
+    #Global regular lat/lon 5deg deg grid
+    export waveinterpGRD=''
+    export wavepostGRD='glo_500'
+    export waveuoutpGRD=${waveGRD}
+    ;;
+  *)
+    echo "FATAL ERROR: No grid specific wave config values exist for ${waveGRD}. Aborting."
+    exit 1
+    ;;
+esac
+
+# Grids for input wind fields
+export WAVEWND_DID=
+export WAVEWND_FID=
+
+# The start time reflects the number of hindcast hours prior to the cycle initial time
+export FHMAX_WAV=${FHMAX_GFS}
+export WAVHINDH=0
+export FHMIN_WAV=0
+export FHOUT_WAV=3
+export FHMAX_HF_WAV=120
+export FHOUT_HF_WAV=1
+export FHMAX_WAV_IBP=180
+if (( FHMAX_WAV < FHMAX_WAV_IBP )); then export FHMAX_WAV_IBP=${FHMAX_GFS} ; fi
+
+# gridded and point output rate
+export DTFLD_WAV=$(( FHOUT_HF_WAV * 3600 ))
+export DTPNT_WAV=3600
+export FHINCP_WAV=$(( DTPNT_WAV / 3600 ))
+
+# Selected output parameters (gridded)
+export OUTPARS_WAV="WND CUR ICE HS T01 T02 DIR FP DP PHS PTP PDIR CHA"
+
+# Restart file config
+export WAVHCYC=${assim_freq:-6}
+export FHMAX_WAV_CUR=192 # RTOFS forecasts only out to 8 days
+
+# Restart timing business
+
+export RSTTYPE_WAV='T'          # generate second tier of restart files
+rst_dt_gfs=$(( restart_interval_gfs * 3600 ))  # TODO: This calculation needs to move to parsing_namelists_WW3.sh
+if [[ ${rst_dt_gfs} -gt 0 ]]; then
+  export DT_1_RST_WAV=0 #${rst_dt_gfs:-0}   # time between restart files, set to DTRST=1 for a single restart file
+                                        #temporarily set to zero to avoid a clash in requested restart times 
+                                        #which makes the wave model crash a fix for the model issue will be coming
+  export DT_2_RST_WAV=${rst_dt_gfs:-0}   # restart stride for checkpointing restart
+else
+  rst_dt_fhmax=$(( FHMAX_WAV * 3600 ))
+  export DT_1_RST_WAV=0                    # time between restart files, set to DTRST=1 for a single restart file
+  export DT_2_RST_WAV=${rst_dt_fhmax:-0}   # use checkpoint restart file name for creating restart at end of run
+fi
+export RSTIOFF_WAV=0                   # first restart file offset relative to model start
+#
+# Set runmember to default value if not GEFS cpl run
+#  (for a GFS coupled run, RUNMEN would be unset, this should default to -1)
+export RUNMEM=${RUNMEM:--1}
+# Set wave model member tags if ensemble run
+# -1: no suffix, deterministic; xxxNN: extract two last digits to make ofilename prefix=gwesNN
+if [[ ${RUNMEM} = -1 ]]; then
+# No suffix added to model ID in case of deterministic run
+  export waveMEMB=
+else
+# Extract member number only
+  export waveMEMB="${RUNMEM: -2}"
+fi
+
+# Determine if wave component needs input and/or is coupled
+export WW3ATMINP='CPL'
+if [[ ${DO_ICE} == "YES" ]]; then
+  export WW3ICEINP='CPL'
+  export WAVEICE_FID=
+else
+  export WW3ICEINP='YES'
+  export WAVEICE_FID=glix_10m
+fi
+
+export WAVECUR_DID=rtofs
+if [[ ${DO_OCN} == "YES" ]]; then
+  export WW3CURINP='CPL'
+  export WAVECUR_FID=
+else
+  export WW3CURINP='YES'
+  export WAVECUR_FID=glix_10m
+fi
+
+# Determine if input is from perturbed ensemble (T) or single input file (F) for all members
+export WW3ATMIENS='F'
+export WW3ICEIENS='F'
+export WW3CURIENS='F'
+
+export GOFILETYPE=1     # GOFILETYPE=1 one gridded file per output step
+export POFILETYPE=1     # POFILETYPE=1 one point file per output step
+
+# Parameters for ww3_multi/shel.inp
+# Unified output T or F
+export FUNIPNT='T'
+# Output server type (see ww3_shel/multi.inp in WW3 repo)
+export IOSRV=${IOSRV:-'1'}
+# Flag for dedicated output process for unified points
+export FPNTPROC='T'
+# Flag for grids sharing dedicated output processes
+export FGRDPROC='F'
+# Flag for masking computation in two-way nesting
+export FLAGMASKCOMP="F"
+# Flag for masking at printout time.
+export FLAGMASKOUT="F"
+
+echo "END: config.wave"

parm/config/gefs/config.waveinit

@@ -0,0 +1 @@
+../gfs/config.waveinit

scripts/exgfs_wave_init.sh

@@ -203,6 +203,17 @@ source "${HOMEgfs}/ush/preamble.sh"
     fi
   done
 
+# Copy to other members if needed
+if (( NMEM_ENS > 0 )); then
+  for mem in $(seq -f "%03g" 1 "${NMEM_ENS}"); do
+    MEMDIR="mem${mem}" YMD=${PDY} HH=${cyc} generate_com COM_WAVE_PREP_MEM:COM_WAVE_PREP_TMPL
+    mkdir -p "${COM_WAVE_PREP_MEM}"
+    for grdID in ${grdALL}; do
+      ${NLN} "${COM_WAVE_PREP}/${RUN}wave.mod_def.${grdID}" "${COM_WAVE_PREP_MEM}/"
+    done
+  done
+fi
+
 # --------------------------------------------------------------------------- #
 # 2.  Ending
 

ush/parsing_namelists_WW3.sh

@@ -79,17 +79,17 @@ WW3_namelists(){
 
   if [ $waveMULTIGRID = ".true." ]; then
     # ww3_multi template
-    if [ -f $PARMwave/ww3_multi.${NET}.inp.tmpl ]; then
-      cp $PARMwave/ww3_multi.${NET}.inp.tmpl ww3_multi.inp.tmpl
+    if [ -f $PARMwave/ww3_multi.inp.tmpl ]; then
+      cp $PARMwave/ww3_multi.inp.tmpl ww3_multi.inp.tmpl
     fi
     if [ ! -f ww3_multi.inp.tmpl ]; then
       echo "ABNORMAL EXIT: NO TEMPLATE FOR WW3 MULTI INPUT FILE" 
       exit 11 
     fi
   else 
     # ww3_multi template
-    if [ -f $PARMwave/ww3_shel.${NET}.inp.tmpl ]; then
-      cp $PARMwave/ww3_shel.${NET}.inp.tmpl ww3_shel.inp.tmpl
+    if [ -f $PARMwave/ww3_shel.inp.tmpl ]; then
+      cp $PARMwave/ww3_shel.inp.tmpl ww3_shel.inp.tmpl
     fi
     if [ ! -f ww3_shel.inp.tmpl ]; then
       echo "ABNORMAL EXIT: NO TEMPLATE FOR WW3 SHEL INPUT FILE" 

workflow/applications/gefs.py

@@ -19,6 +19,9 @@ def _get_app_configs(self):
         if self.nens > 0:
             configs += ['efcs']
 
+        if self.do_wave:
+            configs += ['waveinit']
+
         return configs
 
     @staticmethod
@@ -32,7 +35,12 @@ def _update_base(base_in):
 
     def get_task_names(self):
 
-        tasks = ['stage_ic', 'fcst']
+        tasks = ['stage_ic']
+
+        if self.do_wave:
+            tasks += ['waveinit']
+
+        tasks += ['fcst']
 
         if self.nens > 0:
             tasks += ['efcs']

workflow/rocoto/gefs_tasks.py

@@ -50,7 +50,7 @@ def stage_ic(self):
         if self.app_config.do_wave:
             prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_WAVIC']}/@Y@m@d@H/mem000/wave"
             for wave_grid in self._configs['waveinit']['waveGRD'].split():
-                data = f"{prefix}/{wave_grid}/@Y@m@d.@H0000.restart.{wave_grid}"
+                data = f"{prefix}/@Y@m@d.@H0000.restart.{wave_grid}"
                 dep_dict = {'type': 'data', 'data': data}
                 deps.append(rocoto.add_dependency(dep_dict))
 
@@ -61,12 +61,25 @@ def stage_ic(self):
 
         return task
 
+    def waveinit(self):
+
+        resources = self.get_resource('waveinit')
+        task = create_wf_task('waveinit', resources, cdump=self.cdump, envar=self.envars, dependency=None)
+
+        return task
+
     def fcst(self):
         # TODO: Add real dependencies
         dependencies = []
         dep_dict = {'type': 'task', 'name': f'{self.cdump}stage_ic'}
         dependencies.append(rocoto.add_dependency(dep_dict))
 
+        if self.app_config.do_wave:
+            dep_dict = {'type': 'task', 'name': f'{self.cdump}waveinit'}
+            dependencies.append(rocoto.add_dependency(dep_dict))
+
+        dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies)
+
         resources = self.get_resource('fcst')
         task = create_wf_task('fcst', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies)
 
@@ -77,6 +90,12 @@ def efcs(self):
         dep_dict = {'type': 'task', 'name': f'{self.cdump}stage_ic'}
         dependencies.append(rocoto.add_dependency(dep_dict))
 
+        if self.app_config.do_wave:
+            dep_dict = {'type': 'task', 'name': f'{self.cdump}waveinit'}
+            dependencies.append(rocoto.add_dependency(dep_dict))
+
+        dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies)
+
         efcsenvars = self.envars.copy()
         efcsenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#'))