Add ability to use GEFS replay ICs (#2559)

The PR allows the use of ICs from PSL's replay analysis. These replay
ICs will be used for GEFS reforecasting and SFS. Two main changes are
associated with these updates: (1) replay ICs being valid at 3Z, and (2)
the use of warm starts.

Resolves #1838

---------

Co-authored-by: Jessica Meixner <Jessica.Meixner@noaa.gov>
Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov>
Co-authored-by: Rahul Mahajan <aerorahul@users.noreply.github.com>

parm/config/gefs/config.base

@@ -294,12 +294,17 @@ export MEMDIR="mem${ENSMEM}"
 
 # initialize ocean ensemble members with perturbations
 #   if true, only occurs for members greater than zero
-export USE_OCN_PERTURB_FILES=@USE_OCN_PERTURB_FILES@
+export REPLAY_ICS=@REPLAY_ICS@
+if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then
+    export OFFSET_START_HOUR=$(( assim_freq / 2 ))
+else
+    export OFFSET_START_HOUR=0
+fi
 
 export DOIAU="NO"  # While we are not doing IAU, we may want to warm start w/ IAU in the future
 # Check if cycle is cold starting
 if [[ "${EXP_WARM_START}" = ".false." ]]; then
-  export IAU_FHROT=0
+  export IAU_FHROT=${OFFSET_START_HOUR}
 else
   if [[ "${DOIAU}" = "YES" ]]; then
     export IAU_FHROT=3

parm/config/gefs/config.efcs

@@ -67,7 +67,7 @@ export EPBL="0.8,0.4,0.2,0.08,0.04"
 export EPBL_TAU="2.16E4,2.592E5,2.592E6,7.776E6,3.1536E7"
 export EPBL_LSCALE="500.E3,1000.E3,2000.E3,2000.E3,2000.E3"
 
-if [[ "${USE_OCN_PERTURB_FILES:-false}" == "true" ]]; then
+if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then
     export ODA_INCUPD="True"
     export ODA_TEMPINC_VAR='t_pert'
     export ODA_SALTINC_VAR='s_pert'

parm/config/gefs/config.fcst

@@ -256,6 +256,9 @@ else
   export io_layout="1,1"
 fi
 
+if (( OFFSET_START_HOUR != 0 )); then
+    export reforecast="YES"
+fi
 # Remember config.efcs will over-ride these values for ensemble forecasts
 # if these variables are re-defined there.
 # Otherwise, the ensemble forecast will inherit from config.fcst

parm/config/gefs/config.stage_ic

@@ -13,12 +13,21 @@ case "${CASE}" in
     export CPL_ICEIC=""
     export CPL_OCNIC=""
     export CPL_WAVIC=""
+    export CPL_MEDIC=""
+    ;;
+  "C96")
+    export CPL_ATMIC=""
+    export CPL_ICEIC=""
+    export CPL_OCNIC=""
+    export CPL_WAVIC=""
+    export CPL_MEDIC=""
     ;;
   "C48")
     export CPL_ATMIC="gefs_test"
     export CPL_ICEIC="gefs_test"
     export CPL_OCNIC="gefs_test"
     export CPL_WAVIC="gefs_test"
+    export CPL_MEDIC="gefs_test"
     ;;
   *)
     echo "FATAL ERROR Unrecognized resolution: ${CASE}"

parm/config/gefs/config.ufs

@@ -324,7 +324,7 @@ if [[ "${skip_mom6}" == "false" ]]; then
       MOM6_RESTART_SETTING='r'
       MOM6_RIVER_RUNOFF='False'
       eps_imesh="2.5e-1"
-      TOPOEDITS="ufs.topo_edits_011818.nc"
+      TOPOEDITS="topo_edits_011818.nc"
       if [[ "${DO_JEDIOCNVAR:-NO}" = "YES" ]]; then
         MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc"
         MOM6_DIAG_MISVAL="0.0"

parm/config/gefs/yaml/defaults.yaml

@@ -9,5 +9,5 @@ base:
   DO_AWIPS: "NO"
   KEEPDATA: "NO" 
   FHMAX_GFS: 120
-  USE_OCN_PERTURB_FILES: "false"
+  REPLAY_ICS: "NO"
 

parm/config/gfs/config.base

@@ -470,6 +470,10 @@ if [[ ${DO_JEDIATMVAR} = "YES" ]]; then
    export DO_VMINMON="NO"     # GSI minimization monitoring
 fi
 
+# If starting ICs that are not at cycle hour
+export REPLAY_ICS="NO"
+export OFFSET_START_HOUR=0
+
 # Number of regional collectives to create soundings for
 export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9}
 

scripts/exgfs_wave_post_pnt.sh

@@ -247,8 +247,9 @@ source "${USHgfs}/preamble.sh"
         -e "s/FORMAT/F/g" \
                                ww3_outp_spec.inp.tmpl > ww3_outp.inp
 
-    ${NLN} mod_def.$waveuoutpGRD mod_def.ww3
-    HMS="${cyc}0000"
+    ${NLN} mod_def.${waveuoutpGRD} mod_def.ww3
+    HH=$(date --utc -d "${PDY:0:8} ${cyc} + ${FHMIN_WAV} hours" +%H)
+    HMS="${HH}0000"
     if [[ -f "${COM_WAVE_HISTORY}/${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${PDY}.${HMS}" ]]; then
       ${NLN} "${COM_WAVE_HISTORY}/${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${PDY}.${HMS}" \
         "./out_pnt.${waveuoutpGRD}"

scripts/exglobal_stage_ic.sh

@@ -8,6 +8,9 @@ GDATE=$(date --utc -d "${PDY} ${cyc} - ${assim_freq} hours" +%Y%m%d%H)
 gPDY="${GDATE:0:8}"
 gcyc="${GDATE:8:2}"
 
+RDATE=$(date --utc -d "${PDY} ${cyc} + ${OFFSET_START_HOUR} hours" +%Y%m%d%H)
+DTG_PREFIX="${RDATE:0:8}.${RDATE:8:2}0000"
+
 MEMDIR_ARRAY=()
 if [[ "${RUN:-}" = "gefs" ]]; then
   # Populate the member_dirs array based on the value of NMEM_ENS
@@ -33,21 +36,22 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do
     # Stage the FV3 restarts to ROTDIR (warm start)
     RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL
     [[ ! -d "${COM_ATMOS_RESTART_PREV}" ]] && mkdir -p "${COM_ATMOS_RESTART_PREV}"
-    for ftype in coupler.res fv_core.res.nc; do
-      src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${PDY}.${cyc}0000.${ftype}"
-      tgt="${COM_ATMOS_RESTART_PREV}/${PDY}.${cyc}0000.${ftype}"
+    prev_atmos_copy_list=(fv_core.res.nc coupler.res)
+    for ftype in "${prev_atmos_copy_list[@]}"; do
+      src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${DTG_PREFIX}.${ftype}"
+      tgt="${COM_ATMOS_RESTART_PREV}/${DTG_PREFIX}.${ftype}"
       ${NCP} "${src}" "${tgt}"
       rc=$?
       ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
       err=$((err + rc))
     done
     for ftype in ca_data fv_core.res fv_srf_wnd.res fv_tracer.res phy_data sfc_data; do
       for ((tt = 1; tt <= ntiles; tt++)); do
-        src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${PDY}.${cyc}0000.${ftype}.tile${tt}.nc"
+        src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${DTG_PREFIX}.${ftype}.tile${tt}.nc"
         if (( tt > 6 )) ; then
-            tgt="${COM_ATMOS_RESTART_PREV}/${PDY}.${cyc}0000.${ftype}.nest0$((tt-5)).tile${tt}.nc"
+            tgt="${COM_ATMOS_RESTART_PREV}/${DTG_PREFIX}.${ftype}.nest0$((tt-5)).tile${tt}.nc"
         else
-            tgt="${COM_ATMOS_RESTART_PREV}/${PDY}.${cyc}0000.${ftype}.tile${tt}.nc"
+            tgt="${COM_ATMOS_RESTART_PREV}/${DTG_PREFIX}.${ftype}.tile${tt}.nc"
         fi
         ${NCP} "${src}" "${tgt}"
         rc=$?
@@ -79,13 +83,26 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do
       fi
     done
   fi
+
+  # Atmosphere Perturbation Files (usually used with replay ICS)
+  # Extra zero on MEMDIR ensure we have a number even if the string is empty
+  if (( 0${MEMDIR:3} > 0 )) && [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then
+      YMD=${PDY} HH=${cyc} declare_from_tmpl COM_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL
+      [[ ! -d "${COM_ATMOS_ANALYSIS}" ]] && mkdir -p "${COM_ATMOS_ANALYSIS}"
+      src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${DTG_PREFIX}.fv3_perturbation.nc"
+      tgt="${COM_ATMOS_ANALYSIS}/${RUN}.t00z.atminc.nc"
+      ${NCP} "${src}" "${tgt}"
+      rc=${?}
+      ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
+      err=$((err + rc))
+  fi
 
   # Stage ocean initial conditions to ROTDIR (warm start)
   if [[ "${DO_OCN:-}" = "YES" ]]; then
     RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_OCEAN_RESTART_PREV:COM_OCEAN_RESTART_TMPL
     [[ ! -d "${COM_OCEAN_RESTART_PREV}" ]] && mkdir -p "${COM_OCEAN_RESTART_PREV}"
-    src="${BASE_CPLIC}/${CPL_OCNIC:-}/${PDY}${cyc}/${MEMDIR}/ocean/${PDY}.${cyc}0000.MOM.res.nc"
-    tgt="${COM_OCEAN_RESTART_PREV}/${PDY}.${cyc}0000.MOM.res.nc"
+    src="${BASE_CPLIC}/${CPL_OCNIC:-}/${PDY}${cyc}/${MEMDIR}/ocean/${DTG_PREFIX}.MOM.res.nc"
+    tgt="${COM_OCEAN_RESTART_PREV}/${DTG_PREFIX}.MOM.res.nc"
     ${NCP} "${src}" "${tgt}"
     rc=$?
     ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
@@ -96,8 +113,8 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do
         ;;
       "025" )
         for nn in $(seq 1 3); do
-          src="${BASE_CPLIC}/${CPL_OCNIC:-}/${PDY}${cyc}/${MEMDIR}/ocean/${PDY}.${cyc}0000.MOM.res_${nn}.nc"
-          tgt="${COM_OCEAN_RESTART_PREV}/${PDY}.${cyc}0000.MOM.res_${nn}.nc"
+          src="${BASE_CPLIC}/${CPL_OCNIC:-}/${PDY}${cyc}/${MEMDIR}/ocean/${DTG_PREFIX}.MOM.res_${nn}.nc"
+          tgt="${COM_OCEAN_RESTART_PREV}/${DTG_PREFIX}.MOM.res_${nn}.nc"
           ${NCP} "${src}" "${tgt}"
           rc=$?
           ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
@@ -113,9 +130,11 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do
 
     # Ocean Perturbation Files
     # Extra zero on MEMDIR ensure we have a number even if the string is empty
-    if (( 0${MEMDIR:3} > 0 )) && [[ "${USE_OCN_PERTURB_FILES:-false}" == "true" ]]; then
-        src="${BASE_CPLIC}/${CPL_OCNIC:-}/${PDY}${cyc}/${MEMDIR}/ocean/${PDY}.${cyc}0000.mom6_increment.nc"
-        tgt="${COM_OCEAN_RESTART_PREV}/${PDY}.${cyc}0000.mom6_increment.nc"
+    if (( 0${MEMDIR:3} > 0 )) && [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then
+        YMD=${PDY} HH=${cyc} declare_from_tmpl COM_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL
+        [[ ! -d "${COM_OCEAN_ANALYSIS}" ]] && mkdir -p "${COM_OCEAN_ANALYSIS}"
+        src="${BASE_CPLIC}/${CPL_OCNIC:-}/${PDY}${cyc}/${MEMDIR}/ocean/${DTG_PREFIX}.mom6_perturbation.nc"
+        tgt="${COM_OCEAN_ANALYSIS}/mom6_increment.nc"
         ${NCP} "${src}" "${tgt}"
         rc=${?}
         ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
@@ -128,8 +147,8 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do
       # Stage the mediator restarts to ROTDIR (warm start/restart the coupled model)
       RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_MED_RESTART_PREV:COM_MED_RESTART_TMPL
       [[ ! -d "${COM_MED_RESTART_PREV}" ]] && mkdir -p "${COM_MED_RESTART_PREV}"
-      src="${BASE_CPLIC}/${CPL_MEDIC:-}/${PDY}${cyc}/${MEMDIR}/med/${PDY}.${cyc}0000.ufs.cpld.cpl.r.nc"
-      tgt="${COM_MED_RESTART_PREV}/${PDY}.${cyc}0000.ufs.cpld.cpl.r.nc"
+      src="${BASE_CPLIC}/${CPL_MEDIC:-}/${PDY}${cyc}/${MEMDIR}/med/${DTG_PREFIX}.ufs.cpld.cpl.r.nc"
+      tgt="${COM_MED_RESTART_PREV}/${DTG_PREFIX}.ufs.cpld.cpl.r.nc"
       if [[ -f "${src}" ]]; then
         ${NCP} "${src}" "${tgt}"
         rc=$?
@@ -146,8 +165,8 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do
   if [[ "${DO_ICE:-}" = "YES" ]]; then
     RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_ICE_RESTART_PREV:COM_ICE_RESTART_TMPL
     [[ ! -d "${COM_ICE_RESTART_PREV}" ]] && mkdir -p "${COM_ICE_RESTART_PREV}"
-    src="${BASE_CPLIC}/${CPL_ICEIC:-}/${PDY}${cyc}/${MEMDIR}/ice/${PDY}.${cyc}0000.cice_model.res.nc"
-    tgt="${COM_ICE_RESTART_PREV}/${PDY}.${cyc}0000.cice_model.res.nc"
+    src="${BASE_CPLIC}/${CPL_ICEIC:-}/${PDY}${cyc}/${MEMDIR}/ice/${DTG_PREFIX}.cice_model.res.nc"
+    tgt="${COM_ICE_RESTART_PREV}/${DTG_PREFIX}.cice_model.res.nc"
     ${NCP} "${src}" "${tgt}"
     rc=$?
     ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
@@ -156,11 +175,11 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do
 
   # Stage the WW3 initial conditions to ROTDIR (warm start; TODO: these should be placed in $RUN.$gPDY/$gcyc)
   if [[ "${DO_WAVE:-}" = "YES" ]]; then
-    YMD=${PDY} HH=${cyc} declare_from_tmpl COM_WAVE_RESTART
-    [[ ! -d "${COM_WAVE_RESTART}" ]] && mkdir -p "${COM_WAVE_RESTART}"
+    YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_WAVE_RESTART_PREV:COM_WAVE_RESTART_TMPL
+    [[ ! -d "${COM_WAVE_RESTART_PREV}" ]] && mkdir -p "${COM_WAVE_RESTART_PREV}"
     for grdID in ${waveGRD}; do # TODO: check if this is a bash array; if so adjust
-      src="${BASE_CPLIC}/${CPL_WAVIC:-}/${PDY}${cyc}/${MEMDIR}/wave/${PDY}.${cyc}0000.restart.${grdID}"
-      tgt="${COM_WAVE_RESTART}/${PDY}.${cyc}0000.restart.${grdID}"
+      src="${BASE_CPLIC}/${CPL_WAVIC:-}/${PDY}${cyc}/${MEMDIR}/wave/${DTG_PREFIX}.restart.${grdID}"
+      tgt="${COM_WAVE_RESTART_PREV}/${DTG_PREFIX}.restart.${grdID}"
       ${NCP} "${src}" "${tgt}"
       rc=$?
       ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"