Add a rotating subset of members for early-cycle enkf (#2427)

The early-cycle EnKF needs the ability to run with fewer members than
the late-cycle due to operational resource constraints. Because of this
requirement, the introduction of a rotating subset of member first-guess
states used by the early-cycle ensemble is also needed in order to
preserve the rotating member initial condition functionality currently
used by the GEFS.

Co-authored-by: Travis J Elless <telless@Orion-login-1.HPC.MsState.Edu>
Co-authored-by: travis elless <travis.j.elless@dlogin07.dogwood.wcoss2.ncep.noaa.gov>

jobs/JGLOBAL_FORECAST

@@ -13,7 +13,7 @@ fi
 
 # Restart conditions for GFS cycle come from GDAS
 rCDUMP=${RUN}
-[[ ${RUN} == "gfs" ]] && export rCDUMP="gdas"
+export rCDUMP="${RUN/gfs/gdas}"
 
 # Ignore possible spelling error (nothing is misspelled)
 # shellcheck disable=SC2153

parm/config/gfs/config.base

@@ -233,7 +233,7 @@ export FHOUT=3           # Will be changed to 1 in config.base if (DOHYBVAR set
 export FHOUT_OCNICE=3
 
 # Cycle to run EnKF  (set to BOTH for both gfs and gdas)
-export EUPD_CYC="gdas"
+export EUPD_CYC="@EUPD_CYC@"
 
 # GFS cycle info
 export gfs_cyc=@gfs_cyc@ # 0: no GFS cycle, 1: 00Z only, 2: 00Z and 12Z only, 4: all 4 cycles.
@@ -295,16 +295,20 @@ export DO_MERGENSST="@DO_MERGENSST@"
 # Hybrid related
 export DOHYBVAR="@DOHYBVAR@"
 export NMEM_ENS=@NMEM_ENS@
-export NMEM_ENS_GFS=@NMEM_ENS@
 export SMOOTH_ENKF="NO"
 export l4densvar=".true."
 export lwrite4danl=".true."
 
+# Early-cycle EnKF parameters
+export NMEM_ENS_GFS=30
+export NMEM_ENS_GFS_OFFSET=20
+export DO_CALC_INCREMENT_ENKF_GFS="NO"
+
 # EnKF output frequency
 if [[ ${DOHYBVAR} = "YES" ]]; then
     export FHMIN_ENKF=3
     export FHMAX_ENKF=9
-    export FHMAX_ENKF_GFS=120
+    export FHMAX_ENKF_GFS=@FHMAX_ENKF_GFS@
     export FHOUT_ENKF_GFS=3
     if [[ ${l4densvar} = ".true." ]]; then
         export FHOUT=1
@@ -349,9 +353,6 @@ export MAKE_ACFTBUFR="@MAKE_ACFTBUFR@"
 # Analysis increments to zero in CALCINCEXEC
 export INCREMENTS_TO_ZERO="'liq_wat_inc','icmr_inc','rwmr_inc','snmr_inc','grle_inc'"
 
-# Write analysis files for early cycle EnKF
-export DO_CALC_INCREMENT_ENKF_GFS="YES"
-
 # Stratospheric increments to zero
 export INCVARS_ZERO_STRAT="'sphum_inc','liq_wat_inc','icmr_inc','rwmr_inc','snmr_inc','grle_inc'"
 export INCVARS_EFOLD="5"

parm/config/gfs/yaml/defaults.yaml

@@ -9,6 +9,8 @@ base:
   FHMAX_GFS: 120
   DO_VRFY_OCEANDA: "NO"
   GSI_SOILANAL: "NO"
+  EUPD_CYC: "gdas"
+  FHMAX_ENKF_GFS: 12
 
 atmanl:
   LAYOUT_X_ATMANL: 8

scripts/exgdas_enkf_ecen.sh

@@ -51,18 +51,22 @@ GPREFIX=${GPREFIX:-""}
 GPREFIX_ENS=${GPREFIX_ENS:-$GPREFIX}
 
 # Variables
-NMEM_ENS=${NMEM_ENS:-80}
 imp_physics=${imp_physics:-99}
 INCREMENTS_TO_ZERO=${INCREMENTS_TO_ZERO:-"'NONE'"}
 DOIAU=${DOIAU_ENKF:-"NO"}
 FHMIN=${FHMIN_ECEN:-3}
 FHMAX=${FHMAX_ECEN:-9}
 FHOUT=${FHOUT_ECEN:-3}
 FHSFC=${FHSFC_ECEN:-$FHMIN}
-if [ $RUN = "enkfgfs" ]; then
+if [ "${RUN}" = "enkfgfs" ]; then
    DO_CALC_INCREMENT=${DO_CALC_INCREMENT_ENKF_GFS:-"NO"}
+   NMEM_ENS=${NMEM_ENS_GFS:-30}
+   ec_offset=${NMEM_ENS_GFS_OFFSET:-20}
+   mem_offset=$((ec_offset * cyc/6))
 else
    DO_CALC_INCREMENT=${DO_CALC_INCREMENT:-"NO"}
+   NMEM_ENS=${NMEM_ENS:-80}
+   mem_offset=0
 fi
 
 # global_chgres stuff
@@ -106,12 +110,17 @@ ENKF_SUFFIX="s"
 for FHR in $(seq $FHMIN $FHOUT $FHMAX); do
 
 for imem in $(seq 1 $NMEM_ENS); do
+   smem=$((imem + mem_offset))
+   if (( smem > 80 )); then
+      smem=$((smem - 80))
+   fi
+   gmemchar="mem"$(printf %03i $smem)
    memchar="mem"$(printf %03i $imem)
 
    MEMDIR=${memchar} YMD=${PDY} HH=${cyc} generate_com -x \
       COM_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL
 
-   MEMDIR=${memchar} RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} generate_com -x \
+   MEMDIR=${gmemchar} RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} generate_com -x \
       COM_ATMOS_HISTORY_MEM_PREV:COM_ATMOS_HISTORY_TMPL
 
    ${NLN} "${COM_ATMOS_HISTORY_MEM_PREV}/${GPREFIX_ENS}atmf00${FHR}${ENKF_SUFFIX}.nc" "./atmges_${memchar}"

scripts/exgdas_enkf_post.sh

@@ -46,10 +46,11 @@ FHMIN=${FHMIN_EPOS:-3}
 FHMAX=${FHMAX_EPOS:-9}
 FHOUT=${FHOUT_EPOS:-3}
 
-if [[ $CDUMP == "gfs" ]]; then
+if [[ "${RUN}" == "enkfgfs" ]]; then
    NMEM_ENS=${NMEM_ENS_GFS:-${NMEM_ENS:-30}}
+else
+   NMEM_ENS=${NMEM_ENS:-80}
 fi
-NMEM_ENS=${NMEM_ENS:-80}
 SMOOTH_ENKF=${SMOOTH_ENKF:-"NO"}
 ENKF_SPREAD=${ENKF_SPREAD:-"NO"}
 

scripts/exgdas_enkf_sfc.sh

@@ -47,7 +47,14 @@ GPREFIX=${GPREFIX:-""}
 GPREFIX_ENS=${GPREFIX_ENS:-${GPREFIX}}
 
 # Variables
-NMEM_ENS=${NMEM_ENS:-80}
+if [ "${RUN}" = "enkfgfs" ]; then
+   NMEM_ENS=${NMEM_ENS_GFS:-30}
+   ec_offset=${NMEM_ENS_GFS_OFFSET:-20}
+   mem_offset=$((ec_offset * cyc/6))
+else
+   NMEM_ENS=${NMEM_ENS:-80}
+   mem_offset=0
+fi
 DOIAU=${DOIAU_ENKF:-"NO"}
 
 # Global_cycle stuff
@@ -132,14 +139,18 @@ if [ $DOIAU = "YES" ]; then
         export TILE_NUM=$n
 
         for imem in $(seq 1 $NMEM_ENS); do
-
+            smem=$((imem + mem_offset))
+            if (( smem > 80 )); then
+               smem=$((smem - 80))
+            fi
+            gmemchar="mem"$(printf %03i "$smem")
             cmem=$(printf %03i $imem)
             memchar="mem$cmem"
 
             MEMDIR=${memchar} YMD=${PDY} HH=${cyc} generate_com \
                 COM_ATMOS_RESTART_MEM:COM_ATMOS_RESTART_TMPL
 
-            MEMDIR=${memchar} RUN="enkfgdas" YMD=${gPDY} HH=${gcyc} generate_com \
+            MEMDIR=${gmemchar} RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} generate_com \
                 COM_ATMOS_RESTART_MEM_PREV:COM_ATMOS_RESTART_TMPL
 
             MEMDIR=${memchar} YMD=${PDY} HH=${cyc} generate_com \
@@ -182,7 +193,7 @@ if [ $DOSFCANL_ENKF = "YES" ]; then
             MEMDIR=${memchar} YMD=${PDY} HH=${cyc} generate_com \
                 COM_ATMOS_RESTART_MEM:COM_ATMOS_RESTART_TMPL
 
-            RUN="${GDUMP_ENS}" MEMDIR=${memchar} YMD=${gPDY} HH=${gcyc} generate_com \
+            RUN="${GDUMP_ENS}" MEMDIR=${gmemchar} YMD=${gPDY} HH=${gcyc} generate_com \
                 COM_ATMOS_RESTART_MEM_PREV:COM_ATMOS_RESTART_TMPL
 
             [[ ${TILE_NUM} -eq 1 ]] && mkdir -p "${COM_ATMOS_RESTART_MEM}"

scripts/exgdas_enkf_update.sh

@@ -56,7 +56,6 @@ ENKFSTAT=${ENKFSTAT:-${APREFIX}enkfstat}
 
 # Namelist parameters
 USE_CORRELATED_OBERRS=${USE_CORRELATED_OBERRS:-"NO"}
-NMEM_ENS=${NMEM_ENS:-80}
 NAM_ENKF=${NAM_ENKF:-""}
 SATOBS_ENKF=${SATOBS_ENKF:-""}
 OZOBS_ENKF=${OZOBS_ENKF:-""}
@@ -81,10 +80,15 @@ cnvw_option=${cnvw_option:-".false."}
 netcdf_diag=${netcdf_diag:-".true."}
 modelspace_vloc=${modelspace_vloc:-".false."} # if true, 'vlocal_eig.dat' is needed
 IAUFHRS_ENKF=${IAUFHRS_ENKF:-6}
-if [ $RUN = "enkfgfs" ]; then
+if [ "${RUN}" = "enkfgfs" ]; then
    DO_CALC_INCREMENT=${DO_CALC_INCREMENT_ENKF_GFS:-"NO"}
+   NMEM_ENS=${NMEM_ENS_GFS:-30}
+   ec_offset=${NMEM_ENS_GFS_OFFSET:-20}
+   mem_offset=$((ec_offset * cyc/6))
 else
    DO_CALC_INCREMENT=${DO_CALC_INCREMENT:-"NO"}
+   NMEM_ENS=${NMEM_ENS:-80}
+   mem_offset=0
 fi
 INCREMENTS_TO_ZERO=${INCREMENTS_TO_ZERO:-"'NONE'"}
 GSI_SOILANAL=${GSI_SOILANAL:-"NO"}
@@ -179,9 +183,14 @@ else
 fi
 nfhrs=$(echo $IAUFHRS_ENKF | sed 's/,/ /g')
 for imem in $(seq 1 $NMEM_ENS); do
+   smem=$((imem + mem_offset))
+   if (( smem > 80 )); then
+      smem=$((smem - 80))
+   fi
+   gmemchar="mem"$(printf %03i $smem)
    memchar="mem"$(printf %03i $imem)
 
-   MEMDIR=${memchar} RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} generate_com -x \
+   MEMDIR=${gmemchar} RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} generate_com -x \
       COM_ATMOS_HISTORY_MEM_PREV:COM_ATMOS_HISTORY_TMPL
 
    MEMDIR=${memchar} YMD=${PDY} HH=${cyc} generate_com -x \

workflow/rocoto/tasks.py

@@ -47,7 +47,10 @@ def __init__(self, app_config: AppConfig, cdump: str) -> None:
         self.HOMEgfs = self._base['HOMEgfs']
         self.rotdir = self._base['ROTDIR']
         self.pslot = self._base['PSLOT']
-        self.nmem = int(self._base['NMEM_ENS'])
+        if self.cdump == "enkfgfs":
+            self.nmem = int(self._base['NMEM_ENS_GFS'])
+        else:
+            self.nmem = int(self._base['NMEM_ENS'])
         self._base['cycle_interval'] = to_timedelta(f'{self._base["assim_freq"]}H')
 
         self.n_tiles = 6  # TODO - this needs to be elsewhere