updates for RRFS_A real-time (#477)

* Updates and fixes for WCOSS2 runs. * Change default value for: DA_CYCLE_INTERV RESTART_INTERVAL RESTART_INTERVAL_LONG When run GSI hybrid with RRFS ensemble, only retrospective experiment needs to check if RRFS ensembles exist. The real-time run does not need to check. * Add message to document the type of GSI analysis in log.cycles Co-authored-by: emc lam <emc.lam@dlogin04.dogwood.wcoss2.ncep.noaa.gov>
NOAA-GSL · Jan 26, 2023 · b48f4ac · b48f4ac
1 parent 62de1e0
commit b48f4ac
Show file tree

Hide file tree

Showing 10 changed files with 71 additions and 60 deletions.
diff --git a/fix/lam/RRFS_CONUS_3km/C3359_fvcom_mask.nc b/fix/lam/RRFS_CONUS_3km/C3359_fvcom_mask.nc
@@ -0,0 +1 @@
+../../.agent/lam/RRFS_CONUS_3km_C3359/C3359_fvcom_mask.nc
diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh
@@ -82,7 +82,7 @@ case "$MACHINE" in
   "WCOSS2")
     ulimit -s unlimited
     export OMP_STACKSIZE=1G
-    export OMP_NUM_THREADS=2
+    export OMP_NUM_THREADS=${TPP_MAKE_ICS}
     export FI_OFI_RXM_SAR_LIMIT=3145728
     export FI_MR_CACHE_MAX_COUNT=0
     export MPICH_OFI_STARTUP_CONNECT=1

diff --git a/scripts/exregional_run_analysis.sh b/scripts/exregional_run_analysis.sh
@@ -84,8 +84,8 @@ case $MACHINE in
   ulimit -s unlimited
   ulimit -a
   export FI_OFI_RXM_SAR_LIMIT=3145728
-  export OMP_STACKSIZE=500M
-  export OMP_NUM_THREADS=1
+  export OMP_STACKSIZE=1G
+  export OMP_NUM_THREADS=${TPP_RUN_ANAL}
   ncores=$(( NNODES_RUN_ANAL*PPN_RUN_ANAL))
   APRUN="mpiexec -n ${ncores} -ppn ${PPN_RUN_ANAL} --cpu-bind core --depth ${OMP_NUM_THREADS}"
   ;;
@@ -360,6 +360,7 @@ if [ ${regional_ensemble_option:-1} -eq 5 ]  && [ ${BKTYPE} != 1  ]; then
   print_info_msg "$VERBOSE" "Do hybrid with FV3LAM ensemble"
   ifhyb=.true.
   print_info_msg "$VERBOSE" " Cycle ${YYYYMMDDHH}: GSI hybrid uses FV3LAM ensemble with n_ens=${nummem}" 
+  echo " ${YYYYMMDDHH}(${cycle_type}): GSI hybrid uses FV3LAM ensemble with n_ens=${nummem}" >> ${EXPTDIR}/log.cycles
   grid_ratio_ens="1"
   ens_fast_read=.true.
 else    
@@ -369,9 +370,11 @@ else
     print_info_msg "$VERBOSE" "Do hybrid with ${memname}"
     ifhyb=.true.
     print_info_msg "$VERBOSE" " Cycle ${YYYYMMDDHH}: GSI hybrid uses ${memname} with n_ens=${nummem}"
+    echo " ${YYYYMMDDHH}(${cycle_type}): GSI hybrid uses ${memname} with n_ens=${nummem}" >> ${EXPTDIR}/log.cycles
   else
     print_info_msg "$VERBOSE" " Cycle ${YYYYMMDDHH}: GSI does pure 3DVAR."
     print_info_msg "$VERBOSE" " Hybrid needs at least ${HYBENSMEM_NMIN} ${memname} ensembles, only ${nummem} available"
+    echo " ${YYYYMMDDHH}(${cycle_type}): GSI dose pure 3DVAR" >> ${EXPTDIR}/log.cycles
   fi
 fi
 

diff --git a/scripts/exregional_run_prepstart.sh b/scripts/exregional_run_prepstart.sh
@@ -188,6 +188,7 @@ if [ ${DO_ENSFCST} = "TRUE" ] &&  [ ${DO_ENKFUPDATE} = "TRUE" ]; then
     print_err_msg_exit "Error: can not find ensemble DA analysis output for running ensemble free forecast, \
   check ${bkpath} for needed files."
   fi
+  SFC_CYC=0
 #
 #-----------------------------------------------------------------------
 #
@@ -919,7 +920,7 @@ fi
 #
 #-----------------------------------------------------------------------
 #
-if [ "${USE_FVCOM}" = "TRUE" ] || [ ${SFC_CYC} -eq 2 ] ; then
+if [ "${USE_FVCOM}" = "TRUE" ] && [ ${SFC_CYC} -eq 2 ] ; then
 
 # Remap the FVCOM output from the 5 lakes onto the RRFS grid
   if [ "${PREP_FVCOM}" = "TRUE" ]; then
@@ -937,24 +938,26 @@ if [ "${USE_FVCOM}" = "TRUE" ] || [ ${SFC_CYC} -eq 2 ] ; then
     cd_vrfy ${modelinputdir}
 # FVCOM_DIR needs to be redefined here to find 
     FVCOM_DIR=${modelinputdir}/fvcom_remap
-  fi
-
-  set -x
-  latest_fvcom_file="${FVCOM_DIR}/${FVCOM_FILE}"
-  if [ ${HH} -gt 12 ]; then 
-    starttime_fvcom="$(date +%Y%m%d -d "${START_DATE}") 12"
+    latest_fvcom_file="${FVCOM_DIR}/${FVCOM_FILE}"
+    fvcomtime=${YYYYJJJHH}
+    fvcom_data_fp="${latest_fvcom_file}_${fvcomtime}.nc"
   else
-    starttime_fvcom="$(date +%Y%m%d -d "${START_DATE}") 00"
+    latest_fvcom_file="${FVCOM_DIR}/${FVCOM_FILE}"
+    if [ ${HH} -gt 12 ]; then 
+      starttime_fvcom="$(date +%Y%m%d -d "${START_DATE}") 12"
+    else
+      starttime_fvcom="$(date +%Y%m%d -d "${START_DATE}") 00"
+    fi
+    for ii in $(seq 0 3)
+    do
+       jumphour=$((${ii} * 12))
+       fvcomtime=$(date +%Y%j%H -d "${starttime_fvcom}  ${jumphour} hours ago")
+       fvcom_data_fp="${latest_fvcom_file}_${fvcomtime}.nc"
+       if [ -f "${fvcom_data_fp}" ]; then
+         break 
+       fi
+    done
   fi
-  for ii in $(seq 0 3)
-  do
-     jumphour=$((${ii} * 12))
-     fvcomtime=$(date +%Y%j%H -d "${starttime_fvcom}  ${jumphour} hours ago")
-     fvcom_data_fp="${latest_fvcom_file}_${fvcomtime}.nc"
-     if [ -f "${fvcom_data_fp}" ]; then
-       break 
-     fi
-  done
 
   if [ ! -f "${fvcom_data_fp}" ]; then
     print_info_msg "\

diff --git a/scripts/exregional_run_recenter.sh b/scripts/exregional_run_recenter.sh
@@ -174,9 +174,9 @@ for imem in  $(seq 1 $nens)
   dynvarfile=${bkpath}/fv_core.res.tile1.nc
   tracerfile=${bkpath}/fv_tracer.res.tile1.nc
   if [ -r "${dynvarfile}" ] && [ -r "${tracerfile}" ] ; then
-    cp_vrfy ${bkpath}/fv_core.res.tile1.nc  ./fv3sar_tile1_mem${memberstring}_dynvar
-    cp_vrfy ${bkpath}/fv_tracer.res.tile1.nc   ./fv3sar_tile1_mem${memberstring}_tracer
-    cp_vrfy ${bkpath}/sfc_data.nc  ./fv3sar_tile1_mem${memberstring}_sfcvar
+    ln -sf ${bkpath}/fv_core.res.tile1.nc  ./fv3sar_tile1_mem${memberstring}_dynvar
+    ln -sf ${bkpath}/fv_tracer.res.tile1.nc   ./fv3sar_tile1_mem${memberstring}_tracer
+    ln -sf ${bkpath}/sfc_data.nc  ./fv3sar_tile1_mem${memberstring}_sfcvar
     ln -sf ${bkpath}/fv_core.res.tile1.nc  ./rec_fv3sar_tile1_mem${memberstring}_dynvar
     ln -sf ${bkpath}/fv_tracer.res.tile1.nc   ./rec_fv3sar_tile1_mem${memberstring}_tracer
     ln -sf ${bkpath}/sfc_data.nc  ./rec_fv3sar_tile1_mem${memberstring}_sfcvar

diff --git a/ush/config.sh.rrfs_a_c3 b/ush/config.sh.rrfs_a_c3
@@ -1,8 +1,8 @@
 MACHINE="wcoss2"
-version="v0.3.2"
+version="v0.3.6"
 ACCOUNT="RRFS_DEV"
 #RESERVATION="rrfsdet"
-EXPT_BASEDIR="/lfs/h2/emc/lam/noscrub/emc.lam/rrfsx/$version/"
+EXPT_BASEDIR="/lfs/h2/emc/lam/noscrub/emc.lam/rrfs/$version/"
 EXPT_SUBDIR="rrfs_conus"
 
 PREDEF_GRID_NAME=RRFS_CONUS_3km
@@ -35,13 +35,13 @@ BOUNDARY_PROC_GROUP_NUM="72"
 
 # avaialble retro period:
 # 20210511-20210531; 20210718-20210801
-DATE_FIRST_CYCL="20221216"
-DATE_LAST_CYCL="20221231"
+DATE_FIRST_CYCL="20230120"
+DATE_LAST_CYCL="20230131"
 CYCL_HRS=( "00" "12" )
 CYCL_HRS_SPINSTART=("03" "15")
 CYCL_HRS_PRODSTART=("09" "21")
-CYCLEMONTH="12"
-CYCLEDAY="16-31"
+CYCLEMONTH="01"
+CYCLEDAY="20-31"
 
 STARTYEAR=${DATE_FIRST_CYCL:0:4}
 STARTMONTH=${DATE_FIRST_CYCL:4:2}
@@ -55,11 +55,11 @@ ENDHOUR="23"
 PREEXISTING_DIR_METHOD="upgrade" # "rename"
 INITIAL_CYCLEDEF="${DATE_FIRST_CYCL}0300 ${DATE_LAST_CYCL}2300 12:00:00"
 BOUNDARY_CYCLEDEF="${DATE_FIRST_CYCL}0000 ${DATE_LAST_CYCL}2300 06:00:00"
-PROD_CYCLEDEF="00 01-05,07-11,13-17,19-23 ${CYCLEDAY} ${CYCLEMONTH} 2022 *"
-PRODLONG_CYCLEDEF="00 00,06,12,18 ${CYCLEDAY} ${CYCLEMONTH} 2022 *"
+PROD_CYCLEDEF="00 01-05,07-11,13-17,19-23 ${CYCLEDAY} ${CYCLEMONTH} ${STARTYEAR} *"
+PRODLONG_CYCLEDEF="00 00,06,12,18 ${CYCLEDAY} ${CYCLEMONTH} ${STARTYEAR} *"
 ARCHIVE_CYCLEDEF="${DATE_FIRST_CYCL}1400 ${DATE_LAST_CYCL}2300 24:00:00"
 if [[ $DO_SPINUP == "TRUE" ]] ; then
-  SPINUP_CYCLEDEF="00 03-08,15-20 ${CYCLEDAY} ${CYCLEMONTH} 2022 *"
+  SPINUP_CYCLEDEF="00 03-08,15-20 ${CYCLEDAY} ${CYCLEMONTH} ${STARTYEAR} *"
 fi
 
 FCST_LEN_HRS="18"
@@ -71,6 +71,7 @@ for i in {0..23}; do FCST_LEN_HRS_CYCLES[$i]=18; done
 for i in {0..23..6}; do FCST_LEN_HRS_CYCLES[$i]=60; done
 DA_CYCLE_INTERV="1"
 RESTART_INTERVAL="1 2 3"
+RESTART_INTERVAL_LONG="1 2 3"
 
 USE_RRFSE_ENS="TRUE"
 CYCL_HRS_HYB_FV3LAM_ENS=("00" "01" "02" "03" "04" "05" "06" "07" "08" "09" "10" "11" "12" "13" "14" "15" "16" "17" "18" "19" "20" "21" "22" "23")
@@ -89,7 +90,7 @@ EXTRN_MDL_NAME_LBCS="FV3GFS"
 envir="para"
 
 NET="rrfs_a"
-TAG="c3v32"
+TAG="c3v36"
 
 ARCHIVEDIR="/NCEPDEV/emc-meso/1year/emc.lam"
 NCL_REGION="conus"
@@ -104,7 +105,7 @@ MODEL="RRFS_A"
 #RUN_TASK_MAKE_OROG="FALSE"
 #RUN_TASK_MAKE_SFC_CLIMO="FALSE"
 
-RUN="rrfs"
+RUN="rrfs_a"
 COMINgfs=""
 
 . set_rrfs_config.sh

diff --git a/ush/config.sh_rrfs_a_enkf_c3 b/ush/config.sh_rrfs_a_enkf_c3
@@ -1,9 +1,9 @@
 MACHINE="wcoss2"
-version="v0.3.2"
+version="v0.3.6"
 ACCOUNT="RRFS_DEV"
 #RESERVATION="rrfsens"
 #RESERVATION_POST="rrfsdet"
-EXPT_BASEDIR="/lfs/h2/emc/lam/noscrub/emc.lam/rrfsx/${version}/"
+EXPT_BASEDIR="/lfs/h2/emc/lam/noscrub/emc.lam/rrfs/${version}/"
 EXPT_SUBDIR="rrfs_conus_ens"
 
 PREDEF_GRID_NAME=RRFS_CONUS_3km
@@ -41,16 +41,16 @@ BOUNDARY_PROC_GROUP_NUM="4"
 
 # avaialble retro period:
 # 20210511-20210531; 20210718-20210801
-DATE_FIRST_CYCL="20221216"
-DATE_LAST_CYCL="20221231"
+DATE_FIRST_CYCL="20230121"
+DATE_LAST_CYCL="20230131"
 CYCL_HRS=( "00" "12" )
 CYCL_HRS=( "18" )
 CYCL_HRS_SPINSTART=("18")
 CYCL_HRS_PRODSTART=("19")
 #CYCL_HRS_STOCH=("00" "12")
 #CYCL_HRS_RECENTER=("19")
-CYCLEMONTH="12"
-CYCLEDAY="16-31"
+CYCLEMONTH="01"
+CYCLEDAY="21-31"
 
 STARTYEAR=${DATE_FIRST_CYCL:0:4}
 STARTMONTH=${DATE_FIRST_CYCL:4:2}
@@ -64,17 +64,17 @@ ENDHOUR="23"
 PREEXISTING_DIR_METHOD="upgrade" # "rename"
 INITIAL_CYCLEDEF="${DATE_FIRST_CYCL}1800 ${DATE_LAST_CYCL}2300 24:00:00"
 BOUNDARY_CYCLEDEF="${DATE_FIRST_CYCL}1800 ${DATE_LAST_CYCL}2300 06:00:00"
-PROD_CYCLEDEF="00 01-05,07-11,13-17,19-23 19-30 12 2022 *"
+PROD_CYCLEDEF="00 01-05,07-11,13-17,19-23 19-30 ${CYCLEMONTH} ${STARTYEAR} *"
 PRODLONG_CYCLEDEF="${DATE_FIRST_CYCL}0000 ${DATE_LAST_CYCL}2300 06:00:00"
 #RECENTER_CYCLEDEF="00 19 * 10 2022 *"
 ARCHIVE_CYCLEDEF="${DATE_FIRST_CYCL}1500 ${DATE_LAST_CYCL}2300 24:00:00"
 if [[ ${DO_ENSFCST} == "TRUE" ]] ; then
   BOUNDARY_LEN_HRS="36"
   DO_SPINUP="FALSE"
   INITIAL_CYCLEDEF="00 01 01 01 2100 *"
+  BOUNDARY_CYCLEDEF="${DATE_FIRST_CYCL}1200 ${DATE_LAST_CYCL}2300 12:00:00"
   PROD_CYCLEDEF="${DATE_FIRST_CYCL}1200 ${DATE_LAST_CYCL}2300 12:00:00"
   PRODLONG_CYCLEDEF="00 01 01 01 2100 *"
-  BOUNDARY_CYCLEDEF="${DATE_FIRST_CYCL}1200 ${DATE_LAST_CYCL}2300 12:00:00"
 fi
 if [[ $DO_SPINUP == "TRUE" ]] ; then
   SPINUP_CYCLEDEF="${DATE_FIRST_CYCL}1800 ${DATE_LAST_CYCL}2300 24:00:00"
@@ -88,15 +88,15 @@ FCST_LEN_HRS_SPINUP="1"
 POSTPROC_LEN_HRS="1"
 #FCST_LEN_HRS_CYCLES=(48 18 18 18 18 18 48 18 18 18 18 18 48 18 18 18 18 18 48 18 18 18 18 18)
 for i in {0..23}; do FCST_LEN_HRS_CYCLES[$i]=1; done
-for i in {0..23..6}; do FCST_LEN_HRS_CYCLES[$i]=6; done 
+for i in {0..23..6}; do FCST_LEN_HRS_CYCLES[$i]=2; done 
 if [[ ${DO_ENSFCST} == "TRUE" ]] ; then
   for i in {0..23..12}; do FCST_LEN_HRS_CYCLES[$i]=24; done 
   POSTPROC_LEN_HRS="24"
   BOUNDARY_PROC_GROUP_NUM="8"
 fi
 DA_CYCLE_INTERV="1"
 RESTART_INTERVAL="1"
-RESTART_INTERVAL_LONG="1 2 3 4 5 6"
+RESTART_INTERVAL_LONG="1 2"
 netcdf_diag=.true.
 binary_diag=.false.
 NFHOUT="3"

diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh
@@ -579,9 +579,9 @@ POSTPROC_LONG_LEN_HRS="1"
 FCST_LEN_HRS="24"
 FCST_LEN_HRS_SPINUP="1"
 FCST_LEN_HRS_CYCLES=( )
-DA_CYCLE_INTERV="3"
-RESTART_INTERVAL="3,6"
-RESTART_INTERVAL_LONG="3,6"
+DA_CYCLE_INTERV="1"
+RESTART_INTERVAL="1 2"
+RESTART_INTERVAL_LONG="1 2"
 CYCL_HRS_HYB_FV3LAM_ENS=( "99" )
 
 #-----------------------------------------------------------------------

diff --git a/ush/set_rrfs_config_general.sh b/ush/set_rrfs_config_general.sh
@@ -150,27 +150,27 @@ if [[ ${PREDEF_GRID_NAME} == "RRFS_CONUS_3km" ]] ; then
   fi
 
   if [[ $MACHINE == "wcoss2" ]] ; then
-    LAYOUT_X="48"
-    LAYOUT_Y="40"
+    LAYOUT_X="31"
+    LAYOUT_Y="32"
     PPN_MAKE_ICS="32"
     TPP_MAKE_ICS="4"
     PPN_MAKE_LBCS="64"
     TPP_MAKE_LBCS="2"
     NNODES_MAKE_ICS="2"
     NNODES_MAKE_LBCS="1"
-    PPN_RUN_ANAL="64"
-    TPP_RUN_ANAL="2"
+    PPN_RUN_ANAL="32"
+    TPP_RUN_ANAL="4"
     NNODES_RUN_ANAL="10"
     PPN_RUN_FCST="64"
     TPP_RUN_FCST="2"
-    NNODES_RUN_FCST="31"
+    NNODES_RUN_FCST="16"
     PPN_RUN_POST="64"
-    NNODES_RUN_POST="1"
+    NNODES_RUN_POST="2"
     PPN_PROC_RADAR="64"
     NNODES_PROC_RADAR="1"
     PPN_RUN_ENKF="32"
     TPP_RUN_ENKF="4"
-    NNODES_RUN_ENKF="6"
+    NNODES_RUN_ENKF="24"
 
     MAXTRIES_GET_EXTRN_ICS="2"
     MAXTRIES_GET_EXTRN_LBCS="2"
@@ -197,16 +197,13 @@ if [[ ${PREDEF_GRID_NAME} == "RRFS_CONUS_3km" ]] ; then
     WTIME_RUN_ANAL="00:50:00"
     WTIME_RUN_PREPSTART="00:20:00"
 
-    START_TIME_SPINUP="02:10:00"
-    START_TIME_PROD="08:20:00"
-    START_TIME_LATE_ANALYSIS="08:40:00"
+    START_TIME_SPINUP="01:10:00"
+    START_TIME_PROD="02:20:00"
+    START_TIME_LATE_ANALYSIS="01:40:00"
 
     FV3GFS_FILE_FMT_ICS="netcdf"
     FV3GFS_FILE_FMT_LBCS="netcdf"
 
-    START_TIME_SPINUP="01:10:00"
-    START_TIME_PROD="02:20:00"
-    START_TIME_LATE_ANALYSIS="01:40:00"
   fi
 
   if [[ $MACHINE == "orion" ]] ; then

diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml
@@ -1206,6 +1206,7 @@ MODULES_RUN_TASK_FP script.
          <timedep><cyclestr offset="&START_TIME_CONVENTIONAL;">@Y@m@d@H@M00</cyclestr></timedep>
          <taskdep task="&PREP_CYC_TN;_#type#{{ uscore_ensmem_name }}"/>
          {%- if use_rrfse_ens %}
+         {%- if do_retro %}
          <or>
            <and>
              <or>
@@ -1224,6 +1225,7 @@ MODULES_RUN_TASK_FP script.
            </and>
          </or>
          {%- endif %}
+         {%- endif %}
        </and>
     </dependency>
 
@@ -1271,6 +1273,7 @@ MODULES_RUN_TASK_FP script.
          <taskdep task="&ANAL_SD_GSI_TN;_#type#"/>
          {%- endif %}
          {%- if use_rrfse_ens %}
+         {%- if do_retro %}
          <or>
            <and>
              <or>
@@ -1308,6 +1311,7 @@ MODULES_RUN_TASK_FP script.
            </and>
          </or>
          {%- endif %}
+         {%- endif %}
        </and>
     </dependency>
 
@@ -2107,6 +2111,7 @@ MODULES_RUN_TASK_FP script.
          <timedep><cyclestr offset="&START_TIME_CONVENTIONAL;">@Y@m@d@H@M00</cyclestr></timedep>
          <taskdep task="&PREP_CYC_TN;_prod{{ uscore_ensmem_name }}"/>
          {%- if use_rrfse_ens %}
+         {%- if do_retro %}
          <or>
            <and>
              <or>
@@ -2152,6 +2157,7 @@ MODULES_RUN_TASK_FP script.
            </and>
          </or>
          {%- endif %}
+         {%- endif %}
        </and>
     </dependency>