diff --git a/fix/lam/RRFS_CONUS_3km/C3359_fvcom_mask.nc b/fix/lam/RRFS_CONUS_3km/C3359_fvcom_mask.nc new file mode 120000 index 000000000..968db6e83 --- /dev/null +++ b/fix/lam/RRFS_CONUS_3km/C3359_fvcom_mask.nc @@ -0,0 +1 @@ +../../.agent/lam/RRFS_CONUS_3km_C3359/C3359_fvcom_mask.nc \ No newline at end of file diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 67002b72e..0335c6c6f 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -82,7 +82,7 @@ case "$MACHINE" in "WCOSS2") ulimit -s unlimited export OMP_STACKSIZE=1G - export OMP_NUM_THREADS=2 + export OMP_NUM_THREADS=${TPP_MAKE_ICS} export FI_OFI_RXM_SAR_LIMIT=3145728 export FI_MR_CACHE_MAX_COUNT=0 export MPICH_OFI_STARTUP_CONNECT=1 diff --git a/scripts/exregional_run_analysis.sh b/scripts/exregional_run_analysis.sh index dd9242519..e06772093 100755 --- a/scripts/exregional_run_analysis.sh +++ b/scripts/exregional_run_analysis.sh @@ -84,8 +84,8 @@ case $MACHINE in ulimit -s unlimited ulimit -a export FI_OFI_RXM_SAR_LIMIT=3145728 - export OMP_STACKSIZE=500M - export OMP_NUM_THREADS=1 + export OMP_STACKSIZE=1G + export OMP_NUM_THREADS=${TPP_RUN_ANAL} ncores=$(( NNODES_RUN_ANAL*PPN_RUN_ANAL)) APRUN="mpiexec -n ${ncores} -ppn ${PPN_RUN_ANAL} --cpu-bind core --depth ${OMP_NUM_THREADS}" ;; @@ -360,6 +360,7 @@ if [ ${regional_ensemble_option:-1} -eq 5 ] && [ ${BKTYPE} != 1 ]; then print_info_msg "$VERBOSE" "Do hybrid with FV3LAM ensemble" ifhyb=.true. print_info_msg "$VERBOSE" " Cycle ${YYYYMMDDHH}: GSI hybrid uses FV3LAM ensemble with n_ens=${nummem}" + echo " ${YYYYMMDDHH}(${cycle_type}): GSI hybrid uses FV3LAM ensemble with n_ens=${nummem}" >> ${EXPTDIR}/log.cycles grid_ratio_ens="1" ens_fast_read=.true. else @@ -369,9 +370,11 @@ else print_info_msg "$VERBOSE" "Do hybrid with ${memname}" ifhyb=.true. print_info_msg "$VERBOSE" " Cycle ${YYYYMMDDHH}: GSI hybrid uses ${memname} with n_ens=${nummem}" + echo " ${YYYYMMDDHH}(${cycle_type}): GSI hybrid uses ${memname} with n_ens=${nummem}" >> ${EXPTDIR}/log.cycles else print_info_msg "$VERBOSE" " Cycle ${YYYYMMDDHH}: GSI does pure 3DVAR." print_info_msg "$VERBOSE" " Hybrid needs at least ${HYBENSMEM_NMIN} ${memname} ensembles, only ${nummem} available" + echo " ${YYYYMMDDHH}(${cycle_type}): GSI dose pure 3DVAR" >> ${EXPTDIR}/log.cycles fi fi diff --git a/scripts/exregional_run_prepstart.sh b/scripts/exregional_run_prepstart.sh index f18bcacb5..ec9c0e263 100755 --- a/scripts/exregional_run_prepstart.sh +++ b/scripts/exregional_run_prepstart.sh @@ -188,6 +188,7 @@ if [ ${DO_ENSFCST} = "TRUE" ] && [ ${DO_ENKFUPDATE} = "TRUE" ]; then print_err_msg_exit "Error: can not find ensemble DA analysis output for running ensemble free forecast, \ check ${bkpath} for needed files." fi + SFC_CYC=0 # #----------------------------------------------------------------------- # @@ -919,7 +920,7 @@ fi # #----------------------------------------------------------------------- # -if [ "${USE_FVCOM}" = "TRUE" ] || [ ${SFC_CYC} -eq 2 ] ; then +if [ "${USE_FVCOM}" = "TRUE" ] && [ ${SFC_CYC} -eq 2 ] ; then # Remap the FVCOM output from the 5 lakes onto the RRFS grid if [ "${PREP_FVCOM}" = "TRUE" ]; then @@ -937,24 +938,26 @@ if [ "${USE_FVCOM}" = "TRUE" ] || [ ${SFC_CYC} -eq 2 ] ; then cd_vrfy ${modelinputdir} # FVCOM_DIR needs to be redefined here to find FVCOM_DIR=${modelinputdir}/fvcom_remap - fi - - set -x - latest_fvcom_file="${FVCOM_DIR}/${FVCOM_FILE}" - if [ ${HH} -gt 12 ]; then - starttime_fvcom="$(date +%Y%m%d -d "${START_DATE}") 12" + latest_fvcom_file="${FVCOM_DIR}/${FVCOM_FILE}" + fvcomtime=${YYYYJJJHH} + fvcom_data_fp="${latest_fvcom_file}_${fvcomtime}.nc" else - starttime_fvcom="$(date +%Y%m%d -d "${START_DATE}") 00" + latest_fvcom_file="${FVCOM_DIR}/${FVCOM_FILE}" + if [ ${HH} -gt 12 ]; then + starttime_fvcom="$(date +%Y%m%d -d "${START_DATE}") 12" + else + starttime_fvcom="$(date +%Y%m%d -d "${START_DATE}") 00" + fi + for ii in $(seq 0 3) + do + jumphour=$((${ii} * 12)) + fvcomtime=$(date +%Y%j%H -d "${starttime_fvcom} ${jumphour} hours ago") + fvcom_data_fp="${latest_fvcom_file}_${fvcomtime}.nc" + if [ -f "${fvcom_data_fp}" ]; then + break + fi + done fi - for ii in $(seq 0 3) - do - jumphour=$((${ii} * 12)) - fvcomtime=$(date +%Y%j%H -d "${starttime_fvcom} ${jumphour} hours ago") - fvcom_data_fp="${latest_fvcom_file}_${fvcomtime}.nc" - if [ -f "${fvcom_data_fp}" ]; then - break - fi - done if [ ! -f "${fvcom_data_fp}" ]; then print_info_msg "\ diff --git a/scripts/exregional_run_recenter.sh b/scripts/exregional_run_recenter.sh index 8c5de2016..e142dcddd 100755 --- a/scripts/exregional_run_recenter.sh +++ b/scripts/exregional_run_recenter.sh @@ -174,9 +174,9 @@ for imem in $(seq 1 $nens) dynvarfile=${bkpath}/fv_core.res.tile1.nc tracerfile=${bkpath}/fv_tracer.res.tile1.nc if [ -r "${dynvarfile}" ] && [ -r "${tracerfile}" ] ; then - cp_vrfy ${bkpath}/fv_core.res.tile1.nc ./fv3sar_tile1_mem${memberstring}_dynvar - cp_vrfy ${bkpath}/fv_tracer.res.tile1.nc ./fv3sar_tile1_mem${memberstring}_tracer - cp_vrfy ${bkpath}/sfc_data.nc ./fv3sar_tile1_mem${memberstring}_sfcvar + ln -sf ${bkpath}/fv_core.res.tile1.nc ./fv3sar_tile1_mem${memberstring}_dynvar + ln -sf ${bkpath}/fv_tracer.res.tile1.nc ./fv3sar_tile1_mem${memberstring}_tracer + ln -sf ${bkpath}/sfc_data.nc ./fv3sar_tile1_mem${memberstring}_sfcvar ln -sf ${bkpath}/fv_core.res.tile1.nc ./rec_fv3sar_tile1_mem${memberstring}_dynvar ln -sf ${bkpath}/fv_tracer.res.tile1.nc ./rec_fv3sar_tile1_mem${memberstring}_tracer ln -sf ${bkpath}/sfc_data.nc ./rec_fv3sar_tile1_mem${memberstring}_sfcvar diff --git a/ush/config.sh.rrfs_a_c3 b/ush/config.sh.rrfs_a_c3 index 6660be6ab..d343f7877 100644 --- a/ush/config.sh.rrfs_a_c3 +++ b/ush/config.sh.rrfs_a_c3 @@ -1,8 +1,8 @@ MACHINE="wcoss2" -version="v0.3.2" +version="v0.3.6" ACCOUNT="RRFS_DEV" #RESERVATION="rrfsdet" -EXPT_BASEDIR="/lfs/h2/emc/lam/noscrub/emc.lam/rrfsx/$version/" +EXPT_BASEDIR="/lfs/h2/emc/lam/noscrub/emc.lam/rrfs/$version/" EXPT_SUBDIR="rrfs_conus" PREDEF_GRID_NAME=RRFS_CONUS_3km @@ -35,13 +35,13 @@ BOUNDARY_PROC_GROUP_NUM="72" # avaialble retro period: # 20210511-20210531; 20210718-20210801 -DATE_FIRST_CYCL="20221216" -DATE_LAST_CYCL="20221231" +DATE_FIRST_CYCL="20230120" +DATE_LAST_CYCL="20230131" CYCL_HRS=( "00" "12" ) CYCL_HRS_SPINSTART=("03" "15") CYCL_HRS_PRODSTART=("09" "21") -CYCLEMONTH="12" -CYCLEDAY="16-31" +CYCLEMONTH="01" +CYCLEDAY="20-31" STARTYEAR=${DATE_FIRST_CYCL:0:4} STARTMONTH=${DATE_FIRST_CYCL:4:2} @@ -55,11 +55,11 @@ ENDHOUR="23" PREEXISTING_DIR_METHOD="upgrade" # "rename" INITIAL_CYCLEDEF="${DATE_FIRST_CYCL}0300 ${DATE_LAST_CYCL}2300 12:00:00" BOUNDARY_CYCLEDEF="${DATE_FIRST_CYCL}0000 ${DATE_LAST_CYCL}2300 06:00:00" -PROD_CYCLEDEF="00 01-05,07-11,13-17,19-23 ${CYCLEDAY} ${CYCLEMONTH} 2022 *" -PRODLONG_CYCLEDEF="00 00,06,12,18 ${CYCLEDAY} ${CYCLEMONTH} 2022 *" +PROD_CYCLEDEF="00 01-05,07-11,13-17,19-23 ${CYCLEDAY} ${CYCLEMONTH} ${STARTYEAR} *" +PRODLONG_CYCLEDEF="00 00,06,12,18 ${CYCLEDAY} ${CYCLEMONTH} ${STARTYEAR} *" ARCHIVE_CYCLEDEF="${DATE_FIRST_CYCL}1400 ${DATE_LAST_CYCL}2300 24:00:00" if [[ $DO_SPINUP == "TRUE" ]] ; then - SPINUP_CYCLEDEF="00 03-08,15-20 ${CYCLEDAY} ${CYCLEMONTH} 2022 *" + SPINUP_CYCLEDEF="00 03-08,15-20 ${CYCLEDAY} ${CYCLEMONTH} ${STARTYEAR} *" fi FCST_LEN_HRS="18" @@ -71,6 +71,7 @@ for i in {0..23}; do FCST_LEN_HRS_CYCLES[$i]=18; done for i in {0..23..6}; do FCST_LEN_HRS_CYCLES[$i]=60; done DA_CYCLE_INTERV="1" RESTART_INTERVAL="1 2 3" +RESTART_INTERVAL_LONG="1 2 3" USE_RRFSE_ENS="TRUE" CYCL_HRS_HYB_FV3LAM_ENS=("00" "01" "02" "03" "04" "05" "06" "07" "08" "09" "10" "11" "12" "13" "14" "15" "16" "17" "18" "19" "20" "21" "22" "23") @@ -89,7 +90,7 @@ EXTRN_MDL_NAME_LBCS="FV3GFS" envir="para" NET="rrfs_a" -TAG="c3v32" +TAG="c3v36" ARCHIVEDIR="/NCEPDEV/emc-meso/1year/emc.lam" NCL_REGION="conus" @@ -104,7 +105,7 @@ MODEL="RRFS_A" #RUN_TASK_MAKE_OROG="FALSE" #RUN_TASK_MAKE_SFC_CLIMO="FALSE" -RUN="rrfs" +RUN="rrfs_a" COMINgfs="" . set_rrfs_config.sh diff --git a/ush/config.sh_rrfs_a_enkf_c3 b/ush/config.sh_rrfs_a_enkf_c3 index cf593bc3c..dab05a700 100644 --- a/ush/config.sh_rrfs_a_enkf_c3 +++ b/ush/config.sh_rrfs_a_enkf_c3 @@ -1,9 +1,9 @@ MACHINE="wcoss2" -version="v0.3.2" +version="v0.3.6" ACCOUNT="RRFS_DEV" #RESERVATION="rrfsens" #RESERVATION_POST="rrfsdet" -EXPT_BASEDIR="/lfs/h2/emc/lam/noscrub/emc.lam/rrfsx/${version}/" +EXPT_BASEDIR="/lfs/h2/emc/lam/noscrub/emc.lam/rrfs/${version}/" EXPT_SUBDIR="rrfs_conus_ens" PREDEF_GRID_NAME=RRFS_CONUS_3km @@ -41,16 +41,16 @@ BOUNDARY_PROC_GROUP_NUM="4" # avaialble retro period: # 20210511-20210531; 20210718-20210801 -DATE_FIRST_CYCL="20221216" -DATE_LAST_CYCL="20221231" +DATE_FIRST_CYCL="20230121" +DATE_LAST_CYCL="20230131" CYCL_HRS=( "00" "12" ) CYCL_HRS=( "18" ) CYCL_HRS_SPINSTART=("18") CYCL_HRS_PRODSTART=("19") #CYCL_HRS_STOCH=("00" "12") #CYCL_HRS_RECENTER=("19") -CYCLEMONTH="12" -CYCLEDAY="16-31" +CYCLEMONTH="01" +CYCLEDAY="21-31" STARTYEAR=${DATE_FIRST_CYCL:0:4} STARTMONTH=${DATE_FIRST_CYCL:4:2} @@ -64,7 +64,7 @@ ENDHOUR="23" PREEXISTING_DIR_METHOD="upgrade" # "rename" INITIAL_CYCLEDEF="${DATE_FIRST_CYCL}1800 ${DATE_LAST_CYCL}2300 24:00:00" BOUNDARY_CYCLEDEF="${DATE_FIRST_CYCL}1800 ${DATE_LAST_CYCL}2300 06:00:00" -PROD_CYCLEDEF="00 01-05,07-11,13-17,19-23 19-30 12 2022 *" +PROD_CYCLEDEF="00 01-05,07-11,13-17,19-23 19-30 ${CYCLEMONTH} ${STARTYEAR} *" PRODLONG_CYCLEDEF="${DATE_FIRST_CYCL}0000 ${DATE_LAST_CYCL}2300 06:00:00" #RECENTER_CYCLEDEF="00 19 * 10 2022 *" ARCHIVE_CYCLEDEF="${DATE_FIRST_CYCL}1500 ${DATE_LAST_CYCL}2300 24:00:00" @@ -72,9 +72,9 @@ if [[ ${DO_ENSFCST} == "TRUE" ]] ; then BOUNDARY_LEN_HRS="36" DO_SPINUP="FALSE" INITIAL_CYCLEDEF="00 01 01 01 2100 *" + BOUNDARY_CYCLEDEF="${DATE_FIRST_CYCL}1200 ${DATE_LAST_CYCL}2300 12:00:00" PROD_CYCLEDEF="${DATE_FIRST_CYCL}1200 ${DATE_LAST_CYCL}2300 12:00:00" PRODLONG_CYCLEDEF="00 01 01 01 2100 *" - BOUNDARY_CYCLEDEF="${DATE_FIRST_CYCL}1200 ${DATE_LAST_CYCL}2300 12:00:00" fi if [[ $DO_SPINUP == "TRUE" ]] ; then SPINUP_CYCLEDEF="${DATE_FIRST_CYCL}1800 ${DATE_LAST_CYCL}2300 24:00:00" @@ -88,7 +88,7 @@ FCST_LEN_HRS_SPINUP="1" POSTPROC_LEN_HRS="1" #FCST_LEN_HRS_CYCLES=(48 18 18 18 18 18 48 18 18 18 18 18 48 18 18 18 18 18 48 18 18 18 18 18) for i in {0..23}; do FCST_LEN_HRS_CYCLES[$i]=1; done -for i in {0..23..6}; do FCST_LEN_HRS_CYCLES[$i]=6; done +for i in {0..23..6}; do FCST_LEN_HRS_CYCLES[$i]=2; done if [[ ${DO_ENSFCST} == "TRUE" ]] ; then for i in {0..23..12}; do FCST_LEN_HRS_CYCLES[$i]=24; done POSTPROC_LEN_HRS="24" @@ -96,7 +96,7 @@ if [[ ${DO_ENSFCST} == "TRUE" ]] ; then fi DA_CYCLE_INTERV="1" RESTART_INTERVAL="1" -RESTART_INTERVAL_LONG="1 2 3 4 5 6" +RESTART_INTERVAL_LONG="1 2" netcdf_diag=.true. binary_diag=.false. NFHOUT="3" diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 658affa4c..8586d87d0 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -579,9 +579,9 @@ POSTPROC_LONG_LEN_HRS="1" FCST_LEN_HRS="24" FCST_LEN_HRS_SPINUP="1" FCST_LEN_HRS_CYCLES=( ) -DA_CYCLE_INTERV="3" -RESTART_INTERVAL="3,6" -RESTART_INTERVAL_LONG="3,6" +DA_CYCLE_INTERV="1" +RESTART_INTERVAL="1 2" +RESTART_INTERVAL_LONG="1 2" CYCL_HRS_HYB_FV3LAM_ENS=( "99" ) #----------------------------------------------------------------------- diff --git a/ush/set_rrfs_config_general.sh b/ush/set_rrfs_config_general.sh index baea3e21a..1af0a36de 100644 --- a/ush/set_rrfs_config_general.sh +++ b/ush/set_rrfs_config_general.sh @@ -150,27 +150,27 @@ if [[ ${PREDEF_GRID_NAME} == "RRFS_CONUS_3km" ]] ; then fi if [[ $MACHINE == "wcoss2" ]] ; then - LAYOUT_X="48" - LAYOUT_Y="40" + LAYOUT_X="31" + LAYOUT_Y="32" PPN_MAKE_ICS="32" TPP_MAKE_ICS="4" PPN_MAKE_LBCS="64" TPP_MAKE_LBCS="2" NNODES_MAKE_ICS="2" NNODES_MAKE_LBCS="1" - PPN_RUN_ANAL="64" - TPP_RUN_ANAL="2" + PPN_RUN_ANAL="32" + TPP_RUN_ANAL="4" NNODES_RUN_ANAL="10" PPN_RUN_FCST="64" TPP_RUN_FCST="2" - NNODES_RUN_FCST="31" + NNODES_RUN_FCST="16" PPN_RUN_POST="64" - NNODES_RUN_POST="1" + NNODES_RUN_POST="2" PPN_PROC_RADAR="64" NNODES_PROC_RADAR="1" PPN_RUN_ENKF="32" TPP_RUN_ENKF="4" - NNODES_RUN_ENKF="6" + NNODES_RUN_ENKF="24" MAXTRIES_GET_EXTRN_ICS="2" MAXTRIES_GET_EXTRN_LBCS="2" @@ -197,16 +197,13 @@ if [[ ${PREDEF_GRID_NAME} == "RRFS_CONUS_3km" ]] ; then WTIME_RUN_ANAL="00:50:00" WTIME_RUN_PREPSTART="00:20:00" - START_TIME_SPINUP="02:10:00" - START_TIME_PROD="08:20:00" - START_TIME_LATE_ANALYSIS="08:40:00" + START_TIME_SPINUP="01:10:00" + START_TIME_PROD="02:20:00" + START_TIME_LATE_ANALYSIS="01:40:00" FV3GFS_FILE_FMT_ICS="netcdf" FV3GFS_FILE_FMT_LBCS="netcdf" - START_TIME_SPINUP="01:10:00" - START_TIME_PROD="02:20:00" - START_TIME_LATE_ANALYSIS="01:40:00" fi if [[ $MACHINE == "orion" ]] ; then diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index 2b3fdbb92..bc26d25fb 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -1206,6 +1206,7 @@ MODULES_RUN_TASK_FP script. @Y@m@d@H@M00 {%- if use_rrfse_ens %} + {%- if do_retro %} @@ -1224,6 +1225,7 @@ MODULES_RUN_TASK_FP script. {%- endif %} + {%- endif %} @@ -1271,6 +1273,7 @@ MODULES_RUN_TASK_FP script. {%- endif %} {%- if use_rrfse_ens %} + {%- if do_retro %} @@ -1308,6 +1311,7 @@ MODULES_RUN_TASK_FP script. {%- endif %} + {%- endif %} @@ -2107,6 +2111,7 @@ MODULES_RUN_TASK_FP script. @Y@m@d@H@M00 {%- if use_rrfse_ens %} + {%- if do_retro %} @@ -2152,6 +2157,7 @@ MODULES_RUN_TASK_FP script. {%- endif %} + {%- endif %}