Skip to content

Commit

Permalink
workflow re-organization (#454)
Browse files Browse the repository at this point in the history 
* Move data preprocessing to the start of each cycle
Separate spinup and prod cycle tasks

* Re-order EnKF related tasks

* Restore access for exregional_run_analysis.sh

* Turn off DA for ensfcst runs

* Add control for post-processing

* updates for wcoss2:
1 remove gzip
2 change to use gfs 16.3
3 change workflow to set resources in head of xml.

Co-authored-by: ming hu <ming.hu@clogin03.cactus.wcoss2.ncep.noaa.gov>
  • Loading branch information
@chunhuazhou
chunhuazhou and ming hu committed Dec 6, 2022
1 parent 7599d35 commit 6bb297f
Show file tree
Hide file tree
Showing 15 changed files with 1,366 additions and 997 deletions.
1 change: 0 additions & 1 deletion scripts/exregional_jedienvar_ioda.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,6 @@ cp_vrfy ${comout}/jedienvar_ioda/anal_gsi/*ges* ${workdir}/GSI_diags
#-----------------------------------------------------------------------
#
cd_vrfy ${workdir}/GSI_diags
gzip -d *.gz
fl=`ls -1 ncdiag*`

for ifl in $fl
Expand Down
49 changes: 20 additions & 29 deletions scripts/exregional_run_analysis.sh
100644 → 100755
View file
Original file line number Diff line number Diff line change
Expand Up @@ -860,16 +860,22 @@ fi
$APRUN ./gsi.x < gsiparm.anl > stdout 2>&1 || print_err_msg_exit "\
Call to executable to run GSI returned with nonzero exit code."

mv fort.207 fit_rad1
sed -e 's/ asm all/ps asm 900/; s/ rej all/ps rej 900/; s/ mon all/ps mon 900/' fort.201 > fit_p1
sed -e 's/ asm all/uv asm 900/; s/ rej all/uv rej 900/; s/ mon all/uv mon 900/' fort.202 > fit_w1
sed -e 's/ asm all/ t asm 900/; s/ rej all/ t rej 900/; s/ mon all/ t mon 900/' fort.203 > fit_t1
sed -e 's/ asm all/ q asm 900/; s/ rej all/ q rej 900/; s/ mon all/ q mon 900/' fort.204 > fit_q1
sed -e 's/ asm all/pw asm 900/; s/ rej all/pw rej 900/; s/ mon all/pw mon 900/' fort.205 > fit_pw1
sed -e 's/ asm all/rw asm 900/; s/ rej all/rw rej 900/; s/ mon all/rw mon 900/' fort.209 > fit_rw1

cat fit_p1 fit_w1 fit_t1 fit_q1 fit_pw1 fit_rad1 fit_rw1 > $comout/rrfs_a.t${HH}z.fits.tm00
cat fort.208 fort.210 fort.211 fort.212 fort.213 fort.220 > $comout/rrfs_a.t${HH}z.fits2.tm00
if [ ${gsi_type} == "ANALYSIS" ]; then
if [ ${ob_type} == "radardbz" ]; then
cat fort.238 > $comout/rrfs_a.t${HH}z.fits3.tm00
else
mv fort.207 fit_rad1
sed -e 's/ asm all/ps asm 900/; s/ rej all/ps rej 900/; s/ mon all/ps mon 900/' fort.201 > fit_p1
sed -e 's/ asm all/uv asm 900/; s/ rej all/uv rej 900/; s/ mon all/uv mon 900/' fort.202 > fit_w1
sed -e 's/ asm all/ t asm 900/; s/ rej all/ t rej 900/; s/ mon all/ t mon 900/' fort.203 > fit_t1
sed -e 's/ asm all/ q asm 900/; s/ rej all/ q rej 900/; s/ mon all/ q mon 900/' fort.204 > fit_q1
sed -e 's/ asm all/pw asm 900/; s/ rej all/pw rej 900/; s/ mon all/pw mon 900/' fort.205 > fit_pw1
sed -e 's/ asm all/rw asm 900/; s/ rej all/rw rej 900/; s/ mon all/rw mon 900/' fort.209 > fit_rw1

cat fit_p1 fit_w1 fit_t1 fit_q1 fit_pw1 fit_rad1 fit_rw1 > $comout/rrfs_a.t${HH}z.fits.tm00
cat fort.208 fort.210 fort.211 fort.212 fort.213 fort.220 > $comout/rrfs_a.t${HH}z.fits2.tm00
fi
fi
#
#-----------------------------------------------------------------------
#
Expand Down Expand Up @@ -941,25 +947,11 @@ if [ $netcdf_diag = ".true." ]; then
listall_cnv="conv_ps conv_q conv_t conv_uv conv_pw conv_rw conv_sst conv_dbz"
listall_rad="hirs2_n14 msu_n14 sndr_g08 sndr_g11 sndr_g11 sndr_g12 sndr_g13 sndr_g08_prep sndr_g11_prep sndr_g12_prep sndr_g13_prep sndrd1_g11 sndrd2_g11 sndrd3_g11 sndrd4_g11 sndrd1_g15 sndrd2_g15 sndrd3_g15 sndrd4_g15 sndrd1_g13 sndrd2_g13 sndrd3_g13 sndrd4_g13 hirs3_n15 hirs3_n16 hirs3_n17 amsua_n15 amsua_n16 amsua_n17 amsua_n18 amsua_n19 amsua_metop-a amsua_metop-b amsua_metop-c amsub_n15 amsub_n16 amsub_n17 hsb_aqua airs_aqua amsua_aqua imgr_g08 imgr_g11 imgr_g12 pcp_ssmi_dmsp pcp_tmi_trmm conv sbuv2_n16 sbuv2_n17 sbuv2_n18 omi_aura ssmi_f13 ssmi_f14 ssmi_f15 hirs4_n18 hirs4_metop-a mhs_n18 mhs_n19 mhs_metop-a mhs_metop-b mhs_metop-c amsre_low_aqua amsre_mid_aqua amsre_hig_aqua ssmis_las_f16 ssmis_uas_f16 ssmis_img_f16 ssmis_env_f16 iasi_metop-a iasi_metop-b iasi_metop-c seviri_m08 seviri_m09 seviri_m10 seviri_m11 cris_npp atms_npp ssmis_f17 cris-fsr_npp cris-fsr_n20 atms_n20 abi_g16"

cat_exec="${EXECDIR}/nc_diag_cat.x"

if [ -f $cat_exec ]; then
print_info_msg "$VERBOSE" "
Copying the nc_diag_cat executable to the run directory..."
cp_vrfy ${cat_exec} ${analworkdir}/nc_diag_cat.x
else
print_err_msg_exit "\
The nc_diag_cat executable specified in cat_exec does not exist:
cat_exec = \"$cat_exec\"
Build GSI and rerun."
fi

for type in $listall_cnv; do
count=$(ls pe*.${type}_${loop}.nc4 | wc -l)
if [[ $count -gt 0 ]]; then
${APRUN} ./nc_diag_cat.x -o diag_${type}_${string}.${YYYYMMDDHH}.nc4 pe*.${type}_${loop}.nc4
gzip diag_${type}_${string}.${YYYYMMDDHH}.nc4*
cp diag_${type}_${string}.${YYYYMMDDHH}.nc4.gz $comout
${APRUN} nc_diag_cat.x -o diag_${type}_${string}.${YYYYMMDDHH}.nc4 pe*.${type}_${loop}.nc4
cp diag_${type}_${string}.${YYYYMMDDHH}.nc4 $comout
echo "diag_${type}_${string}.${YYYYMMDDHH}.nc4*" >> listcnv
numfile_cnv=`expr ${numfile_cnv} + 1`
fi
Expand All @@ -968,9 +960,8 @@ if [ $netcdf_diag = ".true." ]; then
for type in $listall_rad; do
count=$(ls pe*.${type}_${loop}.nc4 | wc -l)
if [[ $count -gt 0 ]]; then
${APRUN} ./nc_diag_cat.x -o diag_${type}_${string}.${YYYYMMDDHH}.nc4 pe*.${type}_${loop}.nc4
gzip diag_${type}_${string}.${YYYYMMDDHH}.nc4*
cp diag_${type}_${string}.${YYYYMMDDHH}.nc4.gz $comout
${APRUN} nc_diag_cat.x -o diag_${type}_${string}.${YYYYMMDDHH}.nc4 pe*.${type}_${loop}.nc4
cp diag_${type}_${string}.${YYYYMMDDHH}.nc4 $comout
echo "diag_${type}_${string}.${YYYYMMDDHH}.nc4*" >> listrad
numfile_rad=`expr ${numfile_rad} + 1`
else
Expand Down
10 changes: 4 additions & 6 deletions scripts/exregional_run_enkf.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ case $MACHINE in
ulimit -s unlimited
ulimit -a
export FI_OFI_RXM_SAR_LIMIT=3145728
export OMP_STACKSIZE=500M
export OMP_STACKSIZE=1G
export OMP_NUM_THREADS=1
ncores=$(( NNODES_RUN_ENKF*PPN_RUN_ENKF ))
APRUN="mpiexec -n ${ncores} -ppn ${PPN_RUN_ENKF} --cpu-bind core --depth ${OMP_NUM_THREADS}"
Expand Down Expand Up @@ -206,14 +206,12 @@ mkdir_vrfy -p ${enkfanal_nwges_dir}
list_ob_type="conv_dbz"
fi
for sub_ob_type in ${list_ob_type} ; do
diagfile0=${observer_nwges_dir}/diag_${sub_ob_type}_ges.${YYYYMMDDHH}.nc4.gz
diagfile0=${observer_nwges_dir}/diag_${sub_ob_type}_ges.${YYYYMMDDHH}.nc4
if [ -s $diagfile0 ]; then
diagfile=$(basename $diagfile0)
cp_vrfy $diagfile0 $diagfile
gzip -d $diagfile && rm -f $diagfile
ncfile0=$(basename -s .gz $diagfile)
ncfile=$(basename -s .nc4 $ncfile0)
mv_vrfy $ncfile0 ${ncfile}_${memcharv0}.nc4
ncfile=$(basename -s .nc4 $diagfile)
mv_vrfy $ncfile.nc4 ${ncfile}_${memcharv0}.nc4
fi
done
else
Expand Down
8 changes: 8 additions & 0 deletions scripts/exregional_run_wgrib2.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,9 +236,13 @@ ln_vrfy -sf --relative ${comout}/${NET}.t${cyc}z.bgsfcf${fhr}.${tmmark}.grib2 $

net4=$(echo ${NET:0:4} | tr '[:upper:]' '[:lower:]')
ln_vrfy -sf --relative ${comout}/${NET}.t${cyc}z.bgdawpf${fhr}.${tmmark}.grib2 ${comout}/${net4}.t${cyc}z.prslev.f${fhr}.conus_3km.grib2
wgrib2 ${comout}/${net4}.t${cyc}z.prslev.f${fhr}.conus_3km.grib2 -s > ${comout}/${net4}.t${cyc}z.prslev.f${fhr}.conus_3km.grib2.idx
ln_vrfy -sf --relative ${comout}/${NET}.t${cyc}z.bgrd3df${fhr}.${tmmark}.grib2 ${comout}/${net4}.t${cyc}z.natlev.f${fhr}.conus_3km.grib2
wgrib2 ${comout}/${net4}.t${cyc}z.natlev.f${fhr}.conus_3km.grib2 -s > ${comout}/${net4}.t${cyc}z.natlev.f${fhr}.conus_3km.grib2.idx
ln_vrfy -sf --relative ${comout}/${NET}.t${cyc}z.bgifif${fhr}.${tmmark}.grib2 ${comout}/${net4}.t${cyc}z.ififip.f${fhr}.conus_3km.grib2
wgrib2 ${comout}/${net4}.t${cyc}z.ififip.f${fhr}.conus_3km.grib2 -s > ${comout}/${net4}.t${cyc}z.ififip.f${fhr}.conus_3km.grib2.idx
ln_vrfy -sf --relative ${comout}/${NET}.t${cyc}z.bgsfcf${fhr}.${tmmark}.grib2 ${comout}/${net4}.t${cyc}z.testbed.f${fhr}.conus_3km.grib2
wgrib2 ${comout}/${net4}.t${cyc}z.testbed.f${fhr}.conus_3km.grib2 -s > ${comout}/${net4}.t${cyc}z.testbed.f${fhr}.conus_3km.grib2.idx
# Remap to additional output grids if requested
if [ ${#ADDNL_OUTPUT_GRIDS[@]} -gt 0 ]; then

Expand Down Expand Up @@ -295,18 +299,22 @@ if [ ${#ADDNL_OUTPUT_GRIDS[@]} -gt 0 ]; then

if [ $leveltype = 'dawp' ]; then
ln_vrfy -fs --relative ${comout}/${grid}_grid/${NET}.t${cyc}z.bg${leveltype}f${fhr}.${tmmark}.grib2 ${comout}/${net4}.t${cyc}z.prslev.f${fhr}.conus_3km.grib2
wgrib2 ${comout}/${net4}.t${cyc}z.prslev.f${fhr}.conus_3km.grib2 -s > ${comout}/${net4}.t${cyc}z.prslev.f${fhr}.conus_3km.grib2.idx
fi

if [ $leveltype = 'rd3d' ]; then
ln_vrfy -fs --relative ${comout}/${grid}_grid/${NET}.t${cyc}z.bg${leveltype}f${fhr}.${tmmark}.grib2 ${comout}/${net4}.t${cyc}z.natlev.f${fhr}.conus_3km.grib2
wgrib2 ${comout}/${net4}.t${cyc}z.natlev.f${fhr}.conus_3km.grib2 -s > ${comout}/${net4}.t${cyc}z.natlev.f${fhr}.conus_3km.grib2.idx
fi

if [ $leveltype = 'ifi' ]; then
ln_vrfy -fs --relative ${comout}/${grid}_grid/${NET}.t${cyc}z.bg${leveltype}f${fhr}.${tmmark}.grib2 ${comout}/${net4}.t${cyc}z.ififip.f${fhr}.conus_3km.grib2
wgrib2 ${comout}/${net4}.t${cyc}z.ififip.f${fhr}.conus_3km.grib2 -s > ${comout}/${net4}.t${cyc}z.ififip.f${fhr}.conus_3km.grib2.idx
fi

if [ $leveltype = 'sfc' ]; then
ln_vrfy -fs --relative ${comout}/${grid}_grid/${NET}.t${cyc}z.bg${leveltype}f${fhr}.${tmmark}.grib2 ${comout}/${net4}.t${cyc}z.testbed.f${fhr}.conus_3km.grib2
wgrib2 ${comout}/${net4}.t${cyc}z.testbed.f${fhr}.conus_3km.grib2 -s > ${comout}/${net4}.t${cyc}z.testbed.f${fhr}.conus_3km.grib2.idx
fi

# Link output for transfer from Jet to web
Expand Down
2 changes: 2 additions & 0 deletions ush/config.sh.RRFS_CONUS_13km_ens
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ DO_ENSEMBLE="TRUE"
#DO_DACYCLE="TRUE"
#DO_SURFACE_CYCLE="TRUE"
DO_SPINUP="TRUE"
#DO_POST_SPINUP="TRUE"
DO_POST_PROD="TRUE"
DO_RETRO="TRUE"
DO_NONVAR_CLDANAL="TRUE"
#DO_ENVAR_RADAR_REF="TRUE"
Expand Down
2 changes: 2 additions & 0 deletions ush/config.sh.RRFS_CONUS_3km
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@ TILE_SETS="full NE,NC,NW,SE,SC,SW"
DO_DACYCLE="TRUE"
DO_SURFACE_CYCLE="TRUE"
DO_SPINUP="TRUE"
#DO_POST_SPINUP="TRUE"
DO_POST_PROD="TRUE"
DO_RETRO="TRUE"
DO_NONVAR_CLDANAL="TRUE"
#USE_RRFSE_ENS="TRUE"
Expand Down
2 changes: 2 additions & 0 deletions ush/config.sh.RRFS_CONUS_3km.2022HWT
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ TILE_SETS="full NE,NC,NW,SE,SC,SW"
DO_DACYCLE="TRUE"
DO_SURFACE_CYCLE="TRUE"
DO_SPINUP="TRUE"
#DO_POST_SPINUP="TRUE"
DO_POST_PROD="TRUE"
DO_RETRO="FALSE"
DO_NONVAR_CLDANAL="TRUE"
DO_ENVAR_RADAR_REF="TRUE"
Expand Down
2 changes: 2 additions & 0 deletions ush/config.sh.RRFS_CONUS_3km_ens
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ DO_ENSEMBLE="TRUE"
#DO_DACYCLE="TRUE"
#DO_SURFACE_CYCLE="TRUE"
DO_SPINUP="TRUE"
#DO_POST_SPINUP="TRUE"
DO_POST_PROD="TRUE"
#DO_RETRO="TRUE"
DO_NONVAR_CLDANAL="TRUE"
DO_SMOKE_DUST="FALSE"
Expand Down
8 changes: 8 additions & 0 deletions ush/config_defaults.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2061,9 +2061,17 @@ DO_SMOKE_DUST="FALSE"
# Flag turn on the runs prepare boundary and cold start initial conditions in
# retrospective experiments.
#
# DO_POST_SPINUP:
# Flag turn on the UPP for spin-up cycle.
#
# DO_POST_PROD:
# Flag turn on the UPP for prod cycle.
#
DO_RETRO="FALSE"
DO_SPINUP="FALSE"
LBCS_ICS_ONLY="FALSE"
DO_POST_SPINUP="FALSE"
DO_POST_PROD="TRUE"
#
#-----------------------------------------------------------------------
#
Expand Down
2 changes: 2 additions & 0 deletions ush/generate_FV3LAM_wflow.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -482,6 +482,8 @@ settings="\
'do_nonvar_cldanal': ${DO_NONVAR_CLDANAL}
'do_refl2tten': ${DO_REFL2TTEN}
'do_spinup': ${DO_SPINUP}
'do_post_spinup': ${DO_POST_SPINUP}
'do_post_prod': ${DO_POST_PROD}
'do_nldn_lght': ${DO_NLDN_LGHT}
'regional_ensemble_option': ${regional_ensemble_option}
'radar_ref_thinning': ${RADAR_REF_THINNING}
Expand Down
2 changes: 1 addition & 1 deletion ush/launch_FV3LAM_wflow.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ elif [ "$MACHINE" = "WCOSS2" ]; then
module load libfabric/1.11.0.0.
module load craype-network-ofi
module load envvar/1.0
module load core/rocoto/1.3.4
module load core/rocoto/1.3.5
else
module purge
module load rocoto
Expand Down
4 changes: 2 additions & 2 deletions ush/set_extrn_mdl_params.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ else
"FV3GFS")
case $MACHINE in
"WCOSS2")
EXTRN_MDL_SYSBASEDIR_ICS="/lfs/h1/ops/prod/com/gfs/v16.2"
EXTRN_MDL_SYSBASEDIR_ICS="/lfs/h1/ops/prod/com/gfs/v16.3"
;;
"HERA")
EXTRN_MDL_SYSBASEDIR_ICS="/scratch1/NCEPDEV/rstprod/com/gfs/prod"
Expand Down Expand Up @@ -282,7 +282,7 @@ else
"FV3GFS")
case $MACHINE in
"WCOSS2")
EXTRN_MDL_SYSBASEDIR_LBCS="/lfs/h1/ops/prod/com/gfs/v16.2"
EXTRN_MDL_SYSBASEDIR_LBCS="/lfs/h1/ops/prod/com/gfs/v16.3"
;;
"HERA")
EXTRN_MDL_SYSBASEDIR_LBCS="/scratch1/NCEPDEV/rstprod/com/gfs/prod"
Expand Down
12 changes: 6 additions & 6 deletions ush/set_rrfs_config.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,14 @@ if [[ $MACHINE == "hera" ]] ; then
fi

if [[ $MACHINE == "wcoss2" ]] ; then
EXTRN_MDL_SOURCE_BASEDIR_ICS=/lfs/h1/ops/prod/com/gfs/v16.2
EXTRN_MDL_SOURCE_BASEDIR_LBCS=/lfs/h1/ops/prod/com/gfs/v16.2
OBSPATH=/lfs/h1/ops/prod/com/obsproc/v1.0
EXTRN_MDL_SOURCE_BASEDIR_ICS=/lfs/h1/ops/prod/com/gfs/v16.3
EXTRN_MDL_SOURCE_BASEDIR_LBCS=/lfs/h1/ops/prod/com/gfs/v16.3
OBSPATH=/lfs/h1/ops/prod/com/obsproc/v1.1
OBSPATH_NSSLMOSIAC=/lfs/h1/ops/prod/dcom/ldmdata/obs/upperair/mrms/conus/MergedReflectivityQC
ENKF_FCST=/lfs/h1/ops/prod/com/gfs/v16.2
SST_ROOT=/lfs/h1/ops/prod/com/gfs/v16.2
ENKF_FCST=/lfs/h1/ops/prod/com/gfs/v16.3
SST_ROOT=/lfs/h1/ops/prod/com/gfs/v16.3
GVF_ROOT=/lfs/h1/ops/prod/dcom/viirs
IMSSNOW_ROOT=/lfs/h1/ops/prod/com/obsproc/v1.0
IMSSNOW_ROOT=/lfs/h1/ops/prod/com/obsproc/v1.1
FIRE_RAVE_DIR=/lfs/h2/emc/physics/noscrub/partha.bhattacharjee/RAVE_rawdata/RAVE_NA
fi

Expand Down
Loading

0 comments on commit 6bb297f

Please sign in to comment.