From b3e8156dfad7971b3705152d062b8ebfecfa7acd Mon Sep 17 00:00:00 2001 From: "russ.treadon" Date: Wed, 15 Dec 2021 20:50:15 +0000 Subject: [PATCH 01/45] Issue #529: add ufsda to checkout, build, and link --- sorc/build_all.sh | 14 ++++++++++++++ sorc/build_ufs_da.sh | 26 ++++++++++++++++++++++++++ sorc/checkout.sh | 11 +++++++++++ sorc/link_workflow.sh | 8 ++++++++ 4 files changed, 59 insertions(+) create mode 100755 sorc/build_ufs_da.sh diff --git a/sorc/build_all.sh b/sorc/build_all.sh index 6c8bfabcd0..0b80a923ce 100755 --- a/sorc/build_all.sh +++ b/sorc/build_all.sh @@ -113,6 +113,20 @@ fi ((err+=$rc)) } +#------------------------------------ +# build ufs_da +#------------------------------------ +$Build_ufs_da && { +echo " .... Building ufs_da .... " +./build_ufs_da.sh > $logs_dir/build_ufs_da.log 2>&1 +rc=$? +if [[ $rc -ne 0 ]] ; then + echo "Fatal error in building ufs_da." + echo "The log file is in $logs_dir/build_ufs_da.log" +fi +((err+=$rc)) +} + #------------------------------------ # build ncep_post #------------------------------------ diff --git a/sorc/build_ufs_da.sh b/sorc/build_ufs_da.sh new file mode 100755 index 0000000000..fff1fd105b --- /dev/null +++ b/sorc/build_ufs_da.sh @@ -0,0 +1,26 @@ +#! /usr/bin/env bash +set -eux + +source ./machine-setup.sh > /dev/null 2>&1 +cwd=`pwd` + +USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} +if [ $USE_PREINST_LIBS = true ]; then + export MOD_PATH=/scratch3/NCEPDEV/nwprod/lib/modulefiles +else + export MOD_PATH=${cwd}/lib/modulefiles +fi + +export BUILD_TARGET=$target +[[ "$target" == wcoss_cray ]] && BUILD_TARGET=cray + +# Check final exec folder exists +if [ ! -d "../exec" ]; then + mkdir ../exec +fi + +cd ufs_da.fd/UFS-DA/ush +./build_UFSDA.sh + +exit + diff --git a/sorc/checkout.sh b/sorc/checkout.sh index 57503a03f5..f9283bc737 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -50,6 +50,17 @@ else echo 'Skip. Directory gsi.fd already exists.' fi +echo ufs_da checkout ... +if [[ ! -d ufs_da.fd ]] ; then + rm -f ${topdir}/checkout-ufs_da.log + git clone --recursive https://github.com/NOAA-EMC/JEDI-T2O.git ufs_da.fd >> ${logdir}/checkout-ufs_da.log 2>&1 + cd ufs_da.fd + git checkout develop + cd ${topdir} +else + echo 'Skip. Directory ufs_da.fd already exists.' +fi + echo gldas checkout ... if [[ ! -d gldas.fd ]] ; then rm -f ${topdir}/checkout-gldas.log diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index a2742743bd..3aa4d02fd5 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -283,6 +283,14 @@ for gsiexe in calc_analysis.x calc_increment_ens_ncio.x calc_increment_ens.x \ $LINK ../sorc/gsi.fd/exec/$gsiexe . done +for ufsdaexe in fv3jedi_addincrement.x fv3jedi_diffstates.x fv3jedi_ensvariance.x fv3jedi_hofx.x \ + fv3jedi_var.x fv3jedi_convertincrement.x fv3jedi_dirac.x fv3jedi_error_covariance_training.x \ + fv3jedi_letkf.x fv3jedi_convertstate.x fv3jedi_eda.x fv3jedi_forecast.x fv3jedi_plot_field.x \ + fv3jedi_data_checker.py fv3jedi_enshofx.x fv3jedi_hofx_nomodel.x fv3jedi_testdata_downloader.py; do + [[ -s $ufsdaexe ]] && rm -f $ufsdaexe + $LINK ../sorc/ufs_da.fd/UFS-DA/bin/$ufsdaexe . +done + for gldasexe in gdas2gldas gldas2gdas gldas_forcing gldas_model gldas_post gldas_rst; do [[ -s $gldasexe ]] && rm -f $gldasexe $LINK ../sorc/gldas.fd/exec/$gldasexe . From 866366600ccaba355b2551b0be4531bf83cd4ab5 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Mon, 20 Dec 2021 14:34:27 -0600 Subject: [PATCH 02/45] atmanal rocoto job scripts --- jobs/rocoto/atmanalpost.sh | 13 +++++++++++++ jobs/rocoto/atmanalprep.sh | 13 +++++++++++++ jobs/rocoto/atmanalrun.sh | 13 +++++++++++++ 3 files changed, 39 insertions(+) create mode 100755 jobs/rocoto/atmanalpost.sh create mode 100755 jobs/rocoto/atmanalprep.sh create mode 100755 jobs/rocoto/atmanalrun.sh diff --git a/jobs/rocoto/atmanalpost.sh b/jobs/rocoto/atmanalpost.sh new file mode 100755 index 0000000000..358786a3b2 --- /dev/null +++ b/jobs/rocoto/atmanalpost.sh @@ -0,0 +1,13 @@ +#!/bin/ksh -x + +############################################################### +# Source FV3GFS workflow modules +. $HOMEgfs/ush/load_fv3gfs_modules.sh +status=$? +[[ $status -ne 0 ]] && exit $status + +############################################################### +# Execute the JJOB +$HOMEgfs/jobs/JUFSDA_POST_GLOBAL_ATMOS_ANALYSIS +status=$? +exit $status diff --git a/jobs/rocoto/atmanalprep.sh b/jobs/rocoto/atmanalprep.sh new file mode 100755 index 0000000000..45719a8760 --- /dev/null +++ b/jobs/rocoto/atmanalprep.sh @@ -0,0 +1,13 @@ +#!/bin/ksh -x + +############################################################### +# Source FV3GFS workflow modules +. $HOMEgfs/ush/load_fv3gfs_modules.sh +status=$? +[[ $status -ne 0 ]] && exit $status + +############################################################### +# Execute the JJOB +$HOMEgfs/jobs/JUFSDA_PREP_GLOBAL_ATMOS_ANALYSIS +status=$? +exit $status diff --git a/jobs/rocoto/atmanalrun.sh b/jobs/rocoto/atmanalrun.sh new file mode 100755 index 0000000000..a216e6cd1a --- /dev/null +++ b/jobs/rocoto/atmanalrun.sh @@ -0,0 +1,13 @@ +#!/bin/ksh -x + +############################################################### +# Source FV3GFS workflow modules +. $HOMEgfs/ush/load_fv3gfs_modules.sh +status=$? +[[ $status -ne 0 ]] && exit $status + +############################################################### +# Execute the JJOB +$HOMEgfs/jobs/JUFSDA_RUN_GLOBAL_ATMOS_ANALYSIS +status=$? +exit $status From b583b44bae56dff1e8fca26c7d1f2e0f9d07b1c5 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Tue, 21 Dec 2021 12:21:44 -0600 Subject: [PATCH 03/45] Start of env and parm/config changes for atmanal --- env/ORION.env | 7 +++++++ parm/config/config.atmanal | 0 parm/config/config.atmanalpost | 0 parm/config/config.atmanalprep | 0 parm/config/config.atmanalrun | 0 parm/config/config.resources | 1 + 6 files changed, 8 insertions(+) create mode 100644 parm/config/config.atmanal create mode 100644 parm/config/config.atmanalpost create mode 100644 parm/config/config.atmanalprep create mode 100755 parm/config/config.atmanalrun diff --git a/env/ORION.env b/env/ORION.env index 214493188a..ad31023b5f 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -4,6 +4,7 @@ if [ $# -ne 1 ]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" + echo "atmanal" echo "anal fcst post vrfy metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -50,6 +51,12 @@ elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $ste export wavempexec=${launcher} export wave_mpmd=${mpmd} +elif [ $step = "atmanal" ]; then + + export CFP_MP=${CFP_MP:-"YES"} + export USE_CFP=${USE_CFP:-"YES"} + export APRUNCFP="$launcher -n \$ncmd --multi-prog" + elif [ $step = "anal" ]; then export MKL_NUM_THREADS=4 diff --git a/parm/config/config.atmanal b/parm/config/config.atmanal new file mode 100644 index 0000000000..e69de29bb2 diff --git a/parm/config/config.atmanalpost b/parm/config/config.atmanalpost new file mode 100644 index 0000000000..e69de29bb2 diff --git a/parm/config/config.atmanalprep b/parm/config/config.atmanalprep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/parm/config/config.atmanalrun b/parm/config/config.atmanalrun new file mode 100755 index 0000000000..e69de29bb2 diff --git a/parm/config/config.resources b/parm/config/config.resources index 71c96acba1..49bc986970 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -9,6 +9,7 @@ if [ $# -ne 1 ]; then echo "Must specify an input task argument to set resource variables!" echo "argument can be any one of the following:" echo "getic init coupled_ic" + echo "atmanalprep atmanalrun atmanalpost" echo "anal analcalc analdiag gldas fcst post vrfy metp arch echgres" echo "eobs ediag eomg eupd ecen esfc efcs epos earc" echo "init_chem mom6ic ocnpost" From 40eeda1c590670ffb428edc9a61615f6ec714772 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Wed, 22 Dec 2021 10:01:35 -0600 Subject: [PATCH 04/45] Changes to parm/config files --- parm/config/config.atmanal | 8 ++++++++ parm/config/config.atmanalpost | 10 ++++++++++ parm/config/config.atmanalprep | 10 ++++++++++ parm/config/config.atmanalrun | 10 ++++++++++ parm/config/config.resources | 24 ++++++++++++++++++++++++ 5 files changed, 62 insertions(+) mode change 100644 => 100755 parm/config/config.atmanal mode change 100644 => 100755 parm/config/config.atmanalpost mode change 100644 => 100755 parm/config/config.atmanalprep diff --git a/parm/config/config.atmanal b/parm/config/config.atmanal old mode 100644 new mode 100755 index e69de29bb2..7e4a4b2e04 --- a/parm/config/config.atmanal +++ b/parm/config/config.atmanal @@ -0,0 +1,8 @@ +#!/bin/ksh -x + +########## config.atmanal ########## +# configuration common to all atm analysis tasks + +echo "BEGIN: config.atmatmanal" + +echo "END: config.atmanal" diff --git a/parm/config/config.atmanalpost b/parm/config/config.atmanalpost old mode 100644 new mode 100755 index e69de29bb2..e5210b9bf4 --- a/parm/config/config.atmanalpost +++ b/parm/config/config.atmanalpost @@ -0,0 +1,10 @@ +#!/bin/ksh -x + +########## config.atmanalpost ########## +# Post Atm Analysis specific + +echo "BEGIN: config.atmanalpost" + +# Get task specific resources +. $EXPDIR/config.resources atmanalpost +echo "END: config.atmanalpost" diff --git a/parm/config/config.atmanalprep b/parm/config/config.atmanalprep old mode 100644 new mode 100755 index e69de29bb2..8764183a51 --- a/parm/config/config.atmanalprep +++ b/parm/config/config.atmanalprep @@ -0,0 +1,10 @@ +#!/bin/ksh -x + +########## config.atmanalprep ########## +# Pre Atm Analysis specific + +echo "BEGIN: config.atmanalprep" + +# Get task specific resources +. $EXPDIR/config.resources atmanalprep +echo "END: config.atmanalprep" diff --git a/parm/config/config.atmanalrun b/parm/config/config.atmanalrun index e69de29bb2..8a580eacd5 100755 --- a/parm/config/config.atmanalrun +++ b/parm/config/config.atmanalrun @@ -0,0 +1,10 @@ +#!/bin/ksh -x + +########## config.atmanalrun ########## +# Atm Analysis specific + +echo "BEGIN: config.atmanalrun" + +# Get task specific resources +. $EXPDIR/config.resources atmanalrun +echo "END: config.atmanalrun" diff --git a/parm/config/config.resources b/parm/config/config.resources index 49bc986970..b41964470c 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -126,6 +126,30 @@ elif [ $step = "waveawipsgridded" ]; then export npe_node_waveawipsgridded=$(echo "$npe_node_max / $nth_waveawipsgridded" | bc) export NTASKS=${npe_waveawipsgridded} +elif [ $step = "atmanalprep" ]; then + + export wtime_atmanalprep="00:10:00" + export npe_atmanalprep=1 + export nth_atmanalprep=1 + export npe_node_atmanalprep=$(echo "$npe_node_max / $nth_atmanalprep" | bc) + export memory_atmanalprep="3072M" + +elif [ $step = "atmanalrun" ]; then + + export wtime_atmanalrun="01:00:00" + export npe_atmanalrun=120 + export npe_atmanalrun_gfs=120 + export nth_atmanalrun=1 + export nth_atmanalrun_gfs=$nth_atmanalrun + export npe_node_atmanalrun=$(echo "$npe_node_max / $nth_atmanalrun" | bc) + +elif [ $step = "atmanalpost" ]; then + + export wtime_atmanalpost="10:00:00" + export npe_atmanalpost=$npe_node_max + export nth_atmanalpost=1 + export npe_node_atmanalpost=$(echo "$npe_node_max / $nth_atmanalpost" | bc) + elif [ $step = "anal" ]; then export wtime_anal="01:00:00" From 6948c0f67f093baa091d52814eec075c2d57ce2d Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Wed, 22 Dec 2021 10:07:40 -0600 Subject: [PATCH 05/45] Make rocoto jobs all use same j-job for atmanal tasks --- jobs/rocoto/atmanalpost.sh | 5 ++++- jobs/rocoto/atmanalprep.sh | 5 ++++- jobs/rocoto/atmanalrun.sh | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/jobs/rocoto/atmanalpost.sh b/jobs/rocoto/atmanalpost.sh index 358786a3b2..18e599a130 100755 --- a/jobs/rocoto/atmanalpost.sh +++ b/jobs/rocoto/atmanalpost.sh @@ -7,7 +7,10 @@ status=$? [[ $status -ne 0 ]] && exit $status ############################################################### +# set JJOB variables +export configs="base atmanal atmanalpost" +export EXSCRIPT=${UFSDAPOSTPY:-$SCRgfs/exufsda_global_atmos_analysis_post.py # Execute the JJOB -$HOMEgfs/jobs/JUFSDA_POST_GLOBAL_ATMOS_ANALYSIS +$HOMEgfs/jobs/JUFSDA_GLOBAL_ATMOS_ANALYSIS status=$? exit $status diff --git a/jobs/rocoto/atmanalprep.sh b/jobs/rocoto/atmanalprep.sh index 45719a8760..0af6fccd25 100755 --- a/jobs/rocoto/atmanalprep.sh +++ b/jobs/rocoto/atmanalprep.sh @@ -7,7 +7,10 @@ status=$? [[ $status -ne 0 ]] && exit $status ############################################################### +# set JJOB variables +export configs="base atmanal atmanalprep" +export EXSCRIPT=${UFSDAPREPPY:-$SCRgfs/exufsda_global_atmos_analysis_prep.py # Execute the JJOB -$HOMEgfs/jobs/JUFSDA_PREP_GLOBAL_ATMOS_ANALYSIS +$HOMEgfs/jobs/JUFSDA_GLOBAL_ATMOS_ANALYSIS status=$? exit $status diff --git a/jobs/rocoto/atmanalrun.sh b/jobs/rocoto/atmanalrun.sh index a216e6cd1a..5e7e712e4e 100755 --- a/jobs/rocoto/atmanalrun.sh +++ b/jobs/rocoto/atmanalrun.sh @@ -7,7 +7,10 @@ status=$? [[ $status -ne 0 ]] && exit $status ############################################################### +# set JJOB variables +export configs="base atmanal atmanalrun" +export EXSCRIPT=${UFSDARUNSH:-$SCRgfs/exufsda_global_atmos_analysis_run.sh # Execute the JJOB -$HOMEgfs/jobs/JUFSDA_RUN_GLOBAL_ATMOS_ANALYSIS +$HOMEgfs/jobs/JUFSDA_GLOBAL_ATMOS_ANALYSIS status=$? exit $status From 9caf5cc24ead22e59dc025071192982771ef116d Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Wed, 22 Dec 2021 15:01:03 -0600 Subject: [PATCH 06/45] start of atmanal config and added to resources --- parm/config/config.atmanal | 8 ++++++++ parm/config/config.resources | 40 +++++++++++++++++++++--------------- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/parm/config/config.atmanal b/parm/config/config.atmanal index 7e4a4b2e04..59c45a451d 100755 --- a/parm/config/config.atmanal +++ b/parm/config/config.atmanal @@ -5,4 +5,12 @@ echo "BEGIN: config.atmatmanal" +export OBS_YAML=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/config/ufsda_global_atm_obs.yaml +export GEOM_BKG=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/templates/geomges.yaml +export GEOM_ANL=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/templates/geomanl.yaml +export BERROR_YAML=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/templates/ufsda_global_3dvar_b.yaml + +export io_layout_x=1 +export io_layout_y=1 + echo "END: config.atmanal" diff --git a/parm/config/config.resources b/parm/config/config.resources index b41964470c..65f1d99033 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -126,22 +126,30 @@ elif [ $step = "waveawipsgridded" ]; then export npe_node_waveawipsgridded=$(echo "$npe_node_max / $nth_waveawipsgridded" | bc) export NTASKS=${npe_waveawipsgridded} -elif [ $step = "atmanalprep" ]; then - - export wtime_atmanalprep="00:10:00" - export npe_atmanalprep=1 - export nth_atmanalprep=1 - export npe_node_atmanalprep=$(echo "$npe_node_max / $nth_atmanalprep" | bc) - export memory_atmanalprep="3072M" - -elif [ $step = "atmanalrun" ]; then - - export wtime_atmanalrun="01:00:00" - export npe_atmanalrun=120 - export npe_atmanalrun_gfs=120 - export nth_atmanalrun=1 - export nth_atmanalrun_gfs=$nth_atmanalrun - export npe_node_atmanalrun=$(echo "$npe_node_max / $nth_atmanalrun" | bc) +elif [ $step = "atmanalprep" -o $step = "atmanalrun" ]; then + + # make below case dependent later + export layout_x=5 + export layout_y=4 + + if [ $step = "atmanalprep" ]; then + + export wtime_atmanalprep="00:10:00" + export npe_atmanalprep=1 + export nth_atmanalprep=1 + export npe_node_atmanalprep=$(echo "$npe_node_max / $nth_atmanalprep" | bc) + export memory_atmanalprep="3072M" + + elif [ $step = "atmanalrun" ]; then + + export wtime_atmanalrun="01:00:00" + export npe_atmanalrun=$(echo "$layout_x * $layout_y * 6" | bc) + export npe_atmanalrun_gfs=$(echo "$layout_x * $layout_y * 6" | bc) + export nth_atmanalrun=1 + export nth_atmanalrun_gfs=$nth_atmanalrun + export npe_node_atmanalrun=$(echo "$npe_node_max / $nth_atmanalrun" | bc) + + fi elif [ $step = "atmanalpost" ]; then From 59945455fc113c81b0f845315ff8e58e02be6197 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Wed, 22 Dec 2021 15:42:02 -0600 Subject: [PATCH 07/45] Rename static B template YAML --- parm/config/config.atmanal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/config.atmanal b/parm/config/config.atmanal index 59c45a451d..93020e6056 100755 --- a/parm/config/config.atmanal +++ b/parm/config/config.atmanal @@ -8,7 +8,7 @@ echo "BEGIN: config.atmatmanal" export OBS_YAML=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/config/ufsda_global_atm_obs.yaml export GEOM_BKG=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/templates/geomges.yaml export GEOM_ANL=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/templates/geomanl.yaml -export BERROR_YAML=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/templates/ufsda_global_3dvar_b.yaml +export BERROR_YAML=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/templates/ufsda_global_atm_3dvar_b.yaml export io_layout_x=1 export io_layout_y=1 From b706b8ab9e673dba1eafe902cfa532883fe81c38 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Tue, 28 Dec 2021 08:15:32 -0600 Subject: [PATCH 08/45] Add APRUN_ATMANAL to ORION env --- env/ORION.env | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/env/ORION.env b/env/ORION.env index ad31023b5f..6cc21279f3 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -57,6 +57,12 @@ elif [ $step = "atmanal" ]; then export USE_CFP=${USE_CFP:-"YES"} export APRUNCFP="$launcher -n \$ncmd --multi-prog" + nth_max=$(($npe_node_max / $npe_node_atmanalrun)) + + export NTHREADS_ATMANAL=${nth_atmanalrun:-$nth_max} + [[ $NTHREADS_ATMANAL -gt $nth_max ]] && export NTHREADS_ATMANAL=$nth_max + export APRUN_ATMANAL="$launcher" + elif [ $step = "anal" ]; then export MKL_NUM_THREADS=4 From 2b54e9714e38f9e3bfce13d2300ea375a7e23e76 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Wed, 29 Dec 2021 14:18:32 -0600 Subject: [PATCH 09/45] Additional config changes for atmanal --- parm/config/config.atmanal | 10 ++++++++++ parm/config/config.atmanalrun | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/parm/config/config.atmanal b/parm/config/config.atmanal index 93020e6056..5e6d5ac3e6 100755 --- a/parm/config/config.atmanal +++ b/parm/config/config.atmanal @@ -9,6 +9,16 @@ export OBS_YAML=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/config/ufsda_global_atm_obs. export GEOM_BKG=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/templates/geomges.yaml export GEOM_ANL=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/templates/geomanl.yaml export BERROR_YAML=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/templates/ufsda_global_atm_3dvar_b.yaml +export ATMANALPREPYAML=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/templates/stage.yaml +export ATMANALYAML=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/templates/ufsda_global_atm_3dvar.yaml +export R2D2_OBS_DB='ufo_eval_ref' +export R2D2_OBS_DUMP='gdas_f006_nofov_nonsst_noxl' +export R2D2_OBS_SRC='ncdiag' +export R2D2_BC_SRC='gsi' +export R2D2_BC_DUMP='oper' +export R2D2_ARCH_DB='local' +export R2D2_CONFIG=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/config/r2d2_config.yaml +export FV3JEDI_STAGE_YAML=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/config/fv3jedi_stage.yaml export io_layout_x=1 export io_layout_y=1 diff --git a/parm/config/config.atmanalrun b/parm/config/config.atmanalrun index 8a580eacd5..e47938dea6 100755 --- a/parm/config/config.atmanalrun +++ b/parm/config/config.atmanalrun @@ -7,4 +7,8 @@ echo "BEGIN: config.atmanalrun" # Get task specific resources . $EXPDIR/config.resources atmanalrun + +# Task specific variables +export JEDIVAREXE=$HOMEgfs/exec/fv3jedi_var.x + echo "END: config.atmanalrun" From a31b44b5d1d410f0387c32d4e6dd7ebf9dec0905 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Wed, 29 Dec 2021 15:10:08 -0600 Subject: [PATCH 10/45] typo/bugfixes --- jobs/rocoto/atmanalpost.sh | 2 +- jobs/rocoto/atmanalprep.sh | 2 +- jobs/rocoto/atmanalrun.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jobs/rocoto/atmanalpost.sh b/jobs/rocoto/atmanalpost.sh index 18e599a130..3cce56253e 100755 --- a/jobs/rocoto/atmanalpost.sh +++ b/jobs/rocoto/atmanalpost.sh @@ -9,7 +9,7 @@ status=$? ############################################################### # set JJOB variables export configs="base atmanal atmanalpost" -export EXSCRIPT=${UFSDAPOSTPY:-$SCRgfs/exufsda_global_atmos_analysis_post.py +export EXSCRIPT=${UFSDAPOSTPY:-$SCRgfs/exufsda_global_atmos_analysis_post.py} # Execute the JJOB $HOMEgfs/jobs/JUFSDA_GLOBAL_ATMOS_ANALYSIS status=$? diff --git a/jobs/rocoto/atmanalprep.sh b/jobs/rocoto/atmanalprep.sh index 0af6fccd25..3facbedee5 100755 --- a/jobs/rocoto/atmanalprep.sh +++ b/jobs/rocoto/atmanalprep.sh @@ -9,7 +9,7 @@ status=$? ############################################################### # set JJOB variables export configs="base atmanal atmanalprep" -export EXSCRIPT=${UFSDAPREPPY:-$SCRgfs/exufsda_global_atmos_analysis_prep.py +export EXSCRIPT=${UFSDAPREPPY:-$SCRgfs/exufsda_global_atmos_analysis_prep.py} # Execute the JJOB $HOMEgfs/jobs/JUFSDA_GLOBAL_ATMOS_ANALYSIS status=$? diff --git a/jobs/rocoto/atmanalrun.sh b/jobs/rocoto/atmanalrun.sh index 5e7e712e4e..657b5a2207 100755 --- a/jobs/rocoto/atmanalrun.sh +++ b/jobs/rocoto/atmanalrun.sh @@ -9,7 +9,7 @@ status=$? ############################################################### # set JJOB variables export configs="base atmanal atmanalrun" -export EXSCRIPT=${UFSDARUNSH:-$SCRgfs/exufsda_global_atmos_analysis_run.sh +export EXSCRIPT=${UFSDARUNSH:-$SCRgfs/exufsda_global_atmos_analysis_run.sh} # Execute the JJOB $HOMEgfs/jobs/JUFSDA_GLOBAL_ATMOS_ANALYSIS status=$? From aeb1765e973247435caf344f1ead06aff5b6e212 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Wed, 29 Dec 2021 15:13:04 -0600 Subject: [PATCH 11/45] reduce wallclock for atmanalpost to 1 hour --- parm/config/config.resources | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 65f1d99033..1a95501a09 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -153,7 +153,7 @@ elif [ $step = "atmanalprep" -o $step = "atmanalrun" ]; then elif [ $step = "atmanalpost" ]; then - export wtime_atmanalpost="10:00:00" + export wtime_atmanalpost="01:00:00" export npe_atmanalpost=$npe_node_max export nth_atmanalpost=1 export npe_node_atmanalpost=$(echo "$npe_node_max / $nth_atmanalpost" | bc) From da78b18191fac2533b7b07bc0a2bfadad4d89182 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Mon, 3 Jan 2022 08:25:16 -0600 Subject: [PATCH 12/45] change r2d2 vars --- parm/config/config.atmanal | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/config.atmanal b/parm/config/config.atmanal index 5e6d5ac3e6..c3da6b0690 100755 --- a/parm/config/config.atmanal +++ b/parm/config/config.atmanal @@ -11,8 +11,8 @@ export GEOM_ANL=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/templates/geomanl.yaml export BERROR_YAML=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/templates/ufsda_global_atm_3dvar_b.yaml export ATMANALPREPYAML=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/templates/stage.yaml export ATMANALYAML=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/templates/ufsda_global_atm_3dvar.yaml -export R2D2_OBS_DB='ufo_eval_ref' -export R2D2_OBS_DUMP='gdas_f006_nofov_nonsst_noxl' +export R2D2_OBS_DB='ufsda_test' +export R2D2_OBS_DUMP='oper' export R2D2_OBS_SRC='ncdiag' export R2D2_BC_SRC='gsi' export R2D2_BC_DUMP='oper' From 6078e7347318ef73bd242c0e243df602f3fa54ef Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Mon, 3 Jan 2022 08:27:23 -0600 Subject: [PATCH 13/45] make r2d2 exp oper_gdas --- parm/config/config.atmanal | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/config.atmanal b/parm/config/config.atmanal index c3da6b0690..80624bdfab 100755 --- a/parm/config/config.atmanal +++ b/parm/config/config.atmanal @@ -12,10 +12,10 @@ export BERROR_YAML=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/templates/ufsda_global_at export ATMANALPREPYAML=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/templates/stage.yaml export ATMANALYAML=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/templates/ufsda_global_atm_3dvar.yaml export R2D2_OBS_DB='ufsda_test' -export R2D2_OBS_DUMP='oper' +export R2D2_OBS_DUMP='oper_gdas' export R2D2_OBS_SRC='ncdiag' export R2D2_BC_SRC='gsi' -export R2D2_BC_DUMP='oper' +export R2D2_BC_DUMP='oper_gdas' export R2D2_ARCH_DB='local' export R2D2_CONFIG=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/config/r2d2_config.yaml export FV3JEDI_STAGE_YAML=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/config/fv3jedi_stage.yaml From bb861013c67321bc706c5bbecef3e632f0188119 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Mon, 3 Jan 2022 12:12:50 -0600 Subject: [PATCH 14/45] Fix for orion env file and add link to j-job --- env/ORION.env | 2 +- sorc/link_workflow.sh | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/env/ORION.env b/env/ORION.env index 6cc21279f3..99b549c592 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -51,7 +51,7 @@ elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $ste export wavempexec=${launcher} export wave_mpmd=${mpmd} -elif [ $step = "atmanal" ]; then +elif [ $step = "atmanalrun" ]; then export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 3aa4d02fd5..c7b1777c24 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -187,6 +187,13 @@ cd ${pwd}/../ush ||exit 8 $LINK ../sorc/gsi.fd/ush/getncdimlen . +#------------------------------ +#--add UFS DA/JEDI files +#------------------------------ +cd ${pwd}/../jobs ||exit 8 + $LINK ../sorc/ufs_da.fd/UFS-DA/jobs/JUFSDA_GLOBAL_ATMOS_ANALYSIS . + + #------------------------------ #--add DA Monitor file (NOTE: ensure to use correct version) #------------------------------ From 9594e5a6c8f6972a79485a56a33b98707b6b63ce Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Tue, 4 Jan 2022 08:46:25 -0600 Subject: [PATCH 15/45] Change rocoto job variables --- jobs/rocoto/atmanalpost.sh | 2 +- jobs/rocoto/atmanalprep.sh | 2 +- jobs/rocoto/atmanalrun.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jobs/rocoto/atmanalpost.sh b/jobs/rocoto/atmanalpost.sh index 3cce56253e..a902bac3ae 100755 --- a/jobs/rocoto/atmanalpost.sh +++ b/jobs/rocoto/atmanalpost.sh @@ -9,7 +9,7 @@ status=$? ############################################################### # set JJOB variables export configs="base atmanal atmanalpost" -export EXSCRIPT=${UFSDAPOSTPY:-$SCRgfs/exufsda_global_atmos_analysis_post.py} +export EXSCRIPT=${UFSDAPOSTPY:-$HOMEgfs/sorc/ufs_da.fd/UFS-DA/scripts/exufsda_global_atmos_analysis_post.py} # Execute the JJOB $HOMEgfs/jobs/JUFSDA_GLOBAL_ATMOS_ANALYSIS status=$? diff --git a/jobs/rocoto/atmanalprep.sh b/jobs/rocoto/atmanalprep.sh index 3facbedee5..7393a3b1fe 100755 --- a/jobs/rocoto/atmanalprep.sh +++ b/jobs/rocoto/atmanalprep.sh @@ -9,7 +9,7 @@ status=$? ############################################################### # set JJOB variables export configs="base atmanal atmanalprep" -export EXSCRIPT=${UFSDAPREPPY:-$SCRgfs/exufsda_global_atmos_analysis_prep.py} +export EXSCRIPT=${UFSDAPREPPY:-$HOMEgfs/sorc/ufs_da.fd/UFS-DA/scripts/exufsda_global_atmos_analysis_prep.py} # Execute the JJOB $HOMEgfs/jobs/JUFSDA_GLOBAL_ATMOS_ANALYSIS status=$? diff --git a/jobs/rocoto/atmanalrun.sh b/jobs/rocoto/atmanalrun.sh index 657b5a2207..53eb802de6 100755 --- a/jobs/rocoto/atmanalrun.sh +++ b/jobs/rocoto/atmanalrun.sh @@ -9,7 +9,7 @@ status=$? ############################################################### # set JJOB variables export configs="base atmanal atmanalrun" -export EXSCRIPT=${UFSDARUNSH:-$SCRgfs/exufsda_global_atmos_analysis_run.sh} +export EXSCRIPT=${UFSDARUNPY:-$HOMEgfs/sorc/ufs_da.fd/UFS-DA/scripts/exufsda_global_atmos_analysis_run.sh} # Execute the JJOB $HOMEgfs/jobs/JUFSDA_GLOBAL_ATMOS_ANALYSIS status=$? From 7836e1322b0a5b32991bcb67d80151248152f7e4 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Tue, 4 Jan 2022 10:04:27 -0600 Subject: [PATCH 16/45] Typo in config.atmanal --- parm/config/config.atmanal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/config.atmanal b/parm/config/config.atmanal index 80624bdfab..9fabb2ab35 100755 --- a/parm/config/config.atmanal +++ b/parm/config/config.atmanal @@ -3,7 +3,7 @@ ########## config.atmanal ########## # configuration common to all atm analysis tasks -echo "BEGIN: config.atmatmanal" +echo "BEGIN: config.atmanal" export OBS_YAML=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/config/ufsda_global_atm_obs.yaml export GEOM_BKG=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/templates/geomges.yaml From 94acdde72f01aa22835cee867c8c6df124a2299b Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Tue, 4 Jan 2022 10:27:43 -0600 Subject: [PATCH 17/45] config.resources bug --- parm/config/config.resources | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 1a95501a09..ea62d57880 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -126,32 +126,24 @@ elif [ $step = "waveawipsgridded" ]; then export npe_node_waveawipsgridded=$(echo "$npe_node_max / $nth_waveawipsgridded" | bc) export NTASKS=${npe_waveawipsgridded} -elif [ $step = "atmanalprep" -o $step = "atmanalrun" ]; then +elif [ $step = "atmanalprep" -o $step = "atmanalrun" -o $step = "atmanalpost" ]; then # make below case dependent later export layout_x=5 export layout_y=4 - if [ $step = "atmanalprep" ]; then - - export wtime_atmanalprep="00:10:00" - export npe_atmanalprep=1 - export nth_atmanalprep=1 - export npe_node_atmanalprep=$(echo "$npe_node_max / $nth_atmanalprep" | bc) - export memory_atmanalprep="3072M" - - elif [ $step = "atmanalrun" ]; then - - export wtime_atmanalrun="01:00:00" - export npe_atmanalrun=$(echo "$layout_x * $layout_y * 6" | bc) - export npe_atmanalrun_gfs=$(echo "$layout_x * $layout_y * 6" | bc) - export nth_atmanalrun=1 - export nth_atmanalrun_gfs=$nth_atmanalrun - export npe_node_atmanalrun=$(echo "$npe_node_max / $nth_atmanalrun" | bc) - - fi - -elif [ $step = "atmanalpost" ]; then + export wtime_atmanalprep="00:10:00" + export npe_atmanalprep=1 + export nth_atmanalprep=1 + export npe_node_atmanalprep=$(echo "$npe_node_max / $nth_atmanalprep" | bc) + export memory_atmanalprep="3072M" + + export wtime_atmanalrun="01:00:00" + export npe_atmanalrun=$(echo "$layout_x * $layout_y * 6" | bc) + export npe_atmanalrun_gfs=$(echo "$layout_x * $layout_y * 6" | bc) + export nth_atmanalrun=1 + export nth_atmanalrun_gfs=$nth_atmanalrun + export npe_node_atmanalrun=$(echo "$npe_node_max / $nth_atmanalrun" | bc) export wtime_atmanalpost="01:00:00" export npe_atmanalpost=$npe_node_max From cc62d143c0ba4c95d549d564195af8142de03647 Mon Sep 17 00:00:00 2001 From: "russ.treadon" Date: Tue, 4 Jan 2022 21:14:18 +0000 Subject: [PATCH 18/45] Add link to ufsda directory --- sorc/link_workflow.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index c7b1777c24..39da6e8db9 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -192,7 +192,8 @@ cd ${pwd}/../ush ||exit 8 #------------------------------ cd ${pwd}/../jobs ||exit 8 $LINK ../sorc/ufs_da.fd/UFS-DA/jobs/JUFSDA_GLOBAL_ATMOS_ANALYSIS . - +cd ${pwd}/../ush ||exit 8 + $LINK ../sorc/ufs_da.fd/UFS-DA/ush/ufsda . #------------------------------ #--add DA Monitor file (NOTE: ensure to use correct version) From bf0221631be72f88cc91d2b76f79d867c3797de6 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Mon, 7 Feb 2022 23:53:02 +0000 Subject: [PATCH 19/45] replace atmanal with atmanalrun in ORION.env --- env/ORION.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/env/ORION.env b/env/ORION.env index 99b549c592..c31646130a 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -4,7 +4,7 @@ if [ $# -ne 1 ]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanal" + echo "atmanalrun" echo "anal fcst post vrfy metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" From 134fbe90f518f49862a03fbc39710de46b5dc282 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 15 Feb 2022 21:04:38 +0000 Subject: [PATCH 20/45] Issue #521: extend rocoto workflow xml generator to include UFS DA --- parm/config/config.base.emc.dyn | 3 ++ parm/config/config.base.nco.static | 3 ++ ush/rocoto/setup_workflow.py | 65 ++++++++++++++++++++---------- ush/rocoto/workflow_utils.py | 2 +- 4 files changed, 51 insertions(+), 22 deletions(-) diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn index 7647fe40e0..075f33f093 100755 --- a/parm/config/config.base.emc.dyn +++ b/parm/config/config.base.emc.dyn @@ -300,6 +300,9 @@ export gldas_cyc=00 export imp_physics=11 # Shared parameters +# DA engine +export DO_UFSDA="NO" + # Hybrid related export DOHYBVAR="YES" export NMEM_ENKF=@NMEM_ENKF@ diff --git a/parm/config/config.base.nco.static b/parm/config/config.base.nco.static index 7c47c27ad4..57b6c7ff79 100755 --- a/parm/config/config.base.nco.static +++ b/parm/config/config.base.nco.static @@ -215,6 +215,9 @@ export WAVE_CDUMP="both" export imp_physics=11 # Shared parameters +# DA engine +export DO_UFSDA="NO" + # Hybrid related export DOHYBVAR="YES" export NMEM_ENKF="80" diff --git a/ush/rocoto/setup_workflow.py b/ush/rocoto/setup_workflow.py index 43ee7ab8ca..86c279cb9b 100755 --- a/ush/rocoto/setup_workflow.py +++ b/ush/rocoto/setup_workflow.py @@ -42,7 +42,7 @@ def main(): print(f'input arg: --expdir = {repr(args.expdir)}') sys.exit(1) - gfs_steps = ['prep', 'anal', 'analdiag', 'analcalc', 'gldas', 'fcst', 'postsnd', 'post', 'vrfy', 'arch'] + gfs_steps = ['analcalc', 'gldas', 'fcst', 'postsnd', 'post', 'vrfy', 'arch'] gfs_steps_gempak = ['gempak'] gfs_steps_awips = ['awips'] gfs_steps_wafs = ['wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25'] @@ -55,8 +55,11 @@ def main(): # From gfsv16b latest # gfs_steps = ['prep', 'anal', 'gldas', 'fcst', 'postsnd', 'post', 'awips', 'gempak', 'vrfy', 'metp', 'arch'] hyb_steps = ['eobs', 'ediag', 'eomg', 'eupd', 'ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc'] + gsida_steps = ['prep', 'anal', 'analdiag'] + ufsda_steps = ['atmanalprep', 'atmanalrun', 'atmanalpost'] - steps = gfs_steps + hyb_steps if _base.get('DOHYBVAR', 'NO') == 'YES' else gfs_steps + steps = gfs_steps + ufsda_steps if _base.get('DO_UFSDA', 'NO') == 'YES' else gfs_steps + gsida_steps + steps = steps + hyb_steps if _base.get('DOHYBVAR', 'NO') == 'YES' else steps steps = steps + metp_steps if _base.get('DO_METP', 'NO') == 'YES' else steps steps = steps + gfs_steps_gempak if _base.get('DO_GEMPAK', 'NO') == 'YES' else steps steps = steps + gfs_steps_awips if _base.get('DO_AWIPS', 'NO') == 'YES' else steps @@ -241,13 +244,23 @@ def get_gdasgfs_resources(dict_configs, cdump='gdas'): do_gldas = base.get('DO_GLDAS', 'NO').upper() do_wave = base.get('DO_WAVE', 'NO').upper() do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper() + do_ufsda = base.get('DO_UFSDA', 'NO').upper() reservation = base.get('RESERVATION', 'NONE').upper() #tasks = ['prep', 'anal', 'fcst', 'post', 'vrfy', 'arch'] - tasks = ['prep', 'anal', 'analcalc'] + if do_ufsda in ['Y', 'YES']: + task_prep = ['atmanalprep'] + task_anal = ['atmanalrun'] + task_diag = ['atmanalpost'] + else: + task_prep = ['prep'] + task_anal = ['anal'] + task_diag = ['analdiag'] + tasks = task_prep + task_anal + tasks += ['analcalc'] if cdump in ['gdas']: - tasks += ['analdiag'] + tasks += task_diag if cdump in ['gdas'] and do_gldas in ['Y', 'YES']: tasks += ['gldas'] if cdump in ['gdas'] and do_wave in ['Y', 'YES'] and do_wave_cdump in ['GDAS', 'BOTH']: @@ -410,6 +423,7 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): do_metp = base.get('DO_METP', 'NO').upper() do_gldas = base.get('DO_GLDAS', 'NO').upper() do_wave = base.get('DO_WAVE', 'NO').upper() + do_ufsda = base.get('DO_UFSDA', 'NO').upper() if do_wave in ['YES']: do_wave_bnd = dict_configs['wavepostsbs'].get('DOBNDPNT_WAVE', "YES").upper() do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper() @@ -418,6 +432,15 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): dict_tasks = OrderedDict() + if do_ufsda in ['Y', 'YES']: + task_prep = 'atmanalprep' + task_anal = 'atmanalrun' + task_diag = 'atmanalpost' + else: + task_prep = 'prep' + task_anal = 'anal' + task_diag = 'analdiag' + # prep deps = [] dep_dict = {'type': 'metatask', 'name': f'{"gdas"}post', 'offset': '-06:00:00'} @@ -434,14 +457,14 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): if gfs_enkf and cdump in ['gfs']: if gfs_cyc == 4: - task = wfu.create_wf_task('prep', cdump=cdump, envar=envars, dependency=dependencies) + task = wfu.create_wf_task(task=task_prep, cdump=cdump, envar=envars, dependency=dependencies) else: - task = wfu.create_wf_task('prep', cdump=cdump, envar=envars, dependency=dependencies, cycledef='gdas') + task = wfu.create_wf_task(task=task_prep, cdump=cdump, envar=envars, dependency=dependencies, cycledef='gdas') else: - task = wfu.create_wf_task('prep', cdump=cdump, envar=envars, dependency=dependencies) + task = wfu.create_wf_task(task=task_prep, cdump=cdump, envar=envars, dependency=dependencies) - dict_tasks[f'{cdump}prep'] = task + dict_tasks[f'{cdump}{task_prep}'] = task # wave tasks in gdas or gfs or both if do_wave_cdump in ['BOTH']: @@ -454,7 +477,7 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): # waveinit if do_wave in ['Y', 'YES'] and cdump in cdumps: deps = [] - dep_dict = {'type': 'task', 'name': '{cdump}prep'} + dep_dict = {'type': 'task', 'name': '{cdump}{task_prep}'} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) @@ -473,7 +496,7 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): # anal deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}prep'} + dep_dict = {'type': 'task', 'name': f'{cdump}{task_prep}'} deps.append(rocoto.add_dependency(dep_dict)) if dohybvar in ['y', 'Y', 'yes', 'YES']: dep_dict = {'type': 'metatask', 'name': f'{"gdas"}epmn', 'offset': '-06:00:00'} @@ -481,16 +504,16 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) else: dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('anal', cdump=cdump, envar=envars, dependency=dependencies) + task = wfu.create_wf_task(task=task_anal, cdump=cdump, envar=envars, dependency=dependencies) - dict_tasks[f'{cdump}anal'] = task + dict_tasks[f'{cdump}{task_anal}'] = task # analcalc deps1 = [] data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loginc.txt' dep_dict = {'type': 'data', 'data': data} deps1.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}anal'} + dep_dict = {'type': 'task', 'name': f'{cdump}{task_anal}'} deps.append(rocoto.add_dependency(dep_dict)) if dohybvar in ['y', 'Y', 'yes', 'YES'] and cdump == 'gdas': dep_dict = {'type': 'task', 'name': f'{"gdas"}echgres', 'offset': '-06:00:00'} @@ -508,7 +531,7 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loginc.txt' dep_dict = {'type': 'data', 'data': data} deps1.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}anal'} + dep_dict = {'type': 'task', 'name': f'{cdump}{task_anal}'} deps1.append(rocoto.add_dependency(dep_dict)) dependencies1 = rocoto.create_dependency(dep_condition='or', dep=deps1) @@ -518,9 +541,9 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): deps2.append(rocoto.add_dependency(dep_dict)) dependencies2 = rocoto.create_dependency(dep_condition='and', dep=deps2) - task = wfu.create_wf_task('analdiag', cdump=cdump, envar=envars, dependency=dependencies2) + task = wfu.create_wf_task(task=task_diag, cdump=cdump, envar=envars, dependency=dependencies2) - dict_tasks[f'{cdump}analdiag'] = task + dict_tasks[f'{cdump}{task_diag}'] = task # gldas if cdump in ['gdas'] and do_gldas in ['Y', 'YES']: @@ -528,7 +551,7 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loginc.txt' dep_dict = {'type': 'data', 'data': data} deps1.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}anal'} + dep_dict = {'type': 'task', 'name': f'{cdump}{task_anal}'} deps1.append(rocoto.add_dependency(dep_dict)) dependencies1 = rocoto.create_dependency(dep_condition='or', dep=deps1) @@ -556,7 +579,7 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): dep_dict = {'type': 'task', 'name': f'{cdump}analcalc'} deps1.append(rocoto.add_dependency(dep_dict)) elif cdump in ['gfs']: - dep_dict = {'type': 'task', 'name': f'{cdump}anal'} + dep_dict = {'type': 'task', 'name': f'{cdump}{task_anal}'} deps1.append(rocoto.add_dependency(dep_dict)) dependencies1 = rocoto.create_dependency(dep_condition='or', dep=deps1) @@ -987,7 +1010,7 @@ def get_hyb_tasks(dict_configs, cycledef='enkf'): # eobs deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}prep'} + dep_dict = {'type': 'task', 'name': f'{cdump}{task_prep}'} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'metatask', 'name': f'{"gdas"}epmn', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) @@ -1402,7 +1425,7 @@ def create_xml(dict_configs): if gfs_cyc != 0: xmlfile.append(dict_to_strings(dict_gfs_resources)) elif gfs_cyc == 0 and dohybvar in ['Y', 'YES'] and eupd_cyc in ['BOTH', 'GFS']: - xmlfile.append(dict_gfs_resources['gfsprep']) + xmlfile.append(dict_gfs_resources['gfs{task_prep}']) xmlfile.append(workflow_header) @@ -1414,7 +1437,7 @@ def create_xml(dict_configs): if gfs_cyc != 0: xmlfile.append(dict_to_strings(dict_gfs_tasks)) elif gfs_cyc == 0 and dohybvar in ['Y', 'YES'] and eupd_cyc in ['BOTH', 'GFS']: - xmlfile.append(dict_gfs_tasks['gfsprep']) + xmlfile.append(dict_gfs_tasks['gfs{task_prep}']) xmlfile.append('\n') xmlfile.append(workflow_footer) diff --git a/ush/rocoto/workflow_utils.py b/ush/rocoto/workflow_utils.py index 6ffd12888e..46f9fab3b0 100755 --- a/ush/rocoto/workflow_utils.py +++ b/ush/rocoto/workflow_utils.py @@ -325,7 +325,7 @@ def get_scheduler(machine): except KeyError: raise UnknownMachineError(f'Unknown machine: {machine}, ABORT!') -def create_wf_task(task, cdump='gdas', cycledef=None, envar=None, dependency=None, \ +def create_wf_task(task=None, cdump='gdas', cycledef=None, envar=None, dependency=None, \ metatask=None, varname=None, varval=None, vardict=None, \ final=False): From dd31c9250de8c994c44e1ece7df8e98d15993d84 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 18 Feb 2022 00:20:44 +0000 Subject: [PATCH 21/45] Issue #521 add missing task_prep to setup_workflow.py --- ush/rocoto/setup_workflow.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ush/rocoto/setup_workflow.py b/ush/rocoto/setup_workflow.py index 86c279cb9b..32436e6b3e 100755 --- a/ush/rocoto/setup_workflow.py +++ b/ush/rocoto/setup_workflow.py @@ -965,6 +965,7 @@ def get_hyb_tasks(dict_configs, cycledef='enkf'): nens = base['NMEM_ENKF'] lobsdiag_forenkf = base.get('lobsdiag_forenkf', '.false.').upper() eupd_cyc = base.get('EUPD_CYC', 'gdas').upper() + do_ufsda = base.get('DO_UFSDA', 'NO').upper() eobs = dict_configs['eobs'] nens_eomg = eobs['NMEM_EOMGGRP'] @@ -1003,6 +1004,11 @@ def get_hyb_tasks(dict_configs, cycledef='enkf'): elif eupd_cyc in ['GDAS']: cdumps = ['gdas'] + if do_ufsda in ['Y', 'YES']: + task_prep = ['atmanalprep'] + else: + task_prep = ['prep'] + for cdump in cdumps: envar_cdump = rocoto.create_envar(name='CDUMP', value=f'{cdump}') @@ -1339,6 +1345,12 @@ def create_xml(dict_configs): dohybvar = base.get('DOHYBVAR', 'NO').upper() gfs_cyc = base.get('gfs_cyc', 0) eupd_cyc = base.get('EUPD_CYC', 'gdas').upper() + do_ufsda = base.get('DO_UFSDA', 'NO').upper() + + if do_ufsda in ['Y', 'YES']: + task_prep = ['atmanalprep'] + else: + task_prep = ['prep'] # Start collecting workflow pieces preamble = get_preamble() From 52e4d1220108b00a8a45e7d0212f84fc1659b93a Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Fri, 18 Mar 2022 15:38:57 -0500 Subject: [PATCH 22/45] Change UFS-DA to GDASApp --- sorc/{build_ufs_da.sh => build_gdas.sh} | 4 ++-- sorc/checkout.sh | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) rename sorc/{build_ufs_da.sh => build_gdas.sh} (91%) diff --git a/sorc/build_ufs_da.sh b/sorc/build_gdas.sh similarity index 91% rename from sorc/build_ufs_da.sh rename to sorc/build_gdas.sh index fff1fd105b..8e2c6ffcad 100755 --- a/sorc/build_ufs_da.sh +++ b/sorc/build_gdas.sh @@ -19,8 +19,8 @@ if [ ! -d "../exec" ]; then mkdir ../exec fi -cd ufs_da.fd/UFS-DA/ush -./build_UFSDA.sh +cd gdas.fd/ush +./build_GDASApp.sh exit diff --git a/sorc/checkout.sh b/sorc/checkout.sh index 2f105abcdb..135151c2ec 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -50,15 +50,15 @@ else echo 'Skip. Directory gsi.fd already exists.' fi -echo ufs_da checkout ... -if [[ ! -d ufs_da.fd ]] ; then - rm -f ${topdir}/checkout-ufs_da.log - git clone --recursive https://github.com/NOAA-EMC/JEDI-T2O.git ufs_da.fd >> ${logdir}/checkout-ufs_da.log 2>&1 - cd ufs_da.fd - git checkout develop +echo gdas checkout ... +if [[ ! -d gdas.fd ]] ; then + rm -f ${topdir}/checkout-gdas.log + git clone --recursive https://github.com/NOAA-EMC/GDASApp.git gdas.fd >> ${logdir}/checkout-gdas.log 2>&1 + cd gdas.fd + git checkout feature/build cd ${topdir} else - echo 'Skip. Directory ufs_da.fd already exists.' + echo 'Skip. Directory gdas.fd already exists.' fi echo gldas checkout ... From a720c3f275b88aed2489abd86e7b99a8330cb4a3 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Thu, 14 Apr 2022 11:09:56 -0500 Subject: [PATCH 23/45] Updated build for GDAS --- sorc/build_all.sh | 12 ++++++------ sorc/build_gdas.sh | 13 ++++++++++--- sorc/checkout.sh | 12 ++++++------ sorc/link_workflow.sh | 6 +++--- 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/sorc/build_all.sh b/sorc/build_all.sh index 3bc18578f2..a470528e77 100755 --- a/sorc/build_all.sh +++ b/sorc/build_all.sh @@ -114,15 +114,15 @@ fi } #------------------------------------ -# build ufs_da +# build gdas #------------------------------------ -$Build_ufs_da && { -echo " .... Building ufs_da .... " -./build_ufs_da.sh > $logs_dir/build_ufs_da.log 2>&1 +$Build_gdas && { +echo " .... Building GDASApp .... " +./build_gdas.sh > $logs_dir/build_gdas.log 2>&1 rc=$? if [[ $rc -ne 0 ]] ; then - echo "Fatal error in building ufs_da." - echo "The log file is in $logs_dir/build_ufs_da.log" + echo "Fatal error in building GDAS." + echo "The log file is in $logs_dir/build_gdas.log" fi ((err+=$rc)) } diff --git a/sorc/build_gdas.sh b/sorc/build_gdas.sh index 8e2c6ffcad..c32df5ebaf 100755 --- a/sorc/build_gdas.sh +++ b/sorc/build_gdas.sh @@ -12,15 +12,22 @@ else fi export BUILD_TARGET=$target -[[ "$target" == wcoss_cray ]] && BUILD_TARGET=cray + +# use more build jobs if on NOAA HPC +build_jobs=4 +case "${target}" in + hera|orion) + build_jobs=10 + ;; +esac # Check final exec folder exists if [ ! -d "../exec" ]; then mkdir ../exec fi -cd gdas.fd/ush -./build_GDASApp.sh +cd gdas.cd +BUILD_JOBS=$build_jobs ./build.sh -t $BUILD_TARGET exit diff --git a/sorc/checkout.sh b/sorc/checkout.sh index 135151c2ec..89f08214fd 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -50,15 +50,15 @@ else echo 'Skip. Directory gsi.fd already exists.' fi -echo gdas checkout ... -if [[ ! -d gdas.fd ]] ; then +echo GDASApp checkout ... +if [[ ! -d gdas.cd ]] ; then rm -f ${topdir}/checkout-gdas.log - git clone --recursive https://github.com/NOAA-EMC/GDASApp.git gdas.fd >> ${logdir}/checkout-gdas.log 2>&1 - cd gdas.fd - git checkout feature/build + git clone --recursive https://github.com/NOAA-EMC/GDASApp.git gdas.cd >> ${logdir}/checkout-gdas.log 2>&1 + cd gdas.cd + git checkout develop cd ${topdir} else - echo 'Skip. Directory gdas.fd already exists.' + echo 'Skip. Directory gdas.cd already exists.' fi echo gldas checkout ... diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index fdae4db612..c5e79bc227 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -184,12 +184,12 @@ cd ${pwd}/../ush ||exit 8 #------------------------------ -#--add UFS DA/JEDI files +#--add GDASApp files #------------------------------ cd ${pwd}/../jobs ||exit 8 - $LINK ../sorc/ufs_da.fd/UFS-DA/jobs/JUFSDA_GLOBAL_ATMOS_ANALYSIS . + $LINK ../sorc/gdas.cd/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS . cd ${pwd}/../ush ||exit 8 - $LINK ../sorc/ufs_da.fd/UFS-DA/ush/ufsda . + $LINK ../sorc/gdas.cd/ush/ufsda . #------------------------------ #--add DA Monitor file (NOTE: ensure to use correct version) From 078b96dd77358b6b3d8194e6f787bea5512263a3 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Thu, 14 Apr 2022 12:36:51 -0500 Subject: [PATCH 24/45] Changes to rocoto scripts --- jobs/rocoto/atmanalpost.sh | 4 ++-- jobs/rocoto/atmanalprep.sh | 4 ++-- jobs/rocoto/atmanalrun.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/jobs/rocoto/atmanalpost.sh b/jobs/rocoto/atmanalpost.sh index a902bac3ae..3a1691dcf0 100755 --- a/jobs/rocoto/atmanalpost.sh +++ b/jobs/rocoto/atmanalpost.sh @@ -9,8 +9,8 @@ status=$? ############################################################### # set JJOB variables export configs="base atmanal atmanalpost" -export EXSCRIPT=${UFSDAPOSTPY:-$HOMEgfs/sorc/ufs_da.fd/UFS-DA/scripts/exufsda_global_atmos_analysis_post.py} +export EXSCRIPT=${GDASPOSTPY:-$HOMEgfs/sorc/gdas.cd/scripts/exgdas_global_atmos_analysis_post.py} # Execute the JJOB -$HOMEgfs/jobs/JUFSDA_GLOBAL_ATMOS_ANALYSIS +$HOMEgfs/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS status=$? exit $status diff --git a/jobs/rocoto/atmanalprep.sh b/jobs/rocoto/atmanalprep.sh index 7393a3b1fe..6dc19fc260 100755 --- a/jobs/rocoto/atmanalprep.sh +++ b/jobs/rocoto/atmanalprep.sh @@ -9,8 +9,8 @@ status=$? ############################################################### # set JJOB variables export configs="base atmanal atmanalprep" -export EXSCRIPT=${UFSDAPREPPY:-$HOMEgfs/sorc/ufs_da.fd/UFS-DA/scripts/exufsda_global_atmos_analysis_prep.py} +export EXSCRIPT=${GDASPREPPY:-$HOMEgfs/sorc/gdas.cd/scripts/exgdas_global_atmos_analysis_prep.py} # Execute the JJOB -$HOMEgfs/jobs/JUFSDA_GLOBAL_ATMOS_ANALYSIS +$HOMEgfs/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS status=$? exit $status diff --git a/jobs/rocoto/atmanalrun.sh b/jobs/rocoto/atmanalrun.sh index 53eb802de6..070fa3468a 100755 --- a/jobs/rocoto/atmanalrun.sh +++ b/jobs/rocoto/atmanalrun.sh @@ -9,8 +9,8 @@ status=$? ############################################################### # set JJOB variables export configs="base atmanal atmanalrun" -export EXSCRIPT=${UFSDARUNPY:-$HOMEgfs/sorc/ufs_da.fd/UFS-DA/scripts/exufsda_global_atmos_analysis_run.sh} +export EXSCRIPT=${GDASRUNSH:-$HOMEgfs/sorc/gdas.cd/scripts/exgdas_global_atmos_analysis_run.sh} # Execute the JJOB -$HOMEgfs/jobs/JUFSDA_GLOBAL_ATMOS_ANALYSIS +$HOMEgfs/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS status=$? exit $status From e34e82f669dd1e8a04efa9800224605d6c16ef4c Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Thu, 14 Apr 2022 14:55:34 -0500 Subject: [PATCH 25/45] some changes to config.atmanal --- parm/config/config.atmanal | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/parm/config/config.atmanal b/parm/config/config.atmanal index 9fabb2ab35..1a4c4f9c26 100755 --- a/parm/config/config.atmanal +++ b/parm/config/config.atmanal @@ -5,20 +5,14 @@ echo "BEGIN: config.atmanal" -export OBS_YAML=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/config/ufsda_global_atm_obs.yaml -export GEOM_BKG=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/templates/geomges.yaml -export GEOM_ANL=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/templates/geomanl.yaml -export BERROR_YAML=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/templates/ufsda_global_atm_3dvar_b.yaml -export ATMANALPREPYAML=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/templates/stage.yaml -export ATMANALYAML=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/templates/ufsda_global_atm_3dvar.yaml +export OBS_YAML_DIR=$HOMEgfs/sorc/gdas.cd/parm/atm/obs/config/ +export OBS_LIST=$HOMEgfs/sorc/gdas.cd/parm/atm/obs/lists/gdas_prototype.yaml export R2D2_OBS_DB='ufsda_test' export R2D2_OBS_DUMP='oper_gdas' export R2D2_OBS_SRC='ncdiag' export R2D2_BC_SRC='gsi' export R2D2_BC_DUMP='oper_gdas' export R2D2_ARCH_DB='local' -export R2D2_CONFIG=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/config/r2d2_config.yaml -export FV3JEDI_STAGE_YAML=$HOMEgfs/sorc/ufs_da.fd/UFS-DA/parm/config/fv3jedi_stage.yaml export io_layout_x=1 export io_layout_y=1 From 431028da562abc38d347ccd4582d75861e9a980f Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Mon, 18 Apr 2022 14:57:20 -0500 Subject: [PATCH 26/45] Changes to link_workflow and temp change to feature branch for gdas --- sorc/checkout.sh | 2 +- sorc/link_workflow.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sorc/checkout.sh b/sorc/checkout.sh index 89f08214fd..c63b5d266b 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -55,7 +55,7 @@ if [[ ! -d gdas.cd ]] ; then rm -f ${topdir}/checkout-gdas.log git clone --recursive https://github.com/NOAA-EMC/GDASApp.git gdas.cd >> ${logdir}/checkout-gdas.log 2>&1 cd gdas.cd - git checkout develop + git checkout feature/cycling cd ${topdir} else echo 'Skip. Directory gdas.cd already exists.' diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index c5e79bc227..5ceeaf69b5 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -287,12 +287,12 @@ for gsiexe in calc_analysis.x calc_increment_ens_ncio.x calc_increment_ens.x \ $LINK ../sorc/gsi.fd/exec/$gsiexe . done -for ufsdaexe in fv3jedi_addincrement.x fv3jedi_diffstates.x fv3jedi_ensvariance.x fv3jedi_hofx.x \ +for gdasexe in fv3jedi_addincrement.x fv3jedi_diffstates.x fv3jedi_ensvariance.x fv3jedi_hofx.x \ fv3jedi_var.x fv3jedi_convertincrement.x fv3jedi_dirac.x fv3jedi_error_covariance_training.x \ fv3jedi_letkf.x fv3jedi_convertstate.x fv3jedi_eda.x fv3jedi_forecast.x fv3jedi_plot_field.x \ fv3jedi_data_checker.py fv3jedi_enshofx.x fv3jedi_hofx_nomodel.x fv3jedi_testdata_downloader.py; do - [[ -s $ufsdaexe ]] && rm -f $ufsdaexe - $LINK ../sorc/ufs_da.fd/UFS-DA/bin/$ufsdaexe . + [[ -s $gdasexe ]] && rm -f $gdasexe + $LINK ../sorc/gdas.cd/build/bin/$gdasexe . done for gldasexe in gdas2gldas gldas2gdas gldas_forcing gldas_model gldas_post gldas_rst; do From cdf86c069851f47168b6df2df34092def2347a08 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Mon, 18 Apr 2022 15:00:34 -0500 Subject: [PATCH 27/45] Changes to config files --- parm/config/config.atmanal | 2 ++ parm/config/config.atmanalrun | 1 + parm/config/config.resources | 4 ++-- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/parm/config/config.atmanal b/parm/config/config.atmanal index 1a4c4f9c26..1b7b494871 100755 --- a/parm/config/config.atmanal +++ b/parm/config/config.atmanal @@ -7,6 +7,8 @@ echo "BEGIN: config.atmanal" export OBS_YAML_DIR=$HOMEgfs/sorc/gdas.cd/parm/atm/obs/config/ export OBS_LIST=$HOMEgfs/sorc/gdas.cd/parm/atm/obs/lists/gdas_prototype.yaml +export ATMVARYAML=$HOMEgfs/sorc/gdas.cd/parm/atm/variational/3dvar_dripcg.yaml +export FV3JEDI_FIX=/work2/noaa/da/cmartin/GDASApp/fix export R2D2_OBS_DB='ufsda_test' export R2D2_OBS_DUMP='oper_gdas' export R2D2_OBS_SRC='ncdiag' diff --git a/parm/config/config.atmanalrun b/parm/config/config.atmanalrun index e47938dea6..a495e95cbe 100755 --- a/parm/config/config.atmanalrun +++ b/parm/config/config.atmanalrun @@ -10,5 +10,6 @@ echo "BEGIN: config.atmanalrun" # Task specific variables export JEDIVAREXE=$HOMEgfs/exec/fv3jedi_var.x +export BERROR_YAML=$HOMEgfs/sorc/gdas.cd/parm/atm/berror/staticb_bump.yaml echo "END: config.atmanalrun" diff --git a/parm/config/config.resources b/parm/config/config.resources index d92e13757f..a74d7c01a8 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -129,8 +129,8 @@ elif [ $step = "waveawipsgridded" ]; then elif [ $step = "atmanalprep" -o $step = "atmanalrun" -o $step = "atmanalpost" ]; then # make below case dependent later - export layout_x=5 - export layout_y=4 + export layout_x=1 + export layout_y=1 export wtime_atmanalprep="00:10:00" export npe_atmanalprep=1 From 9406bca3453d24cb6d892a8eb0572aba9759ec6f Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 26 Apr 2022 14:07:22 +0000 Subject: [PATCH 28/45] Updates to cycle GDASApp * .gitignore - add GDASApp directories and file * env/ORION.env - add sfcanl, add task count to APRUN_ATMANAL * jobs/JGLOBAL_ATMOS_SFCANL - new j-job for surface analysis * jobs/rocoto/postanl.sh - new rocoto job for analysis only post * jobs/rocoto/sfcanl.sh - new rocoto job for surface analysis * parm/config/config.base.emc.dyn - replace DO_UFSDA with DO_JEDIVAR * parm/config/config.base.nco.static - replace DO_UFSDA with DO_JEDIVAR * scripts/exglobal_atmos_sfcanl.sh - new ex script for surface analysis * sorc/checkout.sh - check out head of NOAA-EMC/GSI develop * sorc/link_workflow.sh - remove version numbers from MinMon paths * ush/rocoto/setup_workflow.py - updates for JEDIVAR --- .gitignore | 3 + env/ORION.env | 12 +- jobs/JGLOBAL_ATMOS_SFCANL | 179 +++++++++++++++++ jobs/rocoto/postanl.sh | 50 +++++ jobs/rocoto/sfcanl.sh | 13 ++ parm/config/config.base.emc.dyn | 2 +- parm/config/config.base.nco.static | 2 +- scripts/exglobal_atmos_sfcanl.sh | 302 +++++++++++++++++++++++++++++ sorc/checkout.sh | 1 - sorc/link_workflow.sh | 22 +-- ush/rocoto/setup_workflow.py | 149 ++++++++++---- 11 files changed, 683 insertions(+), 52 deletions(-) create mode 100755 jobs/JGLOBAL_ATMOS_SFCANL create mode 100755 jobs/rocoto/postanl.sh create mode 100755 jobs/rocoto/sfcanl.sh create mode 100755 scripts/exglobal_atmos_sfcanl.sh diff --git a/.gitignore b/.gitignore index e8638c013f..4c297ed4d1 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ sorc/gfs_post.fd sorc/gfs_wafs.fd sorc/gldas.fd sorc/gsi.fd +sorc/gdas.cd sorc/ufs_utils.fd sorc/verif-global.fd @@ -98,6 +99,7 @@ jobs/JGDAS_ENKF_POST jobs/JGDAS_ENKF_SELECT_OBS jobs/JGDAS_ENKF_SFC jobs/JGDAS_ENKF_UPDATE +jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS jobs/JGFS_ATMOS_VMINMON jobs/JGFS_ATMOS_WAFS jobs/JGFS_ATMOS_WAFS_BLENDING @@ -176,6 +178,7 @@ ush/radmon_verf_bcoef.sh ush/radmon_verf_bcor.sh ush/radmon_verf_time.sh ush/trim_rh.sh +ush/ufsda ush/wafs_blending.sh ush/wafs_grib2.regrid.sh ush/wafs_intdsk.sh diff --git a/env/ORION.env b/env/ORION.env index 0246001a5d..86057d7f47 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -5,7 +5,7 @@ if [ $# -ne 1 ]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" echo "atmanalrun" - echo "anal fcst post vrfy metp" + echo "anal sfcanl fcst post vrfy metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" exit 1 @@ -60,7 +60,7 @@ elif [ $step = "atmanalrun" ]; then export NTHREADS_ATMANAL=${nth_atmanalrun:-$nth_max} [[ $NTHREADS_ATMANAL -gt $nth_max ]] && export NTHREADS_ATMANAL=$nth_max - export APRUN_ATMANAL="$launcher" + export APRUN_ATMANAL="$launcher -n $npe_atmanalrun" elif [ $step = "anal" ]; then @@ -91,6 +91,14 @@ elif [ $step = "anal" ]; then export APRUN_GAUSFCANL="$launcher -n $npe_gausfcanl" +elif [ $step = "sfcanl" ]; then + nth_max=$(($npe_node_max / $npe_node_sfcanl)) + + export NTHREADS_CYCLE=${nth_sfcanl:-14} + [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max + npe_sfcanl=${ntiles:-6} + export APRUN_CYCLE="$launcher -n $npe_sfcanl" + elif [ $step = "gldas" ]; then nth_max=$(($npe_node_max / $npe_node_gldas)) diff --git a/jobs/JGLOBAL_ATMOS_SFCANL b/jobs/JGLOBAL_ATMOS_SFCANL new file mode 100755 index 0000000000..c3b2ab26fe --- /dev/null +++ b/jobs/JGLOBAL_ATMOS_SFCANL @@ -0,0 +1,179 @@ +#!/bin/ksh +set -x + +export RUN_ENVIR=${RUN_ENVIR:-"nco"} +export PS4='$SECONDS + ' +date + + +############################# +# Source relevant config files +############################# +export EXPDIR=${EXPDIR:-$HOMEgfs/parm/config} +configs="base sfcanl" +config_path=${EXPDIR:-$NWROOT/gfs.${gfs_ver}/parm/config} +for config in $configs; do + . $config_path/config.$config + status=$? + [[ $status -ne 0 ]] && exit $status +done + + +########################################## +# Source machine runtime environment +########################################## +. $HOMEgfs/env/${machine}.env sfcanl +status=$? +[[ $status -ne 0 ]] && exit $status + + +############################################## +# Obtain unique process id (pid) and make temp directory +############################################## +export pid=${pid:-$$} +export outid=${outid:-"LL$job"} + +export DATA=${DATA:-${DATAROOT}/${jobid:?}} +mkdir -p $DATA +cd $DATA + + +############################################## +# Run setpdy and initialize PDY variables +############################################## +export cycle="t${cyc}z" +setpdy.sh +. ./PDY + + +############################################## +# Determine Job Output Name on System +############################################## +export pgmout="OUTPUT.${pid}" +export pgmerr=errfile + + +############################################## +# Set variables used in the script +############################################## +export CDATE=${CDATE:-${PDY}${cyc}} +export CDUMP=${CDUMP:-${RUN:-"gfs"}} +export COMPONENT=${COMPONENT:-atmos} +if [ $RUN_ENVIR = "nco" ]; then + export ROTDIR=${COMROOT:?}/$NET/$envir +fi +export DO_CALC_ANALYSIS=${DO_CALC_ANALYSIS:-"YES"} + + +############################################## +# Begin JOB SPECIFIC work +############################################## + +GDATE=$($NDATE -$assim_freq $CDATE) +gPDY=$(echo $GDATE | cut -c1-8) +gcyc=$(echo $GDATE | cut -c9-10) +GDUMP=${GDUMP:-"gdas"} + +export OPREFIX="${CDUMP}.t${cyc}z." +export GPREFIX="${GDUMP}.t${gcyc}z." +export APREFIX="${CDUMP}.t${cyc}z." +export GSUFFIX=${GSUFFIX:-$SUFFIX} +export ASUFFIX=${ASUFFIX:-$SUFFIX} + + +if [ $RUN_ENVIR = "nco" -o ${ROTDIR_DUMP:-NO} = "YES" ]; then + export COMIN=${COMIN:-$ROTDIR/$RUN.$PDY/$cyc/$COMPONENT} + export COMOUT=${COMOUT:-$ROTDIR/$RUN.$PDY/$cyc/$COMPONENT} + export COMIN_OBS=${COMIN_OBS:-$ROTDIR/$RUN.$PDY/$cyc/$COMPONENT} + export COMIN_GES_OBS=${COMIN_GES_OBS:-$ROTDIR/$GDUMP.$gPDY/$gcyc/$COMPONENT} +else + export COMOUT="$ROTDIR/$CDUMP.$PDY/$cyc/$COMPONENT" + export COMIN_OBS="$DMPDIR/$CDATE/$CDUMP" + export COMIN_GES_OBS="$DMPDIR/$GDATE/$GDUMP" +fi +mkdir -m 775 -p $COMOUT +# COMIN_GES and COMIN_GES_ENS are used in script +export COMIN_GES="$ROTDIR/$GDUMP.$gPDY/$gcyc/$COMPONENT" +export COMIN_GES_ENS="$ROTDIR/enkfgdas.$gPDY/$gcyc/$COMPONENT" + + +export ATMGES="$COMIN_GES/${GPREFIX}atmf006${GSUFFIX}" +if [ ! -f $ATMGES ]; then + echo "FATAL ERROR: FILE MISSING: ATMGES = $ATMGES" + exit 1 +fi + + +# Get LEVS +if [ ${GSUFFIX} = ".nc" ]; then + export LEVS=$($NCLEN $ATMGES pfull) + status=$? +else + export LEVS=$($NEMSIOGET $ATMGES dimz | awk '{print $2}') + status=$? +fi +[[ $status -ne 0 ]] && exit $status + + +##if [ $DOHYBVAR = "YES" ]; then +## export ATMGES_ENSMEAN="$COMIN_GES_ENS/${GPREFIX}atmf006.ensmean$GSUFFIX" +## if [ ! -f $ATMGES_ENSMEAN ]; then +## echo "FATAL ERROR: FILE MISSING: ATMGES_ENSMEAN = $ATMGES_ENSMEAN" +## exit 2 +## fi +##fi + + +# Link observational data +export PREPQC="${COMOUT}/${OPREFIX}prepbufr" +if [ ! -f $PREPQC ]; then + echo "WARNING: PREPBUFR FILE $PREPQC MISSING" + msg="WARNING : Global PREPBUFR file is missing" + postmsg "$jlogfile" "$msg" +fi +export PREPQCPF="${COMOUT}/${OPREFIX}prepbufr.acft_profiles" +export TCVITL="${COMOUT}/${OPREFIX}syndata.tcvitals.tm00" +[[ $DONST = "YES" ]] && export NSSTBF="${COMOUT}/${OPREFIX}nsstbufr" + + +# Update surface fields with global_cycle +export DOGCYCLE=${DOGCYCLE:-"YES"} + + +############################################################### +# Run relevant script +env +msg="HAS BEGUN on `hostname`" +postmsg "$jlogfile" "$msg" +$LOGSCRIPT + + +${SFCANALSH:-$SCRgfs/exglobal_atmos_sfcanl.sh} +status=$? +[[ $status -ne 0 ]] && exit $status + + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [ -e "$pgmout" ] ; then + cat $pgmout +fi + + +msg="ENDED NORMALLY." +postmsg "$jlogfile" "$msg" + + +########################################## +# Remove the Temporary working directory +########################################## +cd $DATAROOT +[[ $KEEPDATA = "NO" ]] && rm -rf $DATA + +date +exit 0 diff --git a/jobs/rocoto/postanl.sh b/jobs/rocoto/postanl.sh new file mode 100755 index 0000000000..f7c269d976 --- /dev/null +++ b/jobs/rocoto/postanl.sh @@ -0,0 +1,50 @@ +#!/bin/ksh -x + +############################################################### +## NCEP post driver script +## FHRGRP : forecast hour group to post-process (e.g. 0, 1, 2 ...) +## FHRLST : forecast hourlist to be post-process (e.g. anl, f000, f000_f001_f002, ...) +############################################################### + +# Source FV3GFS workflow modules +. $HOMEgfs/ush/load_fv3gfs_modules.sh +status=$? +[[ $status -ne 0 ]] && exit $status + +export COMPONENT=${COMPONENT:-atmos} + +if [ $FHRGRP = 'anl' ]; then + fhrlst="anl" + restart_file=$ROTDIR/${CDUMP}.${PDY}/${cyc}/$COMPONENT/${CDUMP}.t${cyc}z.atm +else + echo "Invalid FHRGRP= $FHRGRP for postanl.sh. exit 99" + exit 99 +fi + + +#--------------------------------------------------------------- +for fhr in $fhrlst; do + + if [ ! -f $restart_file${fhr}.nemsio -a ! -f $restart_file${fhr}.nc -a ! -f $restart_file${fhr}.txt ]; then + echo "Nothing to process for FHR = $fhr, cycle, wait for 5 minutes" + sleep 300 + fi + if [ ! -f $restart_file${fhr}.nemsio -a ! -f $restart_file${fhr}.nc -a ! -f $restart_file${fhr}.txt ]; then + echo "Nothing to process for FHR = $fhr, cycle, skip" + continue + fi + + #master=$ROTDIR/${CDUMP}.${PDY}/${cyc}/$COMPONENT/${CDUMP}.t${cyc}z.master.grb2f${fhr} + pgb0p25=$ROTDIR/${CDUMP}.${PDY}/${cyc}/$COMPONENT/${CDUMP}.t${cyc}z.pgrb2.0p25.f${fhr} + if [ ! -s $pgb0p25 ]; then + export post_times=$fhr + $HOMEgfs/jobs/JGLOBAL_ATMOS_NCEPPOST + status=$? + [[ $status -ne 0 ]] && exit $status + fi + +done + +############################################################### +# Exit out cleanly +exit 0 diff --git a/jobs/rocoto/sfcanl.sh b/jobs/rocoto/sfcanl.sh new file mode 100755 index 0000000000..bd89690324 --- /dev/null +++ b/jobs/rocoto/sfcanl.sh @@ -0,0 +1,13 @@ +#!/bin/ksh -x + +############################################################### +# Source FV3GFS workflow modules +. $HOMEgfs/ush/load_fv3gfs_modules.sh +status=$? +[[ $status -ne 0 ]] && exit $status + +############################################################### +# Execute the JJOB +$HOMEgfs/jobs/JGLOBAL_ATMOS_SFCANL +status=$? +exit $status diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn index b0df6fd898..5c1f06472d 100755 --- a/parm/config/config.base.emc.dyn +++ b/parm/config/config.base.emc.dyn @@ -303,7 +303,7 @@ export imp_physics=11 # Shared parameters # DA engine -export DO_UFSDA="NO" +export DO_JEDIVAR="NO" # Hybrid related export DOHYBVAR="YES" diff --git a/parm/config/config.base.nco.static b/parm/config/config.base.nco.static index 57b6c7ff79..aca8c9bb97 100755 --- a/parm/config/config.base.nco.static +++ b/parm/config/config.base.nco.static @@ -216,7 +216,7 @@ export imp_physics=11 # Shared parameters # DA engine -export DO_UFSDA="NO" +export DO_JEDIVAR="NO" # Hybrid related export DOHYBVAR="YES" diff --git a/scripts/exglobal_atmos_sfcanl.sh b/scripts/exglobal_atmos_sfcanl.sh new file mode 100755 index 0000000000..bcf1139ce8 --- /dev/null +++ b/scripts/exglobal_atmos_sfcanl.sh @@ -0,0 +1,302 @@ +#!/bin/ksh +################################################################################ +#### UNIX Script Documentation Block +# . . +# Script name: exglobal_atmos_sfcanl.sh +# Script description: Makes global model surface analysis files +# +# Author: Russ Treadon Org: NCEP/EMC Date: 2021-12-13 +# +# Abstract: This script makes global model surface analysis filesk +# +# $Id$ +# +# Attributes: +# Language: POSIX shell +# Machine: WCOSS-Dell +# +################################################################################ + +# Set environment. +export VERBOSE=${VERBOSE:-"YES"} +if [ $VERBOSE = "YES" ]; then + echo $(date) EXECUTING $0 $* >&2 + set -x +fi + +# Directories. +pwd=$(pwd) + +# Base variables +CDATE=${CDATE:-"2001010100"} +CDUMP=${CDUMP:-"gdas"} +GDUMP=${GDUMP:-"gdas"} + +# Derived base variables +GDATE=$($NDATE -$assim_freq $CDATE) +BDATE=$($NDATE -3 $CDATE) +PDY=$(echo $CDATE | cut -c1-8) +cyc=$(echo $CDATE | cut -c9-10) +bPDY=$(echo $BDATE | cut -c1-8) +bcyc=$(echo $BDATE | cut -c9-10) + +# Utilities +export NCP=${NCP:-"/bin/cp"} +export NMV=${NMV:-"/bin/mv"} +export NLN=${NLN:-"/bin/ln -sf"} +export CHGRP_CMD=${CHGRP_CMD:-"chgrp ${group_name:-rstprod}"} +export NEMSIOGET=${NEMSIOGET:-${NWPROD}/exec/nemsio_get} +export NCLEN=${NCLEN:-$HOMEgfs/ush/getncdimlen} +COMPRESS=${COMPRESS:-gzip} +UNCOMPRESS=${UNCOMPRESS:-gunzip} +APRUNCFP=${APRUNCFP:-""} + +# IAU +DOIAU=${DOIAU:-"NO"} +export IAUFHRS=${IAUFHRS:-"6"} + +# Surface cycle related parameters +DOGCYCLE=${DOGCYCLE:-"NO"} +CYCLESH=${CYCLESH:-$HOMEgfs/ush/global_cycle.sh} +export CYCLEXEC=${CYCLEXEC:-$HOMEgfs/exec/global_cycle} +NTHREADS_CYCLE=${NTHREADS_CYCLE:-24} +APRUN_CYCLE=${APRUN_CYCLE:-${APRUN:-""}} +export SNOW_NUDGE_COEFF=${SNOW_NUDGE_COEFF:-'-2.'} +export CYCLVARS=${CYCLVARS:-""} +export FHOUR=${FHOUR:-0} +export DELTSFC=${DELTSFC:-6} +export FIXgsm=${FIXgsm:-$HOMEgfs/fix/fix_am} +export FIXfv3=${FIXfv3:-$HOMEgfs/fix/fix_fv3_gmted2010} + +# FV3 specific info (required for global_cycle) +export CASE=${CASE:-"C384"} +ntiles=${ntiles:-6} + +# IAU +DOIAU=${DOIAU:-"NO"} +export IAUFHRS=${IAUFHRS:-"6"} + +# Dependent Scripts and Executables +export NTHREADS_CALCINC=${NTHREADS_CALCINC:-1} +export APRUN_CALCINC=${APRUN_CALCINC:-${APRUN:-""}} +export APRUN_CALCANL=${APRUN_CALCANL:-${APRUN:-""}} +export APRUN_CHGRES=${APRUN_CALCANL:-${APRUN:-""}} + +export CALCANLEXEC=${CALCANLEXEC:-$HOMEgfs/exec/calc_analysis.x} +export CHGRESNCEXEC=${CHGRESNCEXEC:-$HOMEgfs/exec/enkf_chgres_recenter_nc.x} +export CHGRESINCEXEC=${CHGRESINCEXEC:-$HOMEgfs/exec/interp_inc.x} +export NTHREADS_CHGRES=${NTHREADS_CHGRES:-1} +CALCINCPY=${CALCINCPY:-$HOMEgfs/ush/calcinc_gfs.py} +CALCANLPY=${CALCANLPY:-$HOMEgfs/ush/calcanl_gfs.py} + +export APRUN_CHGRES=${APRUN_CALCANL:-${APRUN:-""}} +CHGRESEXEC=${CHGRESEXEC:-$HOMEgfs/exec/enkf_chgres_recenter.x} + +# OPS flags +RUN=${RUN:-""} +SENDECF=${SENDECF:-"NO"} +SENDDBN=${SENDDBN:-"NO"} +RUN_GETGES=${RUN_GETGES:-"NO"} +GETGESSH=${GETGESSH:-"getges.sh"} +export gesenvir=${gesenvir:-$envir} + +# Observations +OPREFIX=${OPREFIX:-""} +OSUFFIX=${OSUFFIX:-""} + +# Guess files +GPREFIX=${GPREFIX:-""} +GSUFFIX=${GSUFFIX:-$SUFFIX} +ATMG03=${ATMG03:-${COMIN_GES}/${GPREFIX}atmf003${GSUFFIX}} +ATMG04=${ATMG04:-${COMIN_GES}/${GPREFIX}atmf004${GSUFFIX}} +ATMG05=${ATMG05:-${COMIN_GES}/${GPREFIX}atmf005${GSUFFIX}} +ATMGES=${ATMGES:-${COMIN_GES}/${GPREFIX}atmf006${GSUFFIX}} +ATMG07=${ATMG07:-${COMIN_GES}/${GPREFIX}atmf007${GSUFFIX}} +ATMG08=${ATMG08:-${COMIN_GES}/${GPREFIX}atmf008${GSUFFIX}} +ATMG09=${ATMG09:-${COMIN_GES}/${GPREFIX}atmf009${GSUFFIX}} + +SFCG03=${SFCG03:-${COMIN_GES}/${GPREFIX}sfcf003${GSUFFIX}} +SFCG04=${SFCG04:-${COMIN_GES}/${GPREFIX}sfcf004${GSUFFIX}} +SFCG05=${SFCG05:-${COMIN_GES}/${GPREFIX}sfcf005${GSUFFIX}} +SFCGES=${SFCGES:-${COMIN_GES}/${GPREFIX}sfcf006${GSUFFIX}} +SFCG07=${SFCG07:-${COMIN_GES}/${GPREFIX}sfcf007${GSUFFIX}} +SFCG08=${SFCG08:-${COMIN_GES}/${GPREFIX}sfcf008${GSUFFIX}} +SFCG09=${SFCG09:-${COMIN_GES}/${GPREFIX}sfcf009${GSUFFIX}} + +# Analysis files +export APREFIX=${APREFIX:-""} +export ASUFFIX=${ASUFFIX:-$SUFFIX} +SFCANL=${SFCANL:-${COMOUT}/${APREFIX}sfcanl${ASUFFIX}} +DTFANL=${DTFANL:-${COMOUT}/${APREFIX}dtfanl.nc} +ATMANL=${ATMANL:-${COMOUT}/${APREFIX}atmanl${ASUFFIX}} + +# Increment files +ATMINC=${ATMINC:-${COMOUT}/${APREFIX}atminc.nc} + +# Set script parameters +export DOIAU=${DOIAU:-"NO"} +DO_CALC_INCREMENT=${DO_CALC_INCREMENT:-"NO"} +DO_CALC_ANALYSIS=${DO_CALC_ANALYSIS:-"NO"} + +# Get header information from Guess files +if [ ${SUFFIX} = ".nc" ]; then + LONB=${LONB:-$($NCLEN $ATMGES grid_xt)} # get LONB + LATB=${LATB:-$($NCLEN $ATMGES grid_yt)} # get LATB + LEVS=${LEVS:-$($NCLEN $ATMGES pfull)} # get LEVS + JCAP=${JCAP:--9999} # there is no jcap in these files +else + LONB=${LONB:-$($NEMSIOGET $ATMGES dimx | grep -i "dimx" | awk -F"= " '{print $2}' | awk -F" " '{print $1}')} # 'get LONB + LATB=${LATB:-$($NEMSIOGET $ATMGES dimy | grep -i "dimy" | awk -F"= " '{print $2}' | awk -F" " '{print $1}')} # 'get LATB + LEVS=${LEVS:-$($NEMSIOGET $ATMGES dimz | grep -i "dimz" | awk -F"= " '{print $2}' | awk -F" " '{print $1}')} # 'get LEVS + JCAP=${JCAP:-$($NEMSIOGET $ATMGES jcap | grep -i "jcap" | awk -F"= " '{print $2}' | awk -F" " '{print $1}')} # 'get JCAP +fi +[ $JCAP -eq -9999 -a $LATB -ne -9999 ] && JCAP=$((LATB-2)) +[ $LONB -eq -9999 -o $LATB -eq -9999 -o $LEVS -eq -9999 -o $JCAP -eq -9999 ] && exit -9999 + + +# Get dimension information based on CASE +res=$(echo $CASE | cut -c2-) +JCAP_CASE=$((res*2-2)) +LATB_CASE=$((res*2)) +LONB_CASE=$((res*4)) + +# logic for netCDF I/O +if [ ${SUFFIX} = ".nc" ]; then + # GSI namelist options to use netCDF background + use_gfs_nemsio=".false." + use_gfs_ncio=".true." +else + # GSI namelist options to use NEMSIO background + use_gfs_nemsio=".true." + use_gfs_ncio=".false." +fi + +# determine if writing or calculating increment +if [ $DO_CALC_INCREMENT = "YES" ]; then + write_fv3_increment=".false." +else + write_fv3_increment=".true." + WRITE_INCR_ZERO="incvars_to_zero= $INCREMENTS_TO_ZERO," + WRITE_ZERO_STRAT="incvars_zero_strat= $INCVARS_ZERO_STRAT," + WRITE_STRAT_EFOLD="incvars_efold= $INCVARS_EFOLD," +fi + + +################################################################################ +# Preprocessing +mkdata=NO +if [ ! -d $DATA ]; then + mkdata=YES + mkdir -p $DATA +fi + +cd $DATA || exit 99 + +[[ $DONST = "YES" ]] && $NLN $NSSTBF nsstbufr + + +############################################################## +# Required model guess files +$NLN $SFCG03 sfcf03 +$NLN $SFCGES sfcf06 +$NLN $SFCG09 sfcf09 + +[[ -f $SFCG04 ]] && $NLN $SFCG04 sfcf04 +[[ -f $SFCG05 ]] && $NLN $SFCG05 sfcf05 +[[ -f $SFCG07 ]] && $NLN $SFCG07 sfcf07 +[[ -f $SFCG08 ]] && $NLN $SFCG08 sfcf08 + + +############################################################## +# Output files +if [ $DONST = "YES" ]; then + $NLN $DTFANL dtfanl +fi + + +############################################################## +# Update surface fields in the FV3 restart's using global_cycle +if [ $DOGCYCLE = "YES" ]; then + + mkdir -p $COMOUT/RESTART + + # Global cycle requires these files + export FNTSFA=${FNTSFA:-$COMIN_OBS/${OPREFIX}rtgssthr.grb} + export FNACNA=${FNACNA:-$COMIN_OBS/${OPREFIX}seaice.5min.blend.grb} + export FNSNOA=${FNSNOA:-$COMIN_OBS/${OPREFIX}snogrb_t${JCAP_CASE}.${LONB_CASE}.${LATB_CASE}} + [[ ! -f $FNSNOA ]] && export FNSNOA="$COMIN_OBS/${OPREFIX}snogrb_t1534.3072.1536" + FNSNOG=${FNSNOG:-$COMIN_GES_OBS/${GPREFIX}snogrb_t${JCAP_CASE}.${LONB_CASE}.${LATB_CASE}} + [[ ! -f $FNSNOG ]] && FNSNOG="$COMIN_GES_OBS/${GPREFIX}snogrb_t1534.3072.1536" + + # Set CYCLVARS by checking grib date of current snogrb vs that of prev cycle + if [ $RUN_GETGES = "YES" ]; then + snoprv=$($GETGESSH -q -t snogrb_$JCAP_CASE -e $gesenvir -n $GDUMP -v $GDATE) + else + snoprv=${snoprv:-$FNSNOG} + fi + + if [ $($WGRIB -4yr $FNSNOA 2>/dev/null | grep -i snowc | awk -F: '{print $3}' | awk -F= '{print $2}') -le \ + $($WGRIB -4yr $snoprv 2>/dev/null | grep -i snowc | awk -F: '{print $3}' | awk -F= '{print $2}') ] ; then + export FNSNOA=" " + export CYCLVARS="FSNOL=99999.,FSNOS=99999.," + else + export SNOW_NUDGE_COEFF=${SNOW_NUDGE_COEFF:-0.} + export CYCLVARS="FSNOL=${SNOW_NUDGE_COEFF},$CYCLVARS" + fi + + if [ $DONST = "YES" ]; then + export NST_ANL=".true." + export GSI_FILE=${GSI_FILE:-$COMOUT/${APREFIX}dtfanl.nc} + else + export NST_ANL=".false." + export GSI_FILE="NULL" + fi + + if [ $DOIAU = "YES" ]; then + # update surface restarts at the beginning of the window, if IAU + # For now assume/hold dtfanl.nc valid at beginning of window + for n in $(seq 1 $ntiles); do + $NLN $COMIN_GES/RESTART/$bPDY.${bcyc}0000.sfc_data.tile${n}.nc $DATA/fnbgsi.00$n + $NLN $COMOUT/RESTART/$bPDY.${bcyc}0000.sfcanl_data.tile${n}.nc $DATA/fnbgso.00$n + $NLN $FIXfv3/$CASE/${CASE}_grid.tile${n}.nc $DATA/fngrid.00$n + $NLN $FIXfv3/$CASE/${CASE}_oro_data.tile${n}.nc $DATA/fnorog.00$n + done + + export APRUNCY=$APRUN_CYCLE + export OMP_NUM_THREADS_CY=$NTHREADS_CYCLE + export MAX_TASKS_CY=$ntiles + + $CYCLESH + export err=$?; err_chk + fi + # update surface restarts at middle of window + for n in $(seq 1 $ntiles); do + $NLN $COMIN_GES/RESTART/$PDY.${cyc}0000.sfc_data.tile${n}.nc $DATA/fnbgsi.00$n + $NLN $COMOUT/RESTART/$PDY.${cyc}0000.sfcanl_data.tile${n}.nc $DATA/fnbgso.00$n + $NLN $FIXfv3/$CASE/${CASE}_grid.tile${n}.nc $DATA/fngrid.00$n + $NLN $FIXfv3/$CASE/${CASE}_oro_data.tile${n}.nc $DATA/fnorog.00$n + done + + export APRUNCY=$APRUN_CYCLE + export OMP_NUM_THREADS_CY=$NTHREADS_CYCLE + export MAX_TASKS_CY=$ntiles + + $CYCLESH + export err=$?; err_chk +fi + + +################################################################################ +# Postprocessing +cd $pwd +[[ $mkdata = "YES" ]] && rm -rf $DATA + + +################################################################################ +set +x +if [ $VERBOSE = "YES" ]; then + echo $(date) EXITING $0 with return code $err >&2 +fi +exit $err + +################################################################################ diff --git a/sorc/checkout.sh b/sorc/checkout.sh index c63b5d266b..7c490361c4 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -43,7 +43,6 @@ if [[ ! -d gsi.fd ]] ; then rm -f ${topdir}/checkout-gsi.log git clone --recursive https://github.com/NOAA-EMC/GSI.git gsi.fd >> ${logdir}/checkout-gsi.log 2>&1 cd gsi.fd - git checkout a62dec6 git submodule update cd ${topdir} else diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 5ceeaf69b5..f8aeefac9b 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -198,16 +198,16 @@ cd ${pwd}/../fix ||exit 8 [[ -d gdas ]] && rm -rf gdas mkdir -p gdas cd gdas - $LINK ../../sorc/gsi.fd/util/Minimization_Monitor/nwprod/gdas.v1.0.0/fix/gdas_minmon_cost.txt . - $LINK ../../sorc/gsi.fd/util/Minimization_Monitor/nwprod/gdas.v1.0.0/fix/gdas_minmon_gnorm.txt . + $LINK ../../sorc/gsi.fd/util/Minimization_Monitor/nwprod/gdas/fix/gdas_minmon_cost.txt . + $LINK ../../sorc/gsi.fd/util/Minimization_Monitor/nwprod/gdas/fix/gdas_minmon_gnorm.txt . $LINK ../../sorc/gsi.fd/util/Ozone_Monitor/nwprod/gdas_oznmon/fix/gdas_oznmon_base.tar . $LINK ../../sorc/gsi.fd/util/Ozone_Monitor/nwprod/gdas_oznmon/fix/gdas_oznmon_satype.txt . $LINK ../../sorc/gsi.fd/util/Radiance_Monitor/nwprod/gdas_radmon/fix/gdas_radmon_base.tar . $LINK ../../sorc/gsi.fd/util/Radiance_Monitor/nwprod/gdas_radmon/fix/gdas_radmon_satype.txt . $LINK ../../sorc/gsi.fd/util/Radiance_Monitor/nwprod/gdas_radmon/fix/gdas_radmon_scaninfo.txt . cd ${pwd}/../jobs ||exit 8 - $LINK ../sorc/gsi.fd/util/Minimization_Monitor/nwprod/gdas.v1.0.0/jobs/JGDAS_ATMOS_VMINMON . - $LINK ../sorc/gsi.fd/util/Minimization_Monitor/nwprod/gfs.v1.0.0/jobs/JGFS_ATMOS_VMINMON . + $LINK ../sorc/gsi.fd/util/Minimization_Monitor/nwprod/gdas/jobs/JGDAS_ATMOS_VMINMON . + $LINK ../sorc/gsi.fd/util/Minimization_Monitor/nwprod/gfs/jobs/JGFS_ATMOS_VMINMON . $LINK ../sorc/gsi.fd/util/Ozone_Monitor/nwprod/gdas_oznmon/jobs/JGDAS_ATMOS_VERFOZN . $LINK ../sorc/gsi.fd/util/Radiance_Monitor/nwprod/gdas_radmon/jobs/JGDAS_ATMOS_VERFRAD . cd ${pwd}/../parm ||exit 8 @@ -215,19 +215,19 @@ cd ${pwd}/../parm ||exit 8 mkdir -p mon cd mon $LINK ../../sorc/gsi.fd/util/Radiance_Monitor/nwprod/gdas_radmon/parm/gdas_radmon.parm da_mon.parm -# $LINK ../../sorc/gsi.fd/util/Minimization_Monitor/nwprod/gdas.v1.0.0/parm/gdas_minmon.parm . -# $LINK ../../sorc/gsi.fd/util/Minimization_Monitor/nwprod/gfs.v1.0.0/parm/gfs_minmon.parm . +# $LINK ../../sorc/gsi.fd/util/Minimization_Monitor/nwprod/gdas/parm/gdas_minmon.parm . +# $LINK ../../sorc/gsi.fd/util/Minimization_Monitor/nwprod/gfs/parm/gfs_minmon.parm . $LINK ../../sorc/gsi.fd/util/Ozone_Monitor/nwprod/gdas_oznmon/parm/gdas_oznmon.parm . # $LINK ../../sorc/gsi.fd/util/Radiance_Monitor/nwprod/gdas_radmon/parm/gdas_radmon.parm . cd ${pwd}/../scripts ||exit 8 - $LINK ../sorc/gsi.fd/util/Minimization_Monitor/nwprod/gdas.v1.0.0/scripts/exgdas_atmos_vminmon.sh . - $LINK ../sorc/gsi.fd/util/Minimization_Monitor/nwprod/gfs.v1.0.0/scripts/exgfs_atmos_vminmon.sh . + $LINK ../sorc/gsi.fd/util/Minimization_Monitor/nwprod/gdas/scripts/exgdas_atmos_vminmon.sh . + $LINK ../sorc/gsi.fd/util/Minimization_Monitor/nwprod/gfs/scripts/exgfs_atmos_vminmon.sh . $LINK ../sorc/gsi.fd/util/Ozone_Monitor/nwprod/gdas_oznmon/scripts/exgdas_atmos_verfozn.sh . $LINK ../sorc/gsi.fd/util/Radiance_Monitor/nwprod/gdas_radmon/scripts/exgdas_atmos_verfrad.sh . cd ${pwd}/../ush ||exit 8 - $LINK ../sorc/gsi.fd/util/Minimization_Monitor/nwprod/minmon_shared.v1.0.1/ush/minmon_xtrct_costs.pl . - $LINK ../sorc/gsi.fd/util/Minimization_Monitor/nwprod/minmon_shared.v1.0.1/ush/minmon_xtrct_gnorms.pl . - $LINK ../sorc/gsi.fd/util/Minimization_Monitor/nwprod/minmon_shared.v1.0.1/ush/minmon_xtrct_reduct.pl . + $LINK ../sorc/gsi.fd/util/Minimization_Monitor/nwprod/minmon_shared/ush/minmon_xtrct_costs.pl . + $LINK ../sorc/gsi.fd/util/Minimization_Monitor/nwprod/minmon_shared/ush/minmon_xtrct_gnorms.pl . + $LINK ../sorc/gsi.fd/util/Minimization_Monitor/nwprod/minmon_shared/ush/minmon_xtrct_reduct.pl . $LINK ../sorc/gsi.fd/util/Ozone_Monitor/nwprod/oznmon_shared/ush/ozn_xtrct.sh . $LINK ../sorc/gsi.fd/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_ck_stdout.sh . $LINK ../sorc/gsi.fd/util/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_err_rpt.sh . diff --git a/ush/rocoto/setup_workflow.py b/ush/rocoto/setup_workflow.py index 32436e6b3e..f87e56ea49 100755 --- a/ush/rocoto/setup_workflow.py +++ b/ush/rocoto/setup_workflow.py @@ -42,7 +42,7 @@ def main(): print(f'input arg: --expdir = {repr(args.expdir)}') sys.exit(1) - gfs_steps = ['analcalc', 'gldas', 'fcst', 'postsnd', 'post', 'vrfy', 'arch'] + gfs_steps = ['prep', 'analcalc', 'gldas', 'fcst', 'postsnd', 'postanl', 'post', 'vrfy', 'arch'] gfs_steps_gempak = ['gempak'] gfs_steps_awips = ['awips'] gfs_steps_wafs = ['wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25'] @@ -55,10 +55,10 @@ def main(): # From gfsv16b latest # gfs_steps = ['prep', 'anal', 'gldas', 'fcst', 'postsnd', 'post', 'awips', 'gempak', 'vrfy', 'metp', 'arch'] hyb_steps = ['eobs', 'ediag', 'eomg', 'eupd', 'ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc'] - gsida_steps = ['prep', 'anal', 'analdiag'] - ufsda_steps = ['atmanalprep', 'atmanalrun', 'atmanalpost'] + gsida_steps = ['anal', 'analdiag'] + jedivar_steps = ['atmanalprep', 'atmanalrun', 'sfcanl', 'atmanalpost'] - steps = gfs_steps + ufsda_steps if _base.get('DO_UFSDA', 'NO') == 'YES' else gfs_steps + gsida_steps + steps = gfs_steps + jedivar_steps if _base.get('DO_JEDIVAR', 'NO') == 'YES' else gfs_steps + gsida_steps steps = steps + hyb_steps if _base.get('DOHYBVAR', 'NO') == 'YES' else steps steps = steps + metp_steps if _base.get('DO_METP', 'NO') == 'YES' else steps steps = steps + gfs_steps_gempak if _base.get('DO_GEMPAK', 'NO') == 'YES' else steps @@ -197,7 +197,7 @@ def get_definitions(base): strings.append(f'''\t\n''') if scheduler in ['slurm']: strings.append(f'''\t\n''') - strings.append(f'''\t\n''') + strings.append(f'''\t\n''') strings.append(f'\t\n') strings.append('\n') strings.append('\t\n') @@ -244,12 +244,11 @@ def get_gdasgfs_resources(dict_configs, cdump='gdas'): do_gldas = base.get('DO_GLDAS', 'NO').upper() do_wave = base.get('DO_WAVE', 'NO').upper() do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper() - do_ufsda = base.get('DO_UFSDA', 'NO').upper() + do_jedivar = base.get('DO_JEDIVAR', 'NO').upper() reservation = base.get('RESERVATION', 'NONE').upper() - #tasks = ['prep', 'anal', 'fcst', 'post', 'vrfy', 'arch'] - if do_ufsda in ['Y', 'YES']: - task_prep = ['atmanalprep'] + if do_jedivar in ['Y', 'YES']: + task_prep = ['prep', 'atmanalprep'] task_anal = ['atmanalrun'] task_diag = ['atmanalpost'] else: @@ -261,13 +260,15 @@ def get_gdasgfs_resources(dict_configs, cdump='gdas'): tasks += ['analcalc'] if cdump in ['gdas']: tasks += task_diag + if do_jedivar in ['Y', 'YES']: + tasks += ['sfcanl'] if cdump in ['gdas'] and do_gldas in ['Y', 'YES']: tasks += ['gldas'] if cdump in ['gdas'] and do_wave in ['Y', 'YES'] and do_wave_cdump in ['GDAS', 'BOTH']: #tasks += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostpnt', 'wavestat'] tasks += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] - tasks += ['fcst', 'post', 'vrfy', 'arch'] + tasks += ['fcst', 'postanl', 'post', 'vrfy', 'arch'] if cdump in ['gfs'] and do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: #tasks += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostpnt', 'wavestat'] @@ -297,11 +298,12 @@ def get_gdasgfs_resources(dict_configs, cdump='gdas'): taskstr = f'{task.upper()}_{cdump.upper()}' strings = [] - strings.append(f'\t\n') if scheduler in ['slurm']: if task in ['arch']: + strings.append(f'\t\n') strings.append(f'\t\n') else: + strings.append(f'\t\n') strings.append(f'\t\n') strings.append(f'\t\n') @@ -377,11 +379,12 @@ def get_hyb_resources(dict_configs): taskstr = f'{task.upper()}_{cdump.upper()}' strings = [] - strings.append(f'\t\n') if scheduler in ['slurm']: if task in ['earc']: + strings.append(f'\t\n') strings.append(f'\t\n') else: + strings.append(f'\t\n') strings.append(f'\t\n') strings.append(f'\t\n') @@ -423,7 +426,7 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): do_metp = base.get('DO_METP', 'NO').upper() do_gldas = base.get('DO_GLDAS', 'NO').upper() do_wave = base.get('DO_WAVE', 'NO').upper() - do_ufsda = base.get('DO_UFSDA', 'NO').upper() + do_jedivar = base.get('DO_JEDIVAR', 'NO').upper() if do_wave in ['YES']: do_wave_bnd = dict_configs['wavepostsbs'].get('DOBNDPNT_WAVE', "YES").upper() do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper() @@ -432,7 +435,7 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): dict_tasks = OrderedDict() - if do_ufsda in ['Y', 'YES']: + if do_jedivar in ['Y', 'YES']: task_prep = 'atmanalprep' task_anal = 'atmanalrun' task_diag = 'atmanalpost' @@ -457,14 +460,40 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): if gfs_enkf and cdump in ['gfs']: if gfs_cyc == 4: - task = wfu.create_wf_task(task=task_prep, cdump=cdump, envar=envars, dependency=dependencies) + task = wfu.create_wf_task(task='prep', cdump=cdump, envar=envars, dependency=dependencies) else: - task = wfu.create_wf_task(task=task_prep, cdump=cdump, envar=envars, dependency=dependencies, cycledef='gdas') + task = wfu.create_wf_task(task='prep', cdump=cdump, envar=envars, dependency=dependencies, cycledef='gdas') else: - task = wfu.create_wf_task(task=task_prep, cdump=cdump, envar=envars, dependency=dependencies) + task = wfu.create_wf_task(task='prep', cdump=cdump, envar=envars, dependency=dependencies) - dict_tasks[f'{cdump}{task_prep}'] = task + dict_tasks[f'{cdump}prep'] = task + + # atmanalprep + if do_jedivar in ['Y', 'YES']: + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{"gdas"}post', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.atmf009{gridsuffix}' + dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&DMPDIR;/{cdump}{dumpsuffix}.@Y@m@d/@H/{cdump}.t@Hz.updated.status.tm00.bufr_d' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + gfs_enkf = True if eupd_cyc in ['BOTH', 'GFS'] and dohybvar in ['Y', 'YES'] else False + + if gfs_enkf and cdump in ['gfs']: + if gfs_cyc == 4: + task = wfu.create_wf_task(task='atmanalprep', cdump=cdump, envar=envars, dependency=dependencies) + else: + task = wfu.create_wf_task(task='atmanalprep', cdump=cdump, envar=envars, dependency=dependencies, cycledef='gdas') + + else: + task = wfu.create_wf_task(task='atmanalprep', cdump=cdump, envar=envars, dependency=dependencies) + + dict_tasks[f'{cdump}atmanalprep'] = task # wave tasks in gdas or gfs or both if do_wave_cdump in ['BOTH']: @@ -477,7 +506,7 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): # waveinit if do_wave in ['Y', 'YES'] and cdump in cdumps: deps = [] - dep_dict = {'type': 'task', 'name': '{cdump}{task_prep}'} + dep_dict = {'type': 'task', 'name': '{cdump}prep'} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) @@ -508,6 +537,19 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): dict_tasks[f'{cdump}{task_anal}'] = task + # sfcanl + if do_jedivar in ['Y', 'YES']: + deps = [] + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loginc.txt' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{cdump}{task_anal}'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('sfcanl', cdump=cdump, envar=envars, dependency=dependencies) + + dict_tasks[f'{cdump}sfcanl'] = task + # analcalc deps1 = [] data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loginc.txt' @@ -515,6 +557,9 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): deps1.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{cdump}{task_anal}'} deps.append(rocoto.add_dependency(dep_dict)) + if do_jedivar in ['Y', 'YES']: + dep_dict = {'type': 'task', 'name': f'{cdump}sfcanl'} + deps.append(rocoto.add_dependency(dep_dict)) if dohybvar in ['y', 'Y', 'yes', 'YES'] and cdump == 'gdas': dep_dict = {'type': 'task', 'name': f'{"gdas"}echgres', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) @@ -533,7 +578,7 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): deps1.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{cdump}{task_anal}'} deps1.append(rocoto.add_dependency(dep_dict)) - dependencies1 = rocoto.create_dependency(dep_condition='or', dep=deps1) + dependencies1 = rocoto.create_dependency(dep_condition='and', dep=deps1) deps2 = [] deps2 = dependencies1 @@ -553,7 +598,10 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): deps1.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{cdump}{task_anal}'} deps1.append(rocoto.add_dependency(dep_dict)) - dependencies1 = rocoto.create_dependency(dep_condition='or', dep=deps1) + if do_jedivar in ['Y', 'YES']: + dep_dict = {'type': 'task', 'name': f'{cdump}sfcanl'} + deps1.append(rocoto.add_dependency(dep_dict)) + dependencies1 = rocoto.create_dependency(dep_condition='and', dep=deps1) deps2 = [] deps2 = dependencies1 @@ -578,9 +626,15 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): else: dep_dict = {'type': 'task', 'name': f'{cdump}analcalc'} deps1.append(rocoto.add_dependency(dep_dict)) + if do_jedivar in ['Y', 'YES']: + dep_dict = {'type': 'task', 'name': f'{cdump}sfcanl'} + deps1.append(rocoto.add_dependency(dep_dict)) elif cdump in ['gfs']: dep_dict = {'type': 'task', 'name': f'{cdump}{task_anal}'} deps1.append(rocoto.add_dependency(dep_dict)) + if do_jedivar in ['Y', 'YES']: + dep_dict = {'type': 'task', 'name': f'{cdump}sfcanl'} + deps1.append(rocoto.add_dependency(dep_dict)) dependencies1 = rocoto.create_dependency(dep_condition='or', dep=deps1) if do_wave in ['Y', 'YES'] and cdump in cdumps: @@ -595,6 +649,25 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): dict_tasks[f'{cdump}fcst'] = task + # postanl + deps = [] + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loganl.txt' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{cdump}analcalc'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) + fhrgrp = rocoto.create_envar(name='FHRGRP', value='anl') + fhrlst = rocoto.create_envar(name='FHRLST', value='anl') + ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') + postenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] + varname1, varname2, varname3 = 'grp', 'dep', 'lst' + varval1, varval2, varval3 = get_postgroups_anl(dict_configs['postanl'], cdump=cdump) + vardict = {varname2: varval2, varname3: varval3} + task = wfu.create_wf_task('postanl', cdump=cdump, envar=postenvars, dependency=dependencies) + + dict_tasks[f'{cdump}postanl'] = task + # post deps = [] data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.log#dep#.txt' @@ -965,7 +1038,7 @@ def get_hyb_tasks(dict_configs, cycledef='enkf'): nens = base['NMEM_ENKF'] lobsdiag_forenkf = base.get('lobsdiag_forenkf', '.false.').upper() eupd_cyc = base.get('EUPD_CYC', 'gdas').upper() - do_ufsda = base.get('DO_UFSDA', 'NO').upper() + do_jedivar = base.get('DO_JEDIVAR', 'NO').upper() eobs = dict_configs['eobs'] nens_eomg = eobs['NMEM_EOMGGRP'] @@ -1004,8 +1077,8 @@ def get_hyb_tasks(dict_configs, cycledef='enkf'): elif eupd_cyc in ['GDAS']: cdumps = ['gdas'] - if do_ufsda in ['Y', 'YES']: - task_prep = ['atmanalprep'] + if do_jedivar in ['Y', 'YES']: + task_prep = ['prep', 'atmanalprep'] else: task_prep = ['prep'] @@ -1016,7 +1089,7 @@ def get_hyb_tasks(dict_configs, cycledef='enkf'): # eobs deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}{task_prep}'} + dep_dict = {'type': 'task', 'name': f'{cdump}prep'} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'metatask', 'name': f'{"gdas"}epmn', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) @@ -1207,6 +1280,14 @@ def get_workflow_footer(): return ''.join(strings) +def get_postgroups_anl(post, cdump='gdas'): + + fhrgrp = 'anl' + fhrdep = 'anl' + fhrlst = 'anl' + + return fhrgrp, fhrdep, fhrlst + def get_postgroups(post, cdump='gdas'): fhmin = post['FHMIN'] @@ -1231,12 +1312,13 @@ def get_postgroups(post, cdump='gdas'): fhrs = np.array_split(fhrs, ngrps) fhrs = [f.tolist() for f in fhrs] - fhrgrp = ' '.join(['anl'] + [f'_{f[0]}-{f[-1]}' for f in fhrs]) - fhrdep = ' '.join(['anl'] + [f[-1] for f in fhrs]) - fhrlst = ' '.join(['anl'] + ['_'.join(f) for f in fhrs]) + fhrgrp = ' '.join([f'_{f[0]}-{f[-1]}' for f in fhrs]) + fhrdep = ' '.join([f[-1] for f in fhrs]) + fhrlst = ' '.join(['_'.join(f) for f in fhrs]) return fhrgrp, fhrdep, fhrlst + def get_awipsgroups(awips, cdump='gdas'): fhmin = awips['FHMIN'] @@ -1345,12 +1427,7 @@ def create_xml(dict_configs): dohybvar = base.get('DOHYBVAR', 'NO').upper() gfs_cyc = base.get('gfs_cyc', 0) eupd_cyc = base.get('EUPD_CYC', 'gdas').upper() - do_ufsda = base.get('DO_UFSDA', 'NO').upper() - - if do_ufsda in ['Y', 'YES']: - task_prep = ['atmanalprep'] - else: - task_prep = ['prep'] + do_jedivar = base.get('DO_JEDIVAR', 'NO').upper() # Start collecting workflow pieces preamble = get_preamble() @@ -1437,7 +1514,7 @@ def create_xml(dict_configs): if gfs_cyc != 0: xmlfile.append(dict_to_strings(dict_gfs_resources)) elif gfs_cyc == 0 and dohybvar in ['Y', 'YES'] and eupd_cyc in ['BOTH', 'GFS']: - xmlfile.append(dict_gfs_resources['gfs{task_prep}']) + xmlfile.append(dict_gfs_resources['gfsprep']) xmlfile.append(workflow_header) @@ -1449,7 +1526,7 @@ def create_xml(dict_configs): if gfs_cyc != 0: xmlfile.append(dict_to_strings(dict_gfs_tasks)) elif gfs_cyc == 0 and dohybvar in ['Y', 'YES'] and eupd_cyc in ['BOTH', 'GFS']: - xmlfile.append(dict_gfs_tasks['gfs{task_prep}']) + xmlfile.append(dict_gfs_tasks['gfsprep']) xmlfile.append('\n') xmlfile.append(workflow_footer) From 2782c076845593c053015c941cb56be761ebda3e Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 9 Jun 2022 17:37:35 +0000 Subject: [PATCH 29/45] Update gsi build and link --- sorc/build_gsi.sh | 4 ++-- sorc/link_workflow.sh | 11 ++++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/sorc/build_gsi.sh b/sorc/build_gsi.sh index b912f6c8d7..d5cbae10e4 100755 --- a/sorc/build_gsi.sh +++ b/sorc/build_gsi.sh @@ -20,8 +20,8 @@ if [ ! -d "../exec" ]; then fi cd gsi.fd/ush/ -./build_all_cmake.sh "PRODUCTION" "$cwd/gsi.fd" -##./build_all_cmake.sh "PRODUCTION" "$cwd/gsi.fd" "NCO" # use this line for pruned NCO install +export BUILD_CLEAN=YES +./build.sh exit diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index f8aeefac9b..d697442d62 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -280,12 +280,17 @@ for ufs_utilsexe in \ done for gsiexe in calc_analysis.x calc_increment_ens_ncio.x calc_increment_ens.x \ - getsfcensmeanp.x getsigensmeanp_smooth.x getsigensstatp.x global_enkf.x global_gsi.x \ - interp_inc.x ncdiag_cat.x oznmon_horiz.x oznmon_time.x radmon_angle.x \ + getsfcensmeanp.x getsigensmeanp_smooth.x getsigensstatp.x \ + interp_inc.x oznmon_horiz.x oznmon_time.x radmon_angle.x \ radmon_bcoef.x radmon_bcor.x radmon_time.x recentersigp.x;do [[ -s $gsiexe ]] && rm -f $gsiexe - $LINK ../sorc/gsi.fd/exec/$gsiexe . + $LINK ../sorc/gsi.fd/install/bin/$gsiexe . done +# Patch until DA scripts are updated new executable names. At that time +# add gsi.x, enkf.x, and ncdiag_cat_serial.x to gsiexe list above. +$LINK ../sorc/gsi.fd/install/bin/gsi.x ./global_gsi.x +$LINK ../sorc/gsi.fd/install/bin/enkf.x ./global_enkf.x +$LINK ../sorc/gsi.fd/install/bin/ncdiag_cat_serial.x ./ncdiag_cat.x for gdasexe in fv3jedi_addincrement.x fv3jedi_diffstates.x fv3jedi_ensvariance.x fv3jedi_hofx.x \ fv3jedi_var.x fv3jedi_convertincrement.x fv3jedi_dirac.x fv3jedi_error_covariance_training.x \ From ce04b51f086741d0cdfb07a6747d08b746c624e7 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 17 Jun 2022 16:16:19 +0000 Subject: [PATCH 30/45] Extend prototype JEDI-GDAS cycling to include JEDI hybvar and lgetkf --- env/ORION.env | 14 ++- jobs/JGDAS_GLOBAL_ATMOS_ENSANAL | 124 +++++++++++++++++++++ jobs/rocoto/atmensanalpost.sh | 16 +++ jobs/rocoto/atmensanalprep.sh | 16 +++ jobs/rocoto/atmensanalrun.sh | 16 +++ scripts/exglobal_atmos_sfcanl.sh | 6 +- sorc/link_workflow.sh | 10 +- ush/forecast_postdet.sh | 2 +- ush/rocoto/setup_workflow.py | 179 ++++++++++++++++++++++--------- 9 files changed, 317 insertions(+), 66 deletions(-) create mode 100755 jobs/JGDAS_GLOBAL_ATMOS_ENSANAL create mode 100755 jobs/rocoto/atmensanalpost.sh create mode 100755 jobs/rocoto/atmensanalprep.sh create mode 100755 jobs/rocoto/atmensanalrun.sh diff --git a/env/ORION.env b/env/ORION.env index 86057d7f47..b272e69a41 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -4,7 +4,7 @@ if [ $# -ne 1 ]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanalrun" + echo "atmanalrun atmensanalrun" echo "anal sfcanl fcst post vrfy metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -62,6 +62,18 @@ elif [ $step = "atmanalrun" ]; then [[ $NTHREADS_ATMANAL -gt $nth_max ]] && export NTHREADS_ATMANAL=$nth_max export APRUN_ATMANAL="$launcher -n $npe_atmanalrun" +elif [ $step = "atmensanalrun" ]; then + + export CFP_MP=${CFP_MP:-"YES"} + export USE_CFP=${USE_CFP:-"YES"} + export APRUNCFP="$launcher -n \$ncmd --multi-prog" + + nth_max=$(($npe_node_max / $npe_node_atmensanalrun)) + + export NTHREADS_ATMENSANAL=${nth_atmensanalrun:-$nth_max} + [[ $NTHREADS_ATMENSANAL -gt $nth_max ]] && export NTHREADS_ATMENSANAL=$nth_max + export APRUN_ATMENSANAL="$launcher -n $npe_atmensanalrun" + elif [ $step = "anal" ]; then export MKL_NUM_THREADS=4 diff --git a/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL b/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL new file mode 100755 index 0000000000..593d9e7640 --- /dev/null +++ b/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL @@ -0,0 +1,124 @@ +#!/bin/bash +set -ex +set -u + +export PS4='$SECONDS + ' +date + + +############################# +# Source relevant config files +############################# +export EXPDIR=${EXPDIR:-$HOMEgfs/parm/config} +config_path=${EXPDIR:-$NWROOT/gfs.${gfs_ver}/parm/config} +for config in $configs; do + . $config_path/config.$config + status=$? + [[ $status -ne 0 ]] && exit $status +done + + +########################################## +# Source machine runtime environment +########################################## +. $HOMEgfs/env/${machine}.env atmensanalrun +status=$? +[[ $status -ne 0 ]] && exit $status + + +############################################## +# Obtain unique process id (pid) and make temp directory +############################################## +export pid=${pid:-$$} +export outid=${outid:-"LL$job"} + +export DATA=${DATA:-${DATAROOT}/${jobid:?}} +mkdir -p $DATA +cd $DATA || (echo "$DATA does not exist. ABORT!"; exit 1) + + +############################################## +# Run setpdy and initialize PDY variables +############################################## +export cycle="t${cyc}z" +setpdy.sh +. ./PDY + + +############################################## +# Determine Job Output Name on System +############################################## +export pgmout="OUTPUT.${pid}" +export pgmerr=errfile + + +############################################## +# Set variables used in the script +############################################## +export CDATE=${CDATE:-${PDY}${cyc}} +export CDUMP=${CDUMP:-${RUN:-"gfs"}} +export COMPONENT=${COMPONENT:-atmos} + +############################################## +# Begin JOB SPECIFIC work +############################################## + +export GDATE=$(date +%Y%m%d%H -d "${CDATE:0:8} ${CDATE:8:2} - ${assim_freq} hours") +gPDY=${GDATE:0:8} +export gcyc=${GDATE:8:2} +export GDUMP=${GDUMP:-"gdas"} + +export OPREFIX="${CDUMP}.t${cyc}z." +export GPREFIX="${GDUMP}.t${gcyc}z." +export APREFIX="${CDUMP}.t${cyc}z." +export GSUFFIX=${GSUFFIX:-$SUFFIX} +export ASUFFIX=${ASUFFIX:-$SUFFIX} + +export COMOUT=${COMOUT:-$ROTDIR/$CDUMP.$PDY/$cyc/$COMPONENT} +export COMOUT_ENS=${COMOUT_ENS:-$ROTDIR/enkf$CDUMP.$PDY/$cyc/$COMPONENT} + +mkdir -p $COMOUT +mkdir -p $COMOUT_ENS + +# COMIN_GES and COMIN_GES_ENS are used in script +export COMIN="$ROTDIR/$CDUMP.$PDY/$cyc/$COMPONENT" +export COMIN_GES="$ROTDIR/$GDUMP.$gPDY/$gcyc/$COMPONENT" +export COMIN_GES_ENS="$ROTDIR/enkfgdas.$gPDY/$gcyc/$COMPONENT" + +# NOTE BELOW IS A HACK FOR TESTING +# PLEASE FIX THIS LATER +# ASK @aerorahul +# HOW TO HANDLE DIFFERENT COMPILERS/ETC. FOR MODEL VS DA +# PROD_UTIL, ETC. DO NOT EXIST FOR JEDI MODULE VERSIONS +module purge +##module use $HOMEgfs/sorc/gdas.cd/modulefiles +module use $HOMEgfs/sorc/gdas.cd_pr1730/modulefiles +module load GDAS/orion +export PYTHONPATH=$HOMEgfs/sorc/gdas.cd/ush/:$PYTHONPATH + +############################################################### +# Run relevant script + +$EXSCRIPT +status=$? +[[ $status -ne 0 ]] && exit $status + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [ -e "$pgmout" ] ; then + cat $pgmout +fi + +########################################## +# Remove the Temporary working directory +########################################## +cd $DATAROOT +[[ $KEEPDATA = "NO" ]] && rm -rf $DATA + +date +exit 0 diff --git a/jobs/rocoto/atmensanalpost.sh b/jobs/rocoto/atmensanalpost.sh new file mode 100755 index 0000000000..9db4e76a46 --- /dev/null +++ b/jobs/rocoto/atmensanalpost.sh @@ -0,0 +1,16 @@ +#!/bin/ksh -x + +############################################################### +# Source FV3GFS workflow modules +. $HOMEgfs/ush/load_fv3gfs_modules.sh +status=$? +[[ $status -ne 0 ]] && exit $status + +############################################################### +# set JJOB variables +export configs="base atmensanal atmensanalpost" +export EXSCRIPT=${GDASPOSTPY:-$HOMEgfs/sorc/gdas.cd/scripts/exgdas_global_atmos_ensanal_post.py} +# Execute the JJOB +$HOMEgfs/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL +status=$? +exit $status diff --git a/jobs/rocoto/atmensanalprep.sh b/jobs/rocoto/atmensanalprep.sh new file mode 100755 index 0000000000..acf4dd2185 --- /dev/null +++ b/jobs/rocoto/atmensanalprep.sh @@ -0,0 +1,16 @@ +#!/bin/ksh -x + +############################################################### +# Source FV3GFS workflow modules +. $HOMEgfs/ush/load_fv3gfs_modules.sh +status=$? +[[ $status -ne 0 ]] && exit $status + +############################################################### +# set JJOB variables +export configs="base atmensanal atmensanalprep" +export EXSCRIPT=${GDASPREPPY:-$HOMEgfs/sorc/gdas.cd/scripts/exgdas_global_atmos_analysis_prep.py} +# Execute the JJOB +$HOMEgfs/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL +status=$? +exit $status diff --git a/jobs/rocoto/atmensanalrun.sh b/jobs/rocoto/atmensanalrun.sh new file mode 100755 index 0000000000..bdb0f37612 --- /dev/null +++ b/jobs/rocoto/atmensanalrun.sh @@ -0,0 +1,16 @@ +#!/bin/ksh -x + +############################################################### +# Source FV3GFS workflow modules +. $HOMEgfs/ush/load_fv3gfs_modules.sh +status=$? +[[ $status -ne 0 ]] && exit $status + +############################################################### +# set JJOB variables +export configs="base atmensanal atmensanalrun" +export EXSCRIPT=${GDASRUNSH:-$HOMEgfs/sorc/gdas.cd/scripts/exgdas_global_atmos_ensanal_run.sh} +# Execute the JJOB +$HOMEgfs/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL +status=$? +exit $status diff --git a/scripts/exglobal_atmos_sfcanl.sh b/scripts/exglobal_atmos_sfcanl.sh index bcf1139ce8..8267f949e7 100755 --- a/scripts/exglobal_atmos_sfcanl.sh +++ b/scripts/exglobal_atmos_sfcanl.sh @@ -245,11 +245,9 @@ if [ $DOGCYCLE = "YES" ]; then fi if [ $DONST = "YES" ]; then - export NST_ANL=".true." - export GSI_FILE=${GSI_FILE:-$COMOUT/${APREFIX}dtfanl.nc} + export NST_FILE=${GSI_FILE:-$COMOUT/${APREFIX}dtfanl.nc} else - export NST_ANL=".false." - export GSI_FILE="NULL" + export NST_FILE="NULL" fi if [ $DOIAU = "YES" ]; then diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index d697442d62..e0aea1283b 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -188,6 +188,7 @@ cd ${pwd}/../ush ||exit 8 #------------------------------ cd ${pwd}/../jobs ||exit 8 $LINK ../sorc/gdas.cd/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS . + $LINK ../sorc/gdas.cd/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL . cd ${pwd}/../ush ||exit 8 $LINK ../sorc/gdas.cd/ush/ufsda . @@ -280,17 +281,12 @@ for ufs_utilsexe in \ done for gsiexe in calc_analysis.x calc_increment_ens_ncio.x calc_increment_ens.x \ - getsfcensmeanp.x getsigensmeanp_smooth.x getsigensstatp.x \ - interp_inc.x oznmon_horiz.x oznmon_time.x radmon_angle.x \ + getsfcensmeanp.x getsigensmeanp_smooth.x getsigensstatp.x enkf.x gsi.x \ + interp_inc.x ncdiag_cat_serial.x oznmon_horiz.x oznmon_time.x radmon_angle.x \ radmon_bcoef.x radmon_bcor.x radmon_time.x recentersigp.x;do [[ -s $gsiexe ]] && rm -f $gsiexe $LINK ../sorc/gsi.fd/install/bin/$gsiexe . done -# Patch until DA scripts are updated new executable names. At that time -# add gsi.x, enkf.x, and ncdiag_cat_serial.x to gsiexe list above. -$LINK ../sorc/gsi.fd/install/bin/gsi.x ./global_gsi.x -$LINK ../sorc/gsi.fd/install/bin/enkf.x ./global_enkf.x -$LINK ../sorc/gsi.fd/install/bin/ncdiag_cat_serial.x ./ncdiag_cat.x for gdasexe in fv3jedi_addincrement.x fv3jedi_diffstates.x fv3jedi_ensvariance.x fv3jedi_hofx.x \ fv3jedi_var.x fv3jedi_convertincrement.x fv3jedi_dirac.x fv3jedi_error_covariance_training.x \ diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 7c7740d257..6401657628 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -96,7 +96,7 @@ EOF read_increment=".false." res_latlon_dynamics="" else - increment_file=$memdir/${CDUMP}.t${cyc}z.${PREFIX_INC}atminc.nc + increment_file=$memdir/${CDUMP}.t${cyc}z.${PREFIX_ATMINC}atminc.nc if [ -f $increment_file ]; then $NLN $increment_file $DATA/INPUT/fv3_increment.nc read_increment=".true." diff --git a/ush/rocoto/setup_workflow.py b/ush/rocoto/setup_workflow.py index f87e56ea49..063d840422 100755 --- a/ush/rocoto/setup_workflow.py +++ b/ush/rocoto/setup_workflow.py @@ -54,12 +54,17 @@ def main(): wav_steps_awips = ['waveawipsbulls', 'waveawipsgridded'] # From gfsv16b latest # gfs_steps = ['prep', 'anal', 'gldas', 'fcst', 'postsnd', 'post', 'awips', 'gempak', 'vrfy', 'metp', 'arch'] - hyb_steps = ['eobs', 'ediag', 'eomg', 'eupd', 'ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc'] +# hyb_steps = ['eobs', 'ediag', 'eomg', 'eupd', 'ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc'] + hyb_steps_anal = ['eobs', 'ediag', 'eomg', 'eupd'] + hyb_steps_util = ['ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc'] gsida_steps = ['anal', 'analdiag'] jedivar_steps = ['atmanalprep', 'atmanalrun', 'sfcanl', 'atmanalpost'] + jediens_steps = ['atmensanalprep', 'atmensanalrun', 'atmensanalpost'] + steps = gfs_steps + jedivar_steps if _base.get('DO_JEDIVAR', 'NO') == 'YES' else gfs_steps + gsida_steps - steps = steps + hyb_steps if _base.get('DOHYBVAR', 'NO') == 'YES' else steps + steps = steps + jediens_steps if _base.get('DO_JEDIENS', 'NO') == 'YES' else steps + hyb_steps_anal + steps = steps + hyb_steps_util if _base.get('DOHYBVAR', 'NO') == 'YES' else steps steps = steps + metp_steps if _base.get('DO_METP', 'NO') == 'YES' else steps steps = steps + gfs_steps_gempak if _base.get('DO_GEMPAK', 'NO') == 'YES' else steps steps = steps + gfs_steps_awips if _base.get('DO_AWIPS', 'NO') == 'YES' else steps @@ -326,16 +331,24 @@ def get_hyb_resources(dict_configs): machine = base.get('machine', wfu.detectMachine()) scheduler = wfu.get_scheduler(machine) lobsdiag_forenkf = base.get('lobsdiag_forenkf', '.false.').upper() + dohybvar = base.get('DOHYBVAR', 'NO').upper() eupd_cyc= base.get('EUPD_CYC', 'gdas').upper() + do_jediens = base.get('DO_JEDIENS', 'NO').upper() reservation = base.get('RESERVATION', 'NONE').upper() dict_resources = OrderedDict() # These tasks can be run in either or both cycles if lobsdiag_forenkf in ['.T.', '.TRUE.']: - tasks1 = ['eobs', 'ediag', 'eupd', 'echgres'] + if do_jediens in ['Y', 'YES']: + tasks1 = ['atmensanalprep', 'atmensanalrun', 'atmensanalpost', 'echgres'] + else: + tasks1 = ['eobs', 'ediag', 'eupd', 'echgres'] else: - tasks1 = ['eobs', 'eomg', 'eupd', 'echgres'] + if do_jediens in ['Y', 'YES']: + tasks1 = ['atmensanalprep', 'atmensanalrun', 'atmensanalpost', 'echgres'] + else: + tasks1 = ['eobs', 'eomg', 'eupd', 'echgres'] if eupd_cyc in ['BOTH']: cdumps = ['gfs', 'gdas'] @@ -361,7 +374,7 @@ def get_hyb_resources(dict_configs): strings.append(f'\t\n') strings.append(f'\t\n') if len(memstr) != 0: - strings.appendf(f'\t\n') + strings.appendf(f'\t\n') strings.append(f'\t\n') dict_resources[f'{cdump}{task}'] = ''.join(strings) @@ -443,7 +456,7 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): task_prep = 'prep' task_anal = 'anal' task_diag = 'analdiag' - + # prep deps = [] dep_dict = {'type': 'metatask', 'name': f'{"gdas"}post', 'offset': '-06:00:00'} @@ -551,10 +564,10 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): dict_tasks[f'{cdump}sfcanl'] = task # analcalc - deps1 = [] + deps = [] data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loginc.txt' dep_dict = {'type': 'data', 'data': data} - deps1.append(rocoto.add_dependency(dep_dict)) + deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{cdump}{task_anal}'} deps.append(rocoto.add_dependency(dep_dict)) if do_jedivar in ['Y', 'YES']: @@ -1037,13 +1050,19 @@ def get_hyb_tasks(dict_configs, cycledef='enkf'): base = dict_configs['base'] nens = base['NMEM_ENKF'] lobsdiag_forenkf = base.get('lobsdiag_forenkf', '.false.').upper() + dohybvar = base.get('DOHYBVAR', 'NO').upper() eupd_cyc = base.get('EUPD_CYC', 'gdas').upper() do_jedivar = base.get('DO_JEDIVAR', 'NO').upper() + do_jediens = base.get('DO_JEDIENS', 'NO').upper() + dumpsuffix = base.get('DUMP_SUFFIX', '') + gridsuffix = base.get('SUFFIX', '') + - eobs = dict_configs['eobs'] - nens_eomg = eobs['NMEM_EOMGGRP'] - neomg_grps = nens / nens_eomg - EOMGGROUPS = ' '.join([f'{x:02d}' for x in range(1, int(neomg_grps) + 1)]) + if do_jediens in ['N', 'NO']: + eobs = dict_configs['eobs'] + nens_eomg = eobs['NMEM_EOMGGRP'] + neomg_grps = nens / nens_eomg + EOMGGROUPS = ' '.join([f'{x:02d}' for x in range(1, int(neomg_grps) + 1)]) efcs = dict_configs['efcs'] nens_efcs = efcs['NMEM_EFCSGRP'] @@ -1062,7 +1081,7 @@ def get_hyb_tasks(dict_configs, cycledef='enkf'): envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;')) envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;')) envars.append(rocoto.create_envar(name='CDATE', value='@Y@m@d@H')) - #envars.append(rocoto.create_envar(name='CDUMP', value=f'{cdump}')) +# envars.append(rocoto.create_envar(name='CDUMP', value=f'{cdump}')) envars.append(rocoto.create_envar(name='PDY', value='@Y@m@d')) envars.append(rocoto.create_envar(name='cyc', value='@H')) @@ -1077,60 +1096,105 @@ def get_hyb_tasks(dict_configs, cycledef='enkf'): elif eupd_cyc in ['GDAS']: cdumps = ['gdas'] - if do_jedivar in ['Y', 'YES']: - task_prep = ['prep', 'atmanalprep'] - else: - task_prep = ['prep'] - for cdump in cdumps: envar_cdump = rocoto.create_envar(name='CDUMP', value=f'{cdump}') envars1 = envars + [envar_cdump] - # eobs - deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}prep'} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': f'{"gdas"}epmn', 'offset': '-06:00:00'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('eobs', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef) - - dict_tasks[f'{cdump}eobs'] = task + if do_jediens in ['N', 'NO']: - # eomn, eomg - if lobsdiag_forenkf in ['.F.', '.FALSE.']: + # eobs deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}eobs'} + dep_dict = {'type': 'task', 'name': f'{cdump}prep'} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - eomgenvars= envars1 + [ensgrp] - task = wfu.create_wf_task('eomg', cdump=cdump, envar=eomgenvars, dependency=dependencies, - metatask='eomn', varname='grp', varval=EOMGGROUPS, cycledef=cycledef) + dep_dict = {'type': 'metatask', 'name': f'{"gdas"}epmn', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('eobs', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef) + + dict_tasks[f'{cdump}eobs'] = task + + # eomn, eomg + if lobsdiag_forenkf in ['.F.', '.FALSE.']: + deps = [] + dep_dict = {'type': 'task', 'name': f'{cdump}eobs'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + eomgenvars= envars1 + [ensgrp] + task = wfu.create_wf_task('eomg', cdump=cdump, envar=eomgenvars, dependency=dependencies, + metatask='eomn', varname='grp', varval=EOMGGROUPS, cycledef=cycledef) + + dict_tasks[f'{cdump}eomn'] = task + + # ediag + else: + deps = [] + dep_dict = {'type': 'task', 'name': f'{cdump}eobs'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('ediag', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef) + + dict_tasks[f'{cdump}ediag'] = task - dict_tasks[f'{cdump}eomn'] = task + # eupd + deps = [] + if lobsdiag_forenkf in ['.F.', '.FALSE.']: + dep_dict = {'type': 'metatask', 'name': f'{cdump}eomn'} + else: + dep_dict = {'type': 'task', 'name': f'{cdump}ediag'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('eupd', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef) + + dict_tasks[f'{cdump}eupd'] = task - # ediag else: + + # atmensanalprep deps = [] - dep_dict = {'type': 'task', 'name': f'{cdump}eobs'} + dep_dict = {'type': 'metatask', 'name': f'{"gdas"}post', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('ediag', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef) + data = f'&ROTDIR;/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.atmf009{gridsuffix}' + dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&DMPDIR;/{cdump}{dumpsuffix}.@Y@m@d/@H/{cdump}.t@Hz.updated.status.tm00.bufr_d' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - dict_tasks[f'{cdump}ediag'] = task + gfs_enkf = True if eupd_cyc in ['BOTH', 'GFS'] and dohybvar in ['Y', 'YES'] else False - # eupd - deps = [] - if lobsdiag_forenkf in ['.F.', '.FALSE.']: - dep_dict = {'type': 'metatask', 'name': f'{cdump}eomn'} - else: - dep_dict = {'type': 'task', 'name': f'{cdump}ediag'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('eupd', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef) + if gfs_enkf and cdump in ['gfs']: + if gfs_cyc == 4: + task = wfu.create_wf_task(task='atmensanalprep', cdump=cdump, envar=envars1, dependency=dependencies) + else: + task = wfu.create_wf_task(task='atmensanalprep', cdump=cdump, envar=envarsl, dependency=dependencies, cycledef='gdas') + + else: + task = wfu.create_wf_task(task='atmensanalprep', cdump=cdump, envar=envars1, dependency=dependencies,cycledef='gdas') - dict_tasks[f'{cdump}eupd'] = task + dict_tasks[f'{cdump}atmensanalprep'] = task + + # atmensanalrun + deps = [] + dep_dict = {'type': 'task', 'name': f'{cdump}atmensanalprep'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': f'{"gdas"}epmn', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('atmensanalrun', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef) + + dict_tasks[f'{cdump}atmensanalrun'] = task + + # atmensanalpost + deps = [] + dep_dict = {'type': 'task', 'name': f'{cdump}atmensanalrun'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + task = wfu.create_wf_task('atmensanalpost', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef) + + dict_tasks[f'{cdump}atmensanalpost'] = task + # All hybrid tasks beyond this point are always executed in the GDAS cycle cdump = 'gdas' @@ -1149,7 +1213,10 @@ def get_hyb_tasks(dict_configs, cycledef='enkf'): deps2 = [] deps2 = dependencies1 - dep_dict = {'type': 'task', 'name': f'{cdump_eupd}eupd'} + if do_jediens in ['N', 'NO']: + dep_dict = {'type': 'task', 'name': f'{cdump_eupd}eupd'} + else: + dep_dict = {'type': 'task', 'name': f'{cdump_eupd}atmensanalrun'} deps2.append(rocoto.add_dependency(dep_dict)) dependencies2 = rocoto.create_dependency(dep_condition='and', dep=deps2) @@ -1175,7 +1242,10 @@ def get_hyb_tasks(dict_configs, cycledef='enkf'): deps2 = [] deps2 = dependencies1 - dep_dict = {'type': 'task', 'name': f'{cdump_eupd}eupd'} + if do_jediens in ['N', 'NO']: + dep_dict = {'type': 'task', 'name': f'{cdump_eupd}eupd'} + else: + dep_dict = {'type': 'task', 'name': f'{cdump_eupd}atmensanalrun'} deps2.append(rocoto.add_dependency(dep_dict)) dependencies2 = rocoto.create_dependency(dep_condition='and', dep=deps2) task = wfu.create_wf_task('esfc', cdump=cdump, envar=envars1, dependency=dependencies2, cycledef=cycledef) @@ -1450,6 +1520,9 @@ def create_xml(dict_configs): 'gdasediag':'gdasediag', 'gdaseomg':'gdaseomn', 'gdaseupd':'gdaseupd', + 'gdasatmensanalprep':'gdasatmensanalprep', + 'gdasatmensanalrun':'gdasatmensanalrun', + 'gdasatmensanalpost':'gdasatmensanalpost', 'gdasecen':'gdasecmn', 'gdasesfc':'gdasesfc', 'gdasefcs':'gdasefmn', From dfbc6c752e57f7692c002bd32e3a878f1029f516 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 17 Jun 2022 17:16:19 +0000 Subject: [PATCH 31/45] Add jobs and configs for JEDI_VAR and JEDI_ENS --- jobs/rocoto/atmanalpost.sh | 2 +- jobs/rocoto/atmanalprep.sh | 2 +- jobs/rocoto/atmanalrun.sh | 2 +- jobs/rocoto/atmensanalpost.sh | 2 +- jobs/rocoto/atmensanalprep.sh | 2 +- jobs/rocoto/atmensanalrun.sh | 2 +- jobs/rocoto/postanl.sh | 2 +- jobs/rocoto/sfcanl.sh | 2 +- parm/config/config.atmanal | 4 ++- parm/config/config.atmanalrun | 1 - parm/config/config.atmensanal | 27 +++++++++++++++++ parm/config/config.atmensanalpost | 10 +++++++ parm/config/config.atmensanalprep | 10 +++++++ parm/config/config.atmensanalrun | 16 ++++++++++ parm/config/config.base.emc.dyn | 1 + parm/config/config.postanl | 46 ++++++++++++++++++++++++++++ parm/config/config.resources | 50 +++++++++++++++++++++++++++++-- parm/config/config.sfcanl | 11 +++++++ 18 files changed, 179 insertions(+), 13 deletions(-) create mode 100755 parm/config/config.atmensanal create mode 100755 parm/config/config.atmensanalpost create mode 100755 parm/config/config.atmensanalprep create mode 100755 parm/config/config.atmensanalrun create mode 100755 parm/config/config.postanl create mode 100644 parm/config/config.sfcanl diff --git a/jobs/rocoto/atmanalpost.sh b/jobs/rocoto/atmanalpost.sh index 3a1691dcf0..a13aac36fc 100755 --- a/jobs/rocoto/atmanalpost.sh +++ b/jobs/rocoto/atmanalpost.sh @@ -1,4 +1,4 @@ -#!/bin/ksh -x +#!/bin/bash -x ############################################################### # Source FV3GFS workflow modules diff --git a/jobs/rocoto/atmanalprep.sh b/jobs/rocoto/atmanalprep.sh index 6dc19fc260..aba0422a88 100755 --- a/jobs/rocoto/atmanalprep.sh +++ b/jobs/rocoto/atmanalprep.sh @@ -1,4 +1,4 @@ -#!/bin/ksh -x +#!/bin/bash -x ############################################################### # Source FV3GFS workflow modules diff --git a/jobs/rocoto/atmanalrun.sh b/jobs/rocoto/atmanalrun.sh index 070fa3468a..6c99feb200 100755 --- a/jobs/rocoto/atmanalrun.sh +++ b/jobs/rocoto/atmanalrun.sh @@ -1,4 +1,4 @@ -#!/bin/ksh -x +#!/bin/bash -x ############################################################### # Source FV3GFS workflow modules diff --git a/jobs/rocoto/atmensanalpost.sh b/jobs/rocoto/atmensanalpost.sh index 9db4e76a46..54c127efbd 100755 --- a/jobs/rocoto/atmensanalpost.sh +++ b/jobs/rocoto/atmensanalpost.sh @@ -1,4 +1,4 @@ -#!/bin/ksh -x +#!/bin/bash -x ############################################################### # Source FV3GFS workflow modules diff --git a/jobs/rocoto/atmensanalprep.sh b/jobs/rocoto/atmensanalprep.sh index acf4dd2185..44fcdbf72d 100755 --- a/jobs/rocoto/atmensanalprep.sh +++ b/jobs/rocoto/atmensanalprep.sh @@ -1,4 +1,4 @@ -#!/bin/ksh -x +#!/bin/bash -x ############################################################### # Source FV3GFS workflow modules diff --git a/jobs/rocoto/atmensanalrun.sh b/jobs/rocoto/atmensanalrun.sh index bdb0f37612..f3aa2639ee 100755 --- a/jobs/rocoto/atmensanalrun.sh +++ b/jobs/rocoto/atmensanalrun.sh @@ -1,4 +1,4 @@ -#!/bin/ksh -x +#!/bin/bash -x ############################################################### # Source FV3GFS workflow modules diff --git a/jobs/rocoto/postanl.sh b/jobs/rocoto/postanl.sh index f7c269d976..ac75d62ccc 100755 --- a/jobs/rocoto/postanl.sh +++ b/jobs/rocoto/postanl.sh @@ -1,4 +1,4 @@ -#!/bin/ksh -x +#!/bin/bash -x ############################################################### ## NCEP post driver script diff --git a/jobs/rocoto/sfcanl.sh b/jobs/rocoto/sfcanl.sh index bd89690324..457b205c43 100755 --- a/jobs/rocoto/sfcanl.sh +++ b/jobs/rocoto/sfcanl.sh @@ -1,4 +1,4 @@ -#!/bin/ksh -x +#!/bin/bash -x ############################################################### # Source FV3GFS workflow modules diff --git a/parm/config/config.atmanal b/parm/config/config.atmanal index 1b7b494871..74b29b550b 100755 --- a/parm/config/config.atmanal +++ b/parm/config/config.atmanal @@ -8,13 +8,15 @@ echo "BEGIN: config.atmanal" export OBS_YAML_DIR=$HOMEgfs/sorc/gdas.cd/parm/atm/obs/config/ export OBS_LIST=$HOMEgfs/sorc/gdas.cd/parm/atm/obs/lists/gdas_prototype.yaml export ATMVARYAML=$HOMEgfs/sorc/gdas.cd/parm/atm/variational/3dvar_dripcg.yaml -export FV3JEDI_FIX=/work2/noaa/da/cmartin/GDASApp/fix +export BERROR_YAML=$HOMEgfs/sorc/gdas.cd/parm/atm/berror/hybvar_bump.yaml +export FV3JEDI_FIX=/work2/noaa/da/rtreadon/GDASApp/fix export R2D2_OBS_DB='ufsda_test' export R2D2_OBS_DUMP='oper_gdas' export R2D2_OBS_SRC='ncdiag' export R2D2_BC_SRC='gsi' export R2D2_BC_DUMP='oper_gdas' export R2D2_ARCH_DB='local' +export INTERP_METHOD='barycentric' export io_layout_x=1 export io_layout_y=1 diff --git a/parm/config/config.atmanalrun b/parm/config/config.atmanalrun index a495e95cbe..e47938dea6 100755 --- a/parm/config/config.atmanalrun +++ b/parm/config/config.atmanalrun @@ -10,6 +10,5 @@ echo "BEGIN: config.atmanalrun" # Task specific variables export JEDIVAREXE=$HOMEgfs/exec/fv3jedi_var.x -export BERROR_YAML=$HOMEgfs/sorc/gdas.cd/parm/atm/berror/staticb_bump.yaml echo "END: config.atmanalrun" diff --git a/parm/config/config.atmensanal b/parm/config/config.atmensanal new file mode 100755 index 0000000000..eb46f1ebb4 --- /dev/null +++ b/parm/config/config.atmensanal @@ -0,0 +1,27 @@ +#!/bin/ksh -x + +########## config.atmensanal ########## +# configuration common to all atm atmensanal analysis tasks + +echo "BEGIN: config.atmensanal" + +export OBS_YAML_DIR=$HOMEgfs/sorc/gdas.cd/parm/atm/obs/config/ +export OBS_LIST=$HOMEgfs/sorc/gdas.cd/parm/atm/obs/lists/lgetkf_prototype.yaml +export BERROR_YAML=$HOMEgfs/sorc/gdas.cd/parm/atm/berror/hybvar_bump.yaml +export ATMENSYAML=$HOMEgfs/sorc/gdas.cd/parm/atm/lgetkf/lgetkf.yaml +##export FV3JEDI_FIX=/work2/noaa/da/cmartin/GDASApp/fix +##export FV3JEDI_FIX=/work2/noaa/da/fhan/GDASApp/fix +export FV3JEDI_FIX=/work2/noaa/da/rtreadon/GDASApp/fix +export R2D2_OBS_DB='ufsda_test' +export R2D2_OBS_DUMP='oper_gdas' +export R2D2_OBS_SRC='ncdiag' +export R2D2_BC_SRC='gsi' +##export R2D2_BC_DUMP='oper_gdas' +export R2D2_BC_DUMP='prgdasens' +export R2D2_ARCH_DB='local' +export INTERP_METHOD='barycentric' + +export io_layout_x=1 # hardwired to 1,1 in yamltools.py +export io_layout_y=1 + +echo "END: config.atmensanal" diff --git a/parm/config/config.atmensanalpost b/parm/config/config.atmensanalpost new file mode 100755 index 0000000000..ad48553d45 --- /dev/null +++ b/parm/config/config.atmensanalpost @@ -0,0 +1,10 @@ +#!/bin/ksh -x + +########## config.atmensanalpost ########## +# Post Atm Analysis specific + +echo "BEGIN: config.atmensanalpost" + +# Get task specific resources +. $EXPDIR/config.resources atmensanalpost +echo "END: config.atmensanalpost" diff --git a/parm/config/config.atmensanalprep b/parm/config/config.atmensanalprep new file mode 100755 index 0000000000..2e03758d06 --- /dev/null +++ b/parm/config/config.atmensanalprep @@ -0,0 +1,10 @@ +#!/bin/ksh -x + +########## config.atmensanalprep ########## +# Pre Atm Analysis specific + +echo "BEGIN: config.atmensanalprep" + +# Get task specific resources +. $EXPDIR/config.resources atmensanalprep +echo "END: config.atmensanalprep" diff --git a/parm/config/config.atmensanalrun b/parm/config/config.atmensanalrun new file mode 100755 index 0000000000..ae109e326d --- /dev/null +++ b/parm/config/config.atmensanalrun @@ -0,0 +1,16 @@ +#!/bin/ksh -x + +########## config.atmensanalrun ########## +# Atm LETKFs specific + +echo "BEGIN: config.atmensanalrun" + +# Get task specific resources +. $EXPDIR/config.resources atmensanalrun + +# Task specific variables +export JEDIENSEXE=$HOMEgfs/exec/fv3jedi_letkf.x +##export JEDIENSEXE="/work/noaa/da/Russ.Treadon/git/global_workflow/ufsda_gdasapp/sorc/gdas.cd_pr1730/build/bin/fv3jedi_letkf.x" +##export JEDIENSEXE="/work/noaa/da/Russ.Treadon/git/global_workflow/ufsda_gdasapp/sorc/gdas.cd_pr1730_test/build/bin/fv3jedi_letkf.x" + +echo "END: config.atmensanalrun" diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn index f905dd0d9b..0315313f3c 100755 --- a/parm/config/config.base.emc.dyn +++ b/parm/config/config.base.emc.dyn @@ -318,6 +318,7 @@ export imp_physics=8 # Shared parameters # DA engine export DO_JEDIVAR="NO" +export DO_JEDIENS="NO" # Hybrid related export DOHYBVAR="YES" diff --git a/parm/config/config.postanl b/parm/config/config.postanl new file mode 100755 index 0000000000..39bc5f1180 --- /dev/null +++ b/parm/config/config.postanl @@ -0,0 +1,46 @@ +#!/bin/ksh -x + +########## config.postanl ########## +# Postanl specific + +echo "BEGIN: config.postanl" + +# Get task specific resources +. $EXPDIR/config.resources postanl + +# Convert nemsio files to grib files using postanl job +#------------------------------------------- + +# No. of concurrent postanl jobs [0 implies sequential] +export NPOSTGRP=2 +export OUTTYP=4 +export MODEL_OUT_FORM=binarynemsiompiio +if [ $OUTPUT_FILE = "netcdf" ]; then + export MODEL_OUT_FORM=netcdfpara +fi + +# Post driver job that calls gfs_nceppost.sh and downstream jobs +export POSTJJOBSH="$HOMEpost/jobs/JGLOBAL_NCEPPOST" +export GFSDOWNSH="$HOMEpost/ush/fv3gfs_downstream_nems.sh" +export GFSDWNSH="$HOMEpost/ush/fv3gfs_dwn_nems.sh" + +export POSTGPSH="$HOMEpost/ush/gfs_nceppost.sh" +export POSTGPEXEC="$HOMEpost/exec/gfs_ncep_post" +export GOESF=NO # goes image +export FLXF=YES # grib2 flux file written by post + +export npe_postgp=$npe_post +export nth_postgp=1 + +export GFS_DOWNSTREAM="YES" +export downset=2 +if [ $machine = "WCOSS_DELL_P3" ]; then + export npe_dwn=28 +else + export npe_dwn=24 +fi + +export GRIBVERSION='grib2' +export SENDCOM="YES" + +echo "END: config.post" diff --git a/parm/config/config.resources b/parm/config/config.resources index 389c73aa8c..0bbef9b615 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -10,7 +10,8 @@ if [ $# -ne 1 ]; then echo "argument can be any one of the following:" echo "getic init coupled_ic aerosol_init" echo "atmanalprep atmanalrun atmanalpost" - echo "anal analcalc analdiag gldas fcst post vrfy metp arch echgres" + echo "atmensanalprep atmensanalrun atmensanalpost" + echo "anal sfcanl analcalc analdiag gldas fcst postanl post vrfy metp arch echgres" echo "eobs ediag eomg eupd ecen esfc efcs epos earc" echo "init_chem mom6ic ocnpost" echo "waveinit waveprep wavepostsbs wavepostbndpnt wavepostbndpntbll wavepostpnt" @@ -149,14 +150,15 @@ elif [ $step = "atmanalprep" -o $step = "atmanalrun" -o $step = "atmanalpost" ]; export npe_node_atmanalprep=$(echo "$npe_node_max / $nth_atmanalprep" | bc) export memory_atmanalprep="3072M" - export wtime_atmanalrun="01:00:00" + export wtime_atmanalrun="00:30:00" export npe_atmanalrun=$(echo "$layout_x * $layout_y * 6" | bc) export npe_atmanalrun_gfs=$(echo "$layout_x * $layout_y * 6" | bc) export nth_atmanalrun=1 export nth_atmanalrun_gfs=$nth_atmanalrun + export native_atmanalrun="--exclusive" export npe_node_atmanalrun=$(echo "$npe_node_max / $nth_atmanalrun" | bc) - export wtime_atmanalpost="01:00:00" + export wtime_atmanalpost="00:30:00" export npe_atmanalpost=$npe_node_max export nth_atmanalpost=1 export npe_node_atmanalpost=$(echo "$npe_node_max / $nth_atmanalpost" | bc) @@ -199,6 +201,13 @@ elif [ $step = "analdiag" ]; then export npe_node_analdiag=$npe_node_max if [[ "$machine" == "WCOSS_C" ]]; then export memory_analdiag="3072M"; fi +elif [ $step = "sfcanl" ]; then + + export wtime_sfcanl="00:10:00" + export npe_sfcanl=6 + export nth_sfcanl=1 + export npe_node_sfcanl=$(echo "$npe_node_max / $nth_sfcanl" | bc) + elif [ $step = "gldas" ]; then export wtime_gldas="00:10:00" @@ -314,6 +323,17 @@ elif [ $step = "ocnpost" ]; then export nth_ocnpost=1 export memory_ocnpost="96G" +elif [ $step = "postanl" ]; then + + export wtime_postanl="00:30:00" + export wtime_postanl_gfs="06:00:00" + export npe_postanl=112 + export nth_postanl=1 + export npe_node_postanl=40 + export npe_node_dwn=$npe_node_max + if [[ "$machine" = "WCOSS_DELL_P3" ]]; then export npe_node_postanl=28 ; fi + if [[ "$machine" == "WCOSS_C" ]]; then export memory_postanl="3072M"; fi + elif [ $step = "post" ]; then export wtime_post="02:00:00" @@ -443,6 +463,30 @@ elif [ $step = "coupled_ic" ]; then export npe_node_coupled_ic=1 export nth_coupled_ic=1 +elif [ $step = "atmensanalprep" -o $step = "atmensanalrun" -o $step = "atmensanalpost" ]; then + + # make below case dependent later + export layout_x=2 + export layout_y=3 + + export wtime_atmensanalprep="00:10:00" + export npe_atmensanalprep=1 + export nth_atmensanalprep=1 + export npe_node_atmensanalprep=$(echo "$npe_node_max / $nth_atmensanalprep" | bc) + + export wtime_atmensanalrun="00:30:00" + export npe_atmensanalrun=$(echo "$layout_x * $layout_y * 6" | bc) + export npe_atmensanalrun_gfs=$(echo "$layout_x * $layout_y * 6" | bc) + export nth_atmensanalrun=1 + export nth_atmensanalrun_gfs=$nth_atmensanalrun + export native_atmensanalrun="--exclusive" + export npe_node_atmensanalrun=$(echo "$npe_node_max / $nth_atmensanalrun" | bc) + + export wtime_atmensanalpost="00:30:00" + export npe_atmensanalpost=$npe_node_max + export nth_atmensanalpost=1 + export npe_node_atmensanalpost=$(echo "$npe_node_max / $nth_atmensanalpost" | bc) + elif [ $step = "eobs" -o $step = "eomg" ]; then export wtime_eobs="00:15:00" diff --git a/parm/config/config.sfcanl b/parm/config/config.sfcanl new file mode 100644 index 0000000000..6f92371712 --- /dev/null +++ b/parm/config/config.sfcanl @@ -0,0 +1,11 @@ +#!/bin/ksh -x + +########## config.sfcanl ########## +# GFS surface analysis specific + +echo "BEGIN: config.sfcanl" + +# Get task specific resources +. $EXPDIR/config.resources sfcanl + +echo "END: config.sfcanl" From 828e9525ff6624d1a29c737fed893cfd823652d3 Mon Sep 17 00:00:00 2001 From: "russ.treadon" Date: Tue, 21 Jun 2022 10:19:14 +0000 Subject: [PATCH 32/45] Remove jobs/JGDAS_GLOBAL_ATMOS_ENSANAL (erroneously committed) --- jobs/JGDAS_GLOBAL_ATMOS_ENSANAL | 124 -------------------------------- 1 file changed, 124 deletions(-) delete mode 100755 jobs/JGDAS_GLOBAL_ATMOS_ENSANAL diff --git a/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL b/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL deleted file mode 100755 index 593d9e7640..0000000000 --- a/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/bash -set -ex -set -u - -export PS4='$SECONDS + ' -date - - -############################# -# Source relevant config files -############################# -export EXPDIR=${EXPDIR:-$HOMEgfs/parm/config} -config_path=${EXPDIR:-$NWROOT/gfs.${gfs_ver}/parm/config} -for config in $configs; do - . $config_path/config.$config - status=$? - [[ $status -ne 0 ]] && exit $status -done - - -########################################## -# Source machine runtime environment -########################################## -. $HOMEgfs/env/${machine}.env atmensanalrun -status=$? -[[ $status -ne 0 ]] && exit $status - - -############################################## -# Obtain unique process id (pid) and make temp directory -############################################## -export pid=${pid:-$$} -export outid=${outid:-"LL$job"} - -export DATA=${DATA:-${DATAROOT}/${jobid:?}} -mkdir -p $DATA -cd $DATA || (echo "$DATA does not exist. ABORT!"; exit 1) - - -############################################## -# Run setpdy and initialize PDY variables -############################################## -export cycle="t${cyc}z" -setpdy.sh -. ./PDY - - -############################################## -# Determine Job Output Name on System -############################################## -export pgmout="OUTPUT.${pid}" -export pgmerr=errfile - - -############################################## -# Set variables used in the script -############################################## -export CDATE=${CDATE:-${PDY}${cyc}} -export CDUMP=${CDUMP:-${RUN:-"gfs"}} -export COMPONENT=${COMPONENT:-atmos} - -############################################## -# Begin JOB SPECIFIC work -############################################## - -export GDATE=$(date +%Y%m%d%H -d "${CDATE:0:8} ${CDATE:8:2} - ${assim_freq} hours") -gPDY=${GDATE:0:8} -export gcyc=${GDATE:8:2} -export GDUMP=${GDUMP:-"gdas"} - -export OPREFIX="${CDUMP}.t${cyc}z." -export GPREFIX="${GDUMP}.t${gcyc}z." -export APREFIX="${CDUMP}.t${cyc}z." -export GSUFFIX=${GSUFFIX:-$SUFFIX} -export ASUFFIX=${ASUFFIX:-$SUFFIX} - -export COMOUT=${COMOUT:-$ROTDIR/$CDUMP.$PDY/$cyc/$COMPONENT} -export COMOUT_ENS=${COMOUT_ENS:-$ROTDIR/enkf$CDUMP.$PDY/$cyc/$COMPONENT} - -mkdir -p $COMOUT -mkdir -p $COMOUT_ENS - -# COMIN_GES and COMIN_GES_ENS are used in script -export COMIN="$ROTDIR/$CDUMP.$PDY/$cyc/$COMPONENT" -export COMIN_GES="$ROTDIR/$GDUMP.$gPDY/$gcyc/$COMPONENT" -export COMIN_GES_ENS="$ROTDIR/enkfgdas.$gPDY/$gcyc/$COMPONENT" - -# NOTE BELOW IS A HACK FOR TESTING -# PLEASE FIX THIS LATER -# ASK @aerorahul -# HOW TO HANDLE DIFFERENT COMPILERS/ETC. FOR MODEL VS DA -# PROD_UTIL, ETC. DO NOT EXIST FOR JEDI MODULE VERSIONS -module purge -##module use $HOMEgfs/sorc/gdas.cd/modulefiles -module use $HOMEgfs/sorc/gdas.cd_pr1730/modulefiles -module load GDAS/orion -export PYTHONPATH=$HOMEgfs/sorc/gdas.cd/ush/:$PYTHONPATH - -############################################################### -# Run relevant script - -$EXSCRIPT -status=$? -[[ $status -ne 0 ]] && exit $status - -############################################## -# End JOB SPECIFIC work -############################################## - -############################################## -# Final processing -############################################## -if [ -e "$pgmout" ] ; then - cat $pgmout -fi - -########################################## -# Remove the Temporary working directory -########################################## -cd $DATAROOT -[[ $KEEPDATA = "NO" ]] && rm -rf $DATA - -date -exit 0 From b448260eb99eb9742e2ffbe4cfa6a5ae3e620ad6 Mon Sep 17 00:00:00 2001 From: "russ.treadon" Date: Tue, 21 Jun 2022 10:41:13 +0000 Subject: [PATCH 33/45] Add external jobs/JGDAS_GLOBAL_ATMOS_ENSANAL to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 7818e1a737..23ddc8b287 100644 --- a/.gitignore +++ b/.gitignore @@ -141,6 +141,7 @@ jobs/JGDAS_ENKF_SELECT_OBS jobs/JGDAS_ENKF_SFC jobs/JGDAS_ENKF_UPDATE jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS +jobs/JGDAS_GLOBAL_ATMOS_ENSANAL jobs/JGFS_ATMOS_VMINMON jobs/JGFS_ATMOS_WAFS jobs/JGFS_ATMOS_WAFS_BLENDING From 444f6a9706fc6480c613eb168c83886d3ec21bb6 Mon Sep 17 00:00:00 2001 From: "russ.treadon" Date: Fri, 24 Jun 2022 16:05:16 +0000 Subject: [PATCH 34/45] Update sfcanl in env and setup_workflow.py --- env/HERA.env | 49 ++++++++++++++++++++++++++++-------- env/JET.env | 29 ++++++++++++++++++--- env/ORION.env | 3 +-- ush/rocoto/setup_workflow.py | 41 +++++++++++++----------------- 4 files changed, 83 insertions(+), 39 deletions(-) diff --git a/env/HERA.env b/env/HERA.env index f63b24be65..8c17a70892 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -4,7 +4,8 @@ if [ $# -ne 1 ]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "anal fcst post vrfy metp" + echo "atmanalrun atmensanalrun" + echo "anal sfcanl fcst post vrfy metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" exit 1 @@ -50,6 +51,30 @@ elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $ste export wavempexec=${launcher} export wave_mpmd=${mpmd} +elif [ $step = "atmanalrun" ]; then + + export CFP_MP=${CFP_MP:-"YES"} + export USE_CFP=${USE_CFP:-"YES"} + export APRUNCFP="$launcher -n \$ncmd --multi-prog" + + nth_max=$(($npe_node_max / $npe_node_atmanalrun)) + + export NTHREADS_ATMANAL=${nth_atmanalrun:-$nth_max} + [[ $NTHREADS_ATMANAL -gt $nth_max ]] && export NTHREADS_ATMANAL=$nth_max + export APRUN_ATMANAL="$launcher -n $npe_atmanalrun" + +elif [ $step = "atmensanalrun" ]; then + + export CFP_MP=${CFP_MP:-"YES"} + export USE_CFP=${USE_CFP:-"YES"} + export APRUNCFP="$launcher -n \$ncmd --multi-prog" + + nth_max=$(($npe_node_max / $npe_node_atmensanalrun)) + + export NTHREADS_ATMENSANAL=${nth_atmensanalrun:-$nth_max} + [[ $NTHREADS_ATMENSANAL -gt $nth_max ]] && export NTHREADS_ATMENSANAL=$nth_max + export APRUN_ATMENSANAL="$launcher -n $npe_atmensanalrun" + elif [ $step = "anal" ]; then export MKL_NUM_THREADS=4 @@ -79,9 +104,13 @@ elif [ $step = "anal" ]; then npe_gausfcanl=${npe_gausfcanl:-1} export APRUN_GAUSFCANL="$launcher -n $npe_gausfcanl" - export NTHREADS_CHGRES=${nth_echgres:-1} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="" +elif [ $step = "sfcanl" ]; then + nth_max=$(($npe_node_max / $npe_node_sfcanl)) + + export NTHREADS_CYCLE=${nth_sfcanl:-14} + [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max + npe_sfcanl=${ntiles:-6} + export APRUN_CYCLE="$launcher -n $npe_sfcanl" elif [ $step = "gldas" ]; then @@ -106,28 +135,28 @@ elif [ $step = "eobs" ]; then export MKL_NUM_THREADS=4 export MKL_CBWR=AUTO + export CFP_MP=${CFP_MP:-"YES"} + export USE_CFP=${USE_CFP:-"YES"} + export APRUNCFP="$launcher -n \$ncmd --multi-prog" + nth_max=$(($npe_node_max / $npe_node_eobs)) export NTHREADS_GSI=${nth_eobs:-$nth_max} [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max export APRUN_GSI="$launcher" +elif [ $step = "eupd" ]; then + export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} export APRUNCFP="$launcher -n \$ncmd --multi-prog" -elif [ $step = "eupd" ]; then - nth_max=$(($npe_node_max / $npe_node_eupd)) export NTHREADS_ENKF=${nth_eupd:-$nth_max} [[ $NTHREADS_ENKF -gt $nth_max ]] && export NTHREADS_ENKF=$nth_max export APRUN_ENKF="$launcher" - export CFP_MP=${CFP_MP:-"YES"} - export USE_CFP=${USE_CFP:-"YES"} - export APRUNCFP="$launcher -n \$ncmd --multi-prog" - elif [ $step = "fcst" ]; then #PEs and PEs/node can differ for GFS and GDAS forecasts if threading differs diff --git a/env/JET.env b/env/JET.env index 8306e423c3..21321b23c9 100755 --- a/env/JET.env +++ b/env/JET.env @@ -4,7 +4,8 @@ if [ $# -ne 1 ]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "anal fcst post vrfy metp" + echo "atmanalrun atmensanalrun" + echo "anal sfcanl fcst post vrfy metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" exit 1 @@ -44,6 +45,22 @@ elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $ste export wavempexec=${launcher} export wave_mpmd=${mpmd} +elif [ $step = "atmanalrun" ]; then + + nth_max=$(($npe_node_max / $npe_node_atmanalrun)) + + export NTHREADS_ATMANAL=${nth_atmanalrun:-$nth_max} + [[ $NTHREADS_ATMANAL -gt $nth_max ]] && export NTHREADS_ATMANAL=$nth_max + export APRUN_ATMANAL="$launcher $npe_atmanalrun" + +elif [ $step = "atmensanalrun" ]; then + + nth_max=$(($npe_node_max / $npe_node_atmensanalrun)) + + export NTHREADS_ATMENSANAL=${nth_atmensanalrun:-$nth_max} + [[ $NTHREADS_ATMENSANAL -gt $nth_max ]] && export NTHREADS_ATMENSANAL=$nth_max + export APRUN_ATMENSANAL="$launcher $npe_atmensanalrun" + elif [ $step = "anal" ]; then nth_max=$(($npe_node_max / $npe_node_anal)) @@ -65,9 +82,13 @@ elif [ $step = "anal" ]; then npe_gausfcanl=${npe_gausfcanl:-1} export APRUN_GAUSFCANL="$launcher $npe_gausfcanl" - export NTHREADS_CHGRES=${nth_echgres:-1} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="" +elif [ $step = "sfcanl" ]; then + nth_max=$(($npe_node_max / $npe_node_sfcanl)) + + export NTHREADS_CYCLE=${nth_sfcanl:-14} + [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max + npe_sfcanl=${ntiles:-6} + export APRUN_CYCLE="$launcher $npe_sfcanl" elif [ $step = "gldas" ]; then diff --git a/env/ORION.env b/env/ORION.env index b272e69a41..9e4953d4e9 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -42,7 +42,7 @@ if [ $step = "prep" -o $step = "prepbufr" ]; then export BACK=${BACK:-"YES"} export sys_tp="ORION" -elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $step = "wavepostbndpnt" -o $step = "wavepostpnt" ]; then +elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $step = "wavepostbndpnt" -o $step = "wavepostbndpntbll" -o $step = "wavepostpnt" ]; then export mpmd="--multi-prog" export CFP_MP="YES" @@ -100,7 +100,6 @@ elif [ $step = "anal" ]; then export NTHREADS_GAUSFCANL=1 npe_gausfcanl=${npe_gausfcanl:-1} - export APRUN_GAUSFCANL="$launcher -n $npe_gausfcanl" elif [ $step = "sfcanl" ]; then diff --git a/ush/rocoto/setup_workflow.py b/ush/rocoto/setup_workflow.py index 28dce1bd74..a25a3cd792 100755 --- a/ush/rocoto/setup_workflow.py +++ b/ush/rocoto/setup_workflow.py @@ -42,7 +42,7 @@ def main(): print(f'input arg: --expdir = {repr(args.expdir)}') sys.exit(1) - gfs_steps = ['prep', 'analcalc', 'gldas', 'fcst', 'postsnd', 'postanl', 'post', 'vrfy', 'arch'] + gfs_steps = ['prep', 'sfcanl', 'analcalc', 'gldas', 'fcst', 'postsnd', 'postanl', 'post', 'vrfy', 'arch'] gfs_steps_gempak = ['gempak'] gfs_steps_awips = ['awips'] gfs_steps_wafs = ['wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25'] @@ -58,7 +58,7 @@ def main(): hyb_steps_anal = ['eobs', 'ediag', 'eomg', 'eupd'] hyb_steps_util = ['ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc'] gsida_steps = ['anal', 'analdiag'] - jedivar_steps = ['atmanalprep', 'atmanalrun', 'sfcanl', 'atmanalpost'] + jedivar_steps = ['atmanalprep', 'atmanalrun', 'atmanalpost'] jediens_steps = ['atmensanalprep', 'atmensanalrun', 'atmensanalpost'] @@ -262,11 +262,10 @@ def get_gdasgfs_resources(dict_configs, cdump='gdas'): task_diag = ['analdiag'] tasks = task_prep + task_anal + tasks += ['sfcanl'] tasks += ['analcalc'] if cdump in ['gdas']: tasks += task_diag - if do_jedivar in ['Y', 'YES']: - tasks += ['sfcanl'] if cdump in ['gdas'] and do_gldas in ['Y', 'YES']: tasks += ['gldas'] if cdump in ['gdas'] and do_wave in ['Y', 'YES'] and do_wave_cdump in ['GDAS', 'BOTH']: @@ -551,17 +550,16 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): dict_tasks[f'{cdump}{task_anal}'] = task # sfcanl - if do_jedivar in ['Y', 'YES']: - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loginc.txt' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}{task_anal}'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('sfcanl', cdump=cdump, envar=envars, dependency=dependencies) + deps = [] + data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loginc.txt' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{cdump}{task_anal}'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + task = wfu.create_wf_task('sfcanl', cdump=cdump, envar=envars, dependency=dependencies) - dict_tasks[f'{cdump}sfcanl'] = task + dict_tasks[f'{cdump}sfcanl'] = task # analcalc deps = [] @@ -570,9 +568,8 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{cdump}{task_anal}'} deps.append(rocoto.add_dependency(dep_dict)) - if do_jedivar in ['Y', 'YES']: - dep_dict = {'type': 'task', 'name': f'{cdump}sfcanl'} - deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{cdump}sfcanl'} + deps.append(rocoto.add_dependency(dep_dict)) if dohybvar in ['y', 'Y', 'yes', 'YES'] and cdump == 'gdas': dep_dict = {'type': 'task', 'name': f'{"gdas"}echgres', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) @@ -611,9 +608,8 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): deps1.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{cdump}{task_anal}'} deps1.append(rocoto.add_dependency(dep_dict)) - if do_jedivar in ['Y', 'YES']: - dep_dict = {'type': 'task', 'name': f'{cdump}sfcanl'} - deps1.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{cdump}sfcanl'} + deps1.append(rocoto.add_dependency(dep_dict)) dependencies1 = rocoto.create_dependency(dep_condition='and', dep=deps1) deps2 = [] @@ -639,9 +635,8 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): else: dep_dict = {'type': 'task', 'name': f'{cdump}analcalc'} deps1.append(rocoto.add_dependency(dep_dict)) - if do_jedivar in ['Y', 'YES']: - dep_dict = {'type': 'task', 'name': f'{cdump}sfcanl'} - deps1.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{cdump}sfcanl'} + deps1.append(rocoto.add_dependency(dep_dict)) elif cdump in ['gfs']: dep_dict = {'type': 'task', 'name': f'{cdump}{task_anal}'} deps1.append(rocoto.add_dependency(dep_dict)) From eab4bc47d689b187e52663d47bb0b8291ff2038d Mon Sep 17 00:00:00 2001 From: "russ.treadon" Date: Wed, 29 Jun 2022 10:22:16 +0000 Subject: [PATCH 35/45] Incoporate review feedback for JEDI, postanl, and sfcanl --- jobs/JGLOBAL_ATMOS_SFCANL | 36 +------------- parm/config/config.atmanal | 4 +- parm/config/config.atmanalpost | 2 +- parm/config/config.atmanalprep | 2 +- parm/config/config.atmanalrun | 2 +- parm/config/config.atmensanal | 6 +-- parm/config/config.atmensanalpost | 2 +- parm/config/config.atmensanalprep | 2 +- parm/config/config.atmensanalrun | 4 +- parm/config/config.postanl | 8 ---- parm/config/config.resources | 30 +++++++++--- parm/config/config.sfcanl | 2 +- scripts/exglobal_atmos_sfcanl.sh | 78 ++----------------------------- sorc/build_all.sh | 16 +++---- sorc/build_gdas.sh | 9 +--- sorc/checkout.sh | 2 +- sorc/link_workflow.sh | 6 ++- 17 files changed, 54 insertions(+), 157 deletions(-) diff --git a/jobs/JGLOBAL_ATMOS_SFCANL b/jobs/JGLOBAL_ATMOS_SFCANL index c3b2ab26fe..5175b46ccd 100755 --- a/jobs/JGLOBAL_ATMOS_SFCANL +++ b/jobs/JGLOBAL_ATMOS_SFCANL @@ -62,7 +62,7 @@ export COMPONENT=${COMPONENT:-atmos} if [ $RUN_ENVIR = "nco" ]; then export ROTDIR=${COMROOT:?}/$NET/$envir fi -export DO_CALC_ANALYSIS=${DO_CALC_ANALYSIS:-"YES"} +##export DO_CALC_ANALYSIS=${DO_CALC_ANALYSIS:-"YES"} ############################################## @@ -104,38 +104,6 @@ if [ ! -f $ATMGES ]; then fi -# Get LEVS -if [ ${GSUFFIX} = ".nc" ]; then - export LEVS=$($NCLEN $ATMGES pfull) - status=$? -else - export LEVS=$($NEMSIOGET $ATMGES dimz | awk '{print $2}') - status=$? -fi -[[ $status -ne 0 ]] && exit $status - - -##if [ $DOHYBVAR = "YES" ]; then -## export ATMGES_ENSMEAN="$COMIN_GES_ENS/${GPREFIX}atmf006.ensmean$GSUFFIX" -## if [ ! -f $ATMGES_ENSMEAN ]; then -## echo "FATAL ERROR: FILE MISSING: ATMGES_ENSMEAN = $ATMGES_ENSMEAN" -## exit 2 -## fi -##fi - - -# Link observational data -export PREPQC="${COMOUT}/${OPREFIX}prepbufr" -if [ ! -f $PREPQC ]; then - echo "WARNING: PREPBUFR FILE $PREPQC MISSING" - msg="WARNING : Global PREPBUFR file is missing" - postmsg "$jlogfile" "$msg" -fi -export PREPQCPF="${COMOUT}/${OPREFIX}prepbufr.acft_profiles" -export TCVITL="${COMOUT}/${OPREFIX}syndata.tcvitals.tm00" -[[ $DONST = "YES" ]] && export NSSTBF="${COMOUT}/${OPREFIX}nsstbufr" - - # Update surface fields with global_cycle export DOGCYCLE=${DOGCYCLE:-"YES"} @@ -143,7 +111,7 @@ export DOGCYCLE=${DOGCYCLE:-"YES"} ############################################################### # Run relevant script env -msg="HAS BEGUN on `hostname`" +msg="HAS BEGUN on $(hostname)" postmsg "$jlogfile" "$msg" $LOGSCRIPT diff --git a/parm/config/config.atmanal b/parm/config/config.atmanal index 74b29b550b..9c11e9e593 100755 --- a/parm/config/config.atmanal +++ b/parm/config/config.atmanal @@ -1,4 +1,4 @@ -#!/bin/ksh -x +#!/bin/bash -x ########## config.atmanal ########## # configuration common to all atm analysis tasks @@ -9,7 +9,7 @@ export OBS_YAML_DIR=$HOMEgfs/sorc/gdas.cd/parm/atm/obs/config/ export OBS_LIST=$HOMEgfs/sorc/gdas.cd/parm/atm/obs/lists/gdas_prototype.yaml export ATMVARYAML=$HOMEgfs/sorc/gdas.cd/parm/atm/variational/3dvar_dripcg.yaml export BERROR_YAML=$HOMEgfs/sorc/gdas.cd/parm/atm/berror/hybvar_bump.yaml -export FV3JEDI_FIX=/work2/noaa/da/rtreadon/GDASApp/fix +export FV3JEDI_FIX=$HOMEgfs/fix/fix_jedi export R2D2_OBS_DB='ufsda_test' export R2D2_OBS_DUMP='oper_gdas' export R2D2_OBS_SRC='ncdiag' diff --git a/parm/config/config.atmanalpost b/parm/config/config.atmanalpost index e5210b9bf4..fd5f3bbbcc 100755 --- a/parm/config/config.atmanalpost +++ b/parm/config/config.atmanalpost @@ -1,4 +1,4 @@ -#!/bin/ksh -x +#!/bin/bash -x ########## config.atmanalpost ########## # Post Atm Analysis specific diff --git a/parm/config/config.atmanalprep b/parm/config/config.atmanalprep index 8764183a51..0014520f5f 100755 --- a/parm/config/config.atmanalprep +++ b/parm/config/config.atmanalprep @@ -1,4 +1,4 @@ -#!/bin/ksh -x +#!/bin/bash -x ########## config.atmanalprep ########## # Pre Atm Analysis specific diff --git a/parm/config/config.atmanalrun b/parm/config/config.atmanalrun index e47938dea6..5aaac6a01d 100755 --- a/parm/config/config.atmanalrun +++ b/parm/config/config.atmanalrun @@ -1,4 +1,4 @@ -#!/bin/ksh -x +#!/bin/bash -x ########## config.atmanalrun ########## # Atm Analysis specific diff --git a/parm/config/config.atmensanal b/parm/config/config.atmensanal index eb46f1ebb4..4f8e244b2e 100755 --- a/parm/config/config.atmensanal +++ b/parm/config/config.atmensanal @@ -1,4 +1,4 @@ -#!/bin/ksh -x +#!/bin/bash -x ########## config.atmensanal ########## # configuration common to all atm atmensanal analysis tasks @@ -9,9 +9,7 @@ export OBS_YAML_DIR=$HOMEgfs/sorc/gdas.cd/parm/atm/obs/config/ export OBS_LIST=$HOMEgfs/sorc/gdas.cd/parm/atm/obs/lists/lgetkf_prototype.yaml export BERROR_YAML=$HOMEgfs/sorc/gdas.cd/parm/atm/berror/hybvar_bump.yaml export ATMENSYAML=$HOMEgfs/sorc/gdas.cd/parm/atm/lgetkf/lgetkf.yaml -##export FV3JEDI_FIX=/work2/noaa/da/cmartin/GDASApp/fix -##export FV3JEDI_FIX=/work2/noaa/da/fhan/GDASApp/fix -export FV3JEDI_FIX=/work2/noaa/da/rtreadon/GDASApp/fix +export FV3JEDI_FIX=$HOMEgfs/fix/fix_jedi export R2D2_OBS_DB='ufsda_test' export R2D2_OBS_DUMP='oper_gdas' export R2D2_OBS_SRC='ncdiag' diff --git a/parm/config/config.atmensanalpost b/parm/config/config.atmensanalpost index ad48553d45..f79ee5b507 100755 --- a/parm/config/config.atmensanalpost +++ b/parm/config/config.atmensanalpost @@ -1,4 +1,4 @@ -#!/bin/ksh -x +#!/bin/bash -x ########## config.atmensanalpost ########## # Post Atm Analysis specific diff --git a/parm/config/config.atmensanalprep b/parm/config/config.atmensanalprep index 2e03758d06..b719b9ac6c 100755 --- a/parm/config/config.atmensanalprep +++ b/parm/config/config.atmensanalprep @@ -1,4 +1,4 @@ -#!/bin/ksh -x +#!/bin/bash -x ########## config.atmensanalprep ########## # Pre Atm Analysis specific diff --git a/parm/config/config.atmensanalrun b/parm/config/config.atmensanalrun index ae109e326d..aeb59d1805 100755 --- a/parm/config/config.atmensanalrun +++ b/parm/config/config.atmensanalrun @@ -1,4 +1,4 @@ -#!/bin/ksh -x +#!/bin/bash -x ########## config.atmensanalrun ########## # Atm LETKFs specific @@ -10,7 +10,5 @@ echo "BEGIN: config.atmensanalrun" # Task specific variables export JEDIENSEXE=$HOMEgfs/exec/fv3jedi_letkf.x -##export JEDIENSEXE="/work/noaa/da/Russ.Treadon/git/global_workflow/ufsda_gdasapp/sorc/gdas.cd_pr1730/build/bin/fv3jedi_letkf.x" -##export JEDIENSEXE="/work/noaa/da/Russ.Treadon/git/global_workflow/ufsda_gdasapp/sorc/gdas.cd_pr1730_test/build/bin/fv3jedi_letkf.x" echo "END: config.atmensanalrun" diff --git a/parm/config/config.postanl b/parm/config/config.postanl index 39bc5f1180..771311c2f6 100755 --- a/parm/config/config.postanl +++ b/parm/config/config.postanl @@ -29,16 +29,8 @@ export POSTGPEXEC="$HOMEpost/exec/gfs_ncep_post" export GOESF=NO # goes image export FLXF=YES # grib2 flux file written by post -export npe_postgp=$npe_post -export nth_postgp=1 - export GFS_DOWNSTREAM="YES" export downset=2 -if [ $machine = "WCOSS_DELL_P3" ]; then - export npe_dwn=28 -else - export npe_dwn=24 -fi export GRIBVERSION='grib2' export SENDCOM="YES" diff --git a/parm/config/config.resources b/parm/config/config.resources index 0bbef9b615..d4c52d52fd 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -138,11 +138,7 @@ elif [ $step = "waveawipsgridded" ]; then export npe_node_waveawipsgridded=$(echo "$npe_node_max / $nth_waveawipsgridded" | bc) export NTASKS=${npe_waveawipsgridded} -elif [ $step = "atmanalprep" -o $step = "atmanalrun" -o $step = "atmanalpost" ]; then - - # make below case dependent later - export layout_x=1 - export layout_y=1 +elif [ $step = "atmanalprep" ]; then export wtime_atmanalprep="00:10:00" export npe_atmanalprep=1 @@ -150,6 +146,12 @@ elif [ $step = "atmanalprep" -o $step = "atmanalrun" -o $step = "atmanalpost" ]; export npe_node_atmanalprep=$(echo "$npe_node_max / $nth_atmanalprep" | bc) export memory_atmanalprep="3072M" +elif [ $step = "atmanalrun" ]; then + + # make below case dependent later + export layout_x=1 + export layout_y=1 + export wtime_atmanalrun="00:30:00" export npe_atmanalrun=$(echo "$layout_x * $layout_y * 6" | bc) export npe_atmanalrun_gfs=$(echo "$layout_x * $layout_y * 6" | bc) @@ -158,6 +160,8 @@ elif [ $step = "atmanalprep" -o $step = "atmanalrun" -o $step = "atmanalpost" ]; export native_atmanalrun="--exclusive" export npe_node_atmanalrun=$(echo "$npe_node_max / $nth_atmanalrun" | bc) +elif [ $step = "atmanalpost" ]; then + export wtime_atmanalpost="00:30:00" export npe_atmanalpost=$npe_node_max export nth_atmanalpost=1 @@ -330,8 +334,12 @@ elif [ $step = "postanl" ]; then export npe_postanl=112 export nth_postanl=1 export npe_node_postanl=40 + export npe_dwn=24 export npe_node_dwn=$npe_node_max - if [[ "$machine" = "WCOSS_DELL_P3" ]]; then export npe_node_postanl=28 ; fi + if [[ "$machine" = "WCOSS_DELL_P3" ]]; then + export npe_dwn=28 + export npe_node_postanl=28 + fi if [[ "$machine" == "WCOSS_C" ]]; then export memory_postanl="3072M"; fi elif [ $step = "post" ]; then @@ -463,7 +471,7 @@ elif [ $step = "coupled_ic" ]; then export npe_node_coupled_ic=1 export nth_coupled_ic=1 -elif [ $step = "atmensanalprep" -o $step = "atmensanalrun" -o $step = "atmensanalpost" ]; then +elif [ $step = "atmensanalprep" ]; then # make below case dependent later export layout_x=2 @@ -474,6 +482,12 @@ elif [ $step = "atmensanalprep" -o $step = "atmensanalrun" -o $step = "atmensana export nth_atmensanalprep=1 export npe_node_atmensanalprep=$(echo "$npe_node_max / $nth_atmensanalprep" | bc) +elif [ $step = "atmensanalrun" ]; then + + # make below case dependent later + export layout_x=2 + export layout_y=3 + export wtime_atmensanalrun="00:30:00" export npe_atmensanalrun=$(echo "$layout_x * $layout_y * 6" | bc) export npe_atmensanalrun_gfs=$(echo "$layout_x * $layout_y * 6" | bc) @@ -482,6 +496,8 @@ elif [ $step = "atmensanalprep" -o $step = "atmensanalrun" -o $step = "atmensana export native_atmensanalrun="--exclusive" export npe_node_atmensanalrun=$(echo "$npe_node_max / $nth_atmensanalrun" | bc) +elif [ $step = "atmensanalpost" ]; then + export wtime_atmensanalpost="00:30:00" export npe_atmensanalpost=$npe_node_max export nth_atmensanalpost=1 diff --git a/parm/config/config.sfcanl b/parm/config/config.sfcanl index 6f92371712..76da647eef 100644 --- a/parm/config/config.sfcanl +++ b/parm/config/config.sfcanl @@ -1,4 +1,4 @@ -#!/bin/ksh -x +#!/bin/bash -x ########## config.sfcanl ########## # GFS surface analysis specific diff --git a/scripts/exglobal_atmos_sfcanl.sh b/scripts/exglobal_atmos_sfcanl.sh index 8267f949e7..ca9cae9b3e 100755 --- a/scripts/exglobal_atmos_sfcanl.sh +++ b/scripts/exglobal_atmos_sfcanl.sh @@ -106,53 +106,10 @@ OSUFFIX=${OSUFFIX:-""} # Guess files GPREFIX=${GPREFIX:-""} -GSUFFIX=${GSUFFIX:-$SUFFIX} -ATMG03=${ATMG03:-${COMIN_GES}/${GPREFIX}atmf003${GSUFFIX}} -ATMG04=${ATMG04:-${COMIN_GES}/${GPREFIX}atmf004${GSUFFIX}} -ATMG05=${ATMG05:-${COMIN_GES}/${GPREFIX}atmf005${GSUFFIX}} -ATMGES=${ATMGES:-${COMIN_GES}/${GPREFIX}atmf006${GSUFFIX}} -ATMG07=${ATMG07:-${COMIN_GES}/${GPREFIX}atmf007${GSUFFIX}} -ATMG08=${ATMG08:-${COMIN_GES}/${GPREFIX}atmf008${GSUFFIX}} -ATMG09=${ATMG09:-${COMIN_GES}/${GPREFIX}atmf009${GSUFFIX}} - -SFCG03=${SFCG03:-${COMIN_GES}/${GPREFIX}sfcf003${GSUFFIX}} -SFCG04=${SFCG04:-${COMIN_GES}/${GPREFIX}sfcf004${GSUFFIX}} -SFCG05=${SFCG05:-${COMIN_GES}/${GPREFIX}sfcf005${GSUFFIX}} -SFCGES=${SFCGES:-${COMIN_GES}/${GPREFIX}sfcf006${GSUFFIX}} -SFCG07=${SFCG07:-${COMIN_GES}/${GPREFIX}sfcf007${GSUFFIX}} -SFCG08=${SFCG08:-${COMIN_GES}/${GPREFIX}sfcf008${GSUFFIX}} -SFCG09=${SFCG09:-${COMIN_GES}/${GPREFIX}sfcf009${GSUFFIX}} # Analysis files export APREFIX=${APREFIX:-""} -export ASUFFIX=${ASUFFIX:-$SUFFIX} -SFCANL=${SFCANL:-${COMOUT}/${APREFIX}sfcanl${ASUFFIX}} DTFANL=${DTFANL:-${COMOUT}/${APREFIX}dtfanl.nc} -ATMANL=${ATMANL:-${COMOUT}/${APREFIX}atmanl${ASUFFIX}} - -# Increment files -ATMINC=${ATMINC:-${COMOUT}/${APREFIX}atminc.nc} - -# Set script parameters -export DOIAU=${DOIAU:-"NO"} -DO_CALC_INCREMENT=${DO_CALC_INCREMENT:-"NO"} -DO_CALC_ANALYSIS=${DO_CALC_ANALYSIS:-"NO"} - -# Get header information from Guess files -if [ ${SUFFIX} = ".nc" ]; then - LONB=${LONB:-$($NCLEN $ATMGES grid_xt)} # get LONB - LATB=${LATB:-$($NCLEN $ATMGES grid_yt)} # get LATB - LEVS=${LEVS:-$($NCLEN $ATMGES pfull)} # get LEVS - JCAP=${JCAP:--9999} # there is no jcap in these files -else - LONB=${LONB:-$($NEMSIOGET $ATMGES dimx | grep -i "dimx" | awk -F"= " '{print $2}' | awk -F" " '{print $1}')} # 'get LONB - LATB=${LATB:-$($NEMSIOGET $ATMGES dimy | grep -i "dimy" | awk -F"= " '{print $2}' | awk -F" " '{print $1}')} # 'get LATB - LEVS=${LEVS:-$($NEMSIOGET $ATMGES dimz | grep -i "dimz" | awk -F"= " '{print $2}' | awk -F" " '{print $1}')} # 'get LEVS - JCAP=${JCAP:-$($NEMSIOGET $ATMGES jcap | grep -i "jcap" | awk -F"= " '{print $2}' | awk -F" " '{print $1}')} # 'get JCAP -fi -[ $JCAP -eq -9999 -a $LATB -ne -9999 ] && JCAP=$((LATB-2)) -[ $LONB -eq -9999 -o $LATB -eq -9999 -o $LEVS -eq -9999 -o $JCAP -eq -9999 ] && exit -9999 - # Get dimension information based on CASE res=$(echo $CASE | cut -c2-) @@ -160,28 +117,6 @@ JCAP_CASE=$((res*2-2)) LATB_CASE=$((res*2)) LONB_CASE=$((res*4)) -# logic for netCDF I/O -if [ ${SUFFIX} = ".nc" ]; then - # GSI namelist options to use netCDF background - use_gfs_nemsio=".false." - use_gfs_ncio=".true." -else - # GSI namelist options to use NEMSIO background - use_gfs_nemsio=".true." - use_gfs_ncio=".false." -fi - -# determine if writing or calculating increment -if [ $DO_CALC_INCREMENT = "YES" ]; then - write_fv3_increment=".false." -else - write_fv3_increment=".true." - WRITE_INCR_ZERO="incvars_to_zero= $INCREMENTS_TO_ZERO," - WRITE_ZERO_STRAT="incvars_zero_strat= $INCVARS_ZERO_STRAT," - WRITE_STRAT_EFOLD="incvars_efold= $INCVARS_EFOLD," -fi - - ################################################################################ # Preprocessing mkdata=NO @@ -192,19 +127,14 @@ fi cd $DATA || exit 99 -[[ $DONST = "YES" ]] && $NLN $NSSTBF nsstbufr +if [ $DONST = "YES" ]; then + export NSSTBF="${COMOUT}/${OPREFIX}nsstbufr" + $NLN $NSSTBF nsstbufr +fi ############################################################## # Required model guess files -$NLN $SFCG03 sfcf03 -$NLN $SFCGES sfcf06 -$NLN $SFCG09 sfcf09 - -[[ -f $SFCG04 ]] && $NLN $SFCG04 sfcf04 -[[ -f $SFCG05 ]] && $NLN $SFCG05 sfcf05 -[[ -f $SFCG07 ]] && $NLN $SFCG07 sfcf07 -[[ -f $SFCG08 ]] && $NLN $SFCG08 sfcf08 ############################################################## diff --git a/sorc/build_all.sh b/sorc/build_all.sh index 52d14ce458..964fee3b74 100755 --- a/sorc/build_all.sh +++ b/sorc/build_all.sh @@ -136,14 +136,14 @@ $Build_gsi && { # build gdas #------------------------------------ $Build_gdas && { -echo " .... Building GDASApp .... " -./build_gdas.sh > $logs_dir/build_gdas.log 2>&1 -rc=$? -if [[ $rc -ne 0 ]] ; then - echo "Fatal error in building GDAS." - echo "The log file is in $logs_dir/build_gdas.log" -fi -((err+=$rc)) + echo " .... Building GDASApp .... " + ./build_gdas.sh > $logs_dir/build_gdas.log 2>&1 + rc=$? + if [[ $rc -ne 0 ]] ; then + echo "Fatal error in building GDAS." + echo "The log file is in $logs_dir/build_gdas.log" + fi + ((err+=$rc)) } #------------------------------------ diff --git a/sorc/build_gdas.sh b/sorc/build_gdas.sh index c32df5ebaf..f9238c9ab0 100755 --- a/sorc/build_gdas.sh +++ b/sorc/build_gdas.sh @@ -2,14 +2,7 @@ set -eux source ./machine-setup.sh > /dev/null 2>&1 -cwd=`pwd` - -USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} -if [ $USE_PREINST_LIBS = true ]; then - export MOD_PATH=/scratch3/NCEPDEV/nwprod/lib/modulefiles -else - export MOD_PATH=${cwd}/lib/modulefiles -fi +cwd=$(pwd) export BUILD_TARGET=$target diff --git a/sorc/checkout.sh b/sorc/checkout.sh index 5219b087b6..e83ad2b68d 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -131,7 +131,7 @@ mkdir -p ${logdir} # The checkout version should always be a speciifc commit (hash or tag), not a branch errs=0 checkout "ufs_model.fd" "https://github.com/ufs-community/ufs-weather-model" "${ufs_model_hash:-Prototype-P8c}"; errs=$((errs + $?)) -checkout "gsi.fd" "https://github.com/NOAA-EMC/GSI.git" "develop" ; errs=$((errs + $?)) +checkout "gsi.fd" "https://github.com/NOAA-EMC/GSI.git" "c10644e" ; errs=$((errs + $?)) checkout "gdas.cd" "https://github.com/NOAA-EMC/GDASApp.git" "feature/cycling" ; errs=$((errs + $?)) checkout "gldas.fd" "https://github.com/NOAA-EMC/GLDAS.git" "gldas_gfsv16_release.v.1.28.0" ; errs=$((errs + $?)) checkout "ufs_utils.fd" "https://github.com/ufs-community/UFS_UTILS.git" "04ad17e" ; errs=$((errs + $?)) diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 751e09b9a9..b06f3a5e51 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -66,7 +66,8 @@ for dir in fix_aer \ fix_cpl \ fix_wave \ fix_reg2grb2 \ - fix_ugwd + fix_ugwd \ + fix_jedi do if [ -d $dir ]; then [[ $RUN_ENVIR = nco ]] && chmod -R 755 $dir @@ -176,7 +177,8 @@ cd ${pwd}/../jobs ||exit 8 $LINK ../sorc/gdas.cd/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS . $LINK ../sorc/gdas.cd/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL . cd ${pwd}/../ush ||exit 8 - $LINK ../sorc/gdas.cd/ush/ufsda . + $LINK ../sorc/gdas.cd/ush/ufsda . + #------------------------------ #--add DA Monitor file (NOTE: ensure to use correct version) From 26901d6f6abe62eaeea47f551a912165c7bda00a Mon Sep 17 00:00:00 2001 From: "russ.treadon" Date: Wed, 29 Jun 2022 11:23:27 +0000 Subject: [PATCH 36/45] Remove DOGCYCLE from sfcanl script, correct typo in config.postanl --- parm/config/config.postanl | 2 +- scripts/exglobal_atmos_sfcanl.sh | 111 +++++++++++++++---------------- 2 files changed, 55 insertions(+), 58 deletions(-) diff --git a/parm/config/config.postanl b/parm/config/config.postanl index 771311c2f6..8e32f191e4 100755 --- a/parm/config/config.postanl +++ b/parm/config/config.postanl @@ -35,4 +35,4 @@ export downset=2 export GRIBVERSION='grib2' export SENDCOM="YES" -echo "END: config.post" +echo "END: config.postanl" diff --git a/scripts/exglobal_atmos_sfcanl.sh b/scripts/exglobal_atmos_sfcanl.sh index ca9cae9b3e..1d6ac67b0f 100755 --- a/scripts/exglobal_atmos_sfcanl.sh +++ b/scripts/exglobal_atmos_sfcanl.sh @@ -56,7 +56,6 @@ DOIAU=${DOIAU:-"NO"} export IAUFHRS=${IAUFHRS:-"6"} # Surface cycle related parameters -DOGCYCLE=${DOGCYCLE:-"NO"} CYCLESH=${CYCLESH:-$HOMEgfs/ush/global_cycle.sh} export CYCLEXEC=${CYCLEXEC:-$HOMEgfs/exec/global_cycle} NTHREADS_CYCLE=${NTHREADS_CYCLE:-24} @@ -146,63 +145,46 @@ fi ############################################################## # Update surface fields in the FV3 restart's using global_cycle -if [ $DOGCYCLE = "YES" ]; then - - mkdir -p $COMOUT/RESTART - - # Global cycle requires these files - export FNTSFA=${FNTSFA:-$COMIN_OBS/${OPREFIX}rtgssthr.grb} - export FNACNA=${FNACNA:-$COMIN_OBS/${OPREFIX}seaice.5min.blend.grb} - export FNSNOA=${FNSNOA:-$COMIN_OBS/${OPREFIX}snogrb_t${JCAP_CASE}.${LONB_CASE}.${LATB_CASE}} - [[ ! -f $FNSNOA ]] && export FNSNOA="$COMIN_OBS/${OPREFIX}snogrb_t1534.3072.1536" - FNSNOG=${FNSNOG:-$COMIN_GES_OBS/${GPREFIX}snogrb_t${JCAP_CASE}.${LONB_CASE}.${LATB_CASE}} - [[ ! -f $FNSNOG ]] && FNSNOG="$COMIN_GES_OBS/${GPREFIX}snogrb_t1534.3072.1536" - - # Set CYCLVARS by checking grib date of current snogrb vs that of prev cycle - if [ $RUN_GETGES = "YES" ]; then - snoprv=$($GETGESSH -q -t snogrb_$JCAP_CASE -e $gesenvir -n $GDUMP -v $GDATE) - else - snoprv=${snoprv:-$FNSNOG} - fi - - if [ $($WGRIB -4yr $FNSNOA 2>/dev/null | grep -i snowc | awk -F: '{print $3}' | awk -F= '{print $2}') -le \ - $($WGRIB -4yr $snoprv 2>/dev/null | grep -i snowc | awk -F: '{print $3}' | awk -F= '{print $2}') ] ; then - export FNSNOA=" " - export CYCLVARS="FSNOL=99999.,FSNOS=99999.," - else - export SNOW_NUDGE_COEFF=${SNOW_NUDGE_COEFF:-0.} - export CYCLVARS="FSNOL=${SNOW_NUDGE_COEFF},$CYCLVARS" - fi - - if [ $DONST = "YES" ]; then - export NST_FILE=${GSI_FILE:-$COMOUT/${APREFIX}dtfanl.nc} - else - export NST_FILE="NULL" - fi - - if [ $DOIAU = "YES" ]; then - # update surface restarts at the beginning of the window, if IAU - # For now assume/hold dtfanl.nc valid at beginning of window - for n in $(seq 1 $ntiles); do - $NLN $COMIN_GES/RESTART/$bPDY.${bcyc}0000.sfc_data.tile${n}.nc $DATA/fnbgsi.00$n - $NLN $COMOUT/RESTART/$bPDY.${bcyc}0000.sfcanl_data.tile${n}.nc $DATA/fnbgso.00$n - $NLN $FIXfv3/$CASE/${CASE}_grid.tile${n}.nc $DATA/fngrid.00$n - $NLN $FIXfv3/$CASE/${CASE}_oro_data.tile${n}.nc $DATA/fnorog.00$n - done - - export APRUNCY=$APRUN_CYCLE - export OMP_NUM_THREADS_CY=$NTHREADS_CYCLE - export MAX_TASKS_CY=$ntiles - - $CYCLESH - export err=$?; err_chk - fi - # update surface restarts at middle of window +mkdir -p $COMOUT/RESTART + +# Global cycle requires these files +export FNTSFA=${FNTSFA:-$COMIN_OBS/${OPREFIX}rtgssthr.grb} +export FNACNA=${FNACNA:-$COMIN_OBS/${OPREFIX}seaice.5min.blend.grb} +export FNSNOA=${FNSNOA:-$COMIN_OBS/${OPREFIX}snogrb_t${JCAP_CASE}.${LONB_CASE}.${LATB_CASE}} +[[ ! -f $FNSNOA ]] && export FNSNOA="$COMIN_OBS/${OPREFIX}snogrb_t1534.3072.1536" +FNSNOG=${FNSNOG:-$COMIN_GES_OBS/${GPREFIX}snogrb_t${JCAP_CASE}.${LONB_CASE}.${LATB_CASE}} +[[ ! -f $FNSNOG ]] && FNSNOG="$COMIN_GES_OBS/${GPREFIX}snogrb_t1534.3072.1536" + +# Set CYCLVARS by checking grib date of current snogrb vs that of prev cycle +if [ $RUN_GETGES = "YES" ]; then + snoprv=$($GETGESSH -q -t snogrb_$JCAP_CASE -e $gesenvir -n $GDUMP -v $GDATE) +else + snoprv=${snoprv:-$FNSNOG} +fi + +if [ $($WGRIB -4yr $FNSNOA 2>/dev/null | grep -i snowc | awk -F: '{print $3}' | awk -F= '{print $2}') -le \ + $($WGRIB -4yr $snoprv 2>/dev/null | grep -i snowc | awk -F: '{print $3}' | awk -F= '{print $2}') ] ; then + export FNSNOA=" " + export CYCLVARS="FSNOL=99999.,FSNOS=99999.," +else + export SNOW_NUDGE_COEFF=${SNOW_NUDGE_COEFF:-0.} + export CYCLVARS="FSNOL=${SNOW_NUDGE_COEFF},$CYCLVARS" +fi + +if [ $DONST = "YES" ]; then + export NST_FILE=${GSI_FILE:-$COMOUT/${APREFIX}dtfanl.nc} +else + export NST_FILE="NULL" +fi + +if [ $DOIAU = "YES" ]; then + # update surface restarts at the beginning of the window, if IAU + # For now assume/hold dtfanl.nc valid at beginning of window for n in $(seq 1 $ntiles); do - $NLN $COMIN_GES/RESTART/$PDY.${cyc}0000.sfc_data.tile${n}.nc $DATA/fnbgsi.00$n - $NLN $COMOUT/RESTART/$PDY.${cyc}0000.sfcanl_data.tile${n}.nc $DATA/fnbgso.00$n - $NLN $FIXfv3/$CASE/${CASE}_grid.tile${n}.nc $DATA/fngrid.00$n - $NLN $FIXfv3/$CASE/${CASE}_oro_data.tile${n}.nc $DATA/fnorog.00$n + $NLN $COMIN_GES/RESTART/$bPDY.${bcyc}0000.sfc_data.tile${n}.nc $DATA/fnbgsi.00$n + $NLN $COMOUT/RESTART/$bPDY.${bcyc}0000.sfcanl_data.tile${n}.nc $DATA/fnbgso.00$n + $NLN $FIXfv3/$CASE/${CASE}_grid.tile${n}.nc $DATA/fngrid.00$n + $NLN $FIXfv3/$CASE/${CASE}_oro_data.tile${n}.nc $DATA/fnorog.00$n done export APRUNCY=$APRUN_CYCLE @@ -213,6 +195,21 @@ if [ $DOGCYCLE = "YES" ]; then export err=$?; err_chk fi +# Update surface restarts at middle of window +for n in $(seq 1 $ntiles); do + $NLN $COMIN_GES/RESTART/$PDY.${cyc}0000.sfc_data.tile${n}.nc $DATA/fnbgsi.00$n + $NLN $COMOUT/RESTART/$PDY.${cyc}0000.sfcanl_data.tile${n}.nc $DATA/fnbgso.00$n + $NLN $FIXfv3/$CASE/${CASE}_grid.tile${n}.nc $DATA/fngrid.00$n + $NLN $FIXfv3/$CASE/${CASE}_oro_data.tile${n}.nc $DATA/fnorog.00$n +done + +export APRUNCY=$APRUN_CYCLE +export OMP_NUM_THREADS_CY=$NTHREADS_CYCLE +export MAX_TASKS_CY=$ntiles + +$CYCLESH +export err=$?; err_chk + ################################################################################ # Postprocessing From 667472055b8db74e05467f0c7b02eb6a67bce9d1 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 1 Jul 2022 15:36:29 +0000 Subject: [PATCH 37/45] Refactor JEDI-VAR and JEDI-ENS jobs, remove postanl (#871) - refactor JEDI-VAR and JEDI-ENS rocoto jobs to be consistent with other jobs - remove postanl job, config, and xml generation - split JEDI-VAR and JEDI-ENS resources into job specific sections - link function specific JEDI-VAR and JEDI-ENS j-jobs --- jobs/rocoto/atmanalpost.sh | 5 +--- jobs/rocoto/atmanalprep.sh | 5 +--- jobs/rocoto/atmanalrun.sh | 5 +--- jobs/rocoto/atmensanalpost.sh | 5 +--- jobs/rocoto/atmensanalprep.sh | 5 +--- jobs/rocoto/atmensanalrun.sh | 5 +--- jobs/rocoto/postanl.sh | 50 ----------------------------------- parm/config/config.postanl | 38 -------------------------- parm/config/config.resources | 21 +-------------- sorc/link_workflow.sh | 8 ++++-- ush/rocoto/setup_workflow.py | 37 ++++---------------------- 11 files changed, 18 insertions(+), 166 deletions(-) delete mode 100755 jobs/rocoto/postanl.sh delete mode 100755 parm/config/config.postanl diff --git a/jobs/rocoto/atmanalpost.sh b/jobs/rocoto/atmanalpost.sh index a13aac36fc..90a9b9bace 100755 --- a/jobs/rocoto/atmanalpost.sh +++ b/jobs/rocoto/atmanalpost.sh @@ -7,10 +7,7 @@ status=$? [[ $status -ne 0 ]] && exit $status ############################################################### -# set JJOB variables -export configs="base atmanal atmanalpost" -export EXSCRIPT=${GDASPOSTPY:-$HOMEgfs/sorc/gdas.cd/scripts/exgdas_global_atmos_analysis_post.py} # Execute the JJOB -$HOMEgfs/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS +$HOMEgfs/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_POST status=$? exit $status diff --git a/jobs/rocoto/atmanalprep.sh b/jobs/rocoto/atmanalprep.sh index aba0422a88..e4b76c8407 100755 --- a/jobs/rocoto/atmanalprep.sh +++ b/jobs/rocoto/atmanalprep.sh @@ -7,10 +7,7 @@ status=$? [[ $status -ne 0 ]] && exit $status ############################################################### -# set JJOB variables -export configs="base atmanal atmanalprep" -export EXSCRIPT=${GDASPREPPY:-$HOMEgfs/sorc/gdas.cd/scripts/exgdas_global_atmos_analysis_prep.py} # Execute the JJOB -$HOMEgfs/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS +$HOMEgfs/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_PREP status=$? exit $status diff --git a/jobs/rocoto/atmanalrun.sh b/jobs/rocoto/atmanalrun.sh index 6c99feb200..cebe478b7e 100755 --- a/jobs/rocoto/atmanalrun.sh +++ b/jobs/rocoto/atmanalrun.sh @@ -7,10 +7,7 @@ status=$? [[ $status -ne 0 ]] && exit $status ############################################################### -# set JJOB variables -export configs="base atmanal atmanalrun" -export EXSCRIPT=${GDASRUNSH:-$HOMEgfs/sorc/gdas.cd/scripts/exgdas_global_atmos_analysis_run.sh} # Execute the JJOB -$HOMEgfs/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS +$HOMEgfs/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_RUN status=$? exit $status diff --git a/jobs/rocoto/atmensanalpost.sh b/jobs/rocoto/atmensanalpost.sh index 54c127efbd..ea6e490f20 100755 --- a/jobs/rocoto/atmensanalpost.sh +++ b/jobs/rocoto/atmensanalpost.sh @@ -7,10 +7,7 @@ status=$? [[ $status -ne 0 ]] && exit $status ############################################################### -# set JJOB variables -export configs="base atmensanal atmensanalpost" -export EXSCRIPT=${GDASPOSTPY:-$HOMEgfs/sorc/gdas.cd/scripts/exgdas_global_atmos_ensanal_post.py} # Execute the JJOB -$HOMEgfs/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL +$HOMEgfs/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_POST status=$? exit $status diff --git a/jobs/rocoto/atmensanalprep.sh b/jobs/rocoto/atmensanalprep.sh index 44fcdbf72d..5ed434c6bf 100755 --- a/jobs/rocoto/atmensanalprep.sh +++ b/jobs/rocoto/atmensanalprep.sh @@ -7,10 +7,7 @@ status=$? [[ $status -ne 0 ]] && exit $status ############################################################### -# set JJOB variables -export configs="base atmensanal atmensanalprep" -export EXSCRIPT=${GDASPREPPY:-$HOMEgfs/sorc/gdas.cd/scripts/exgdas_global_atmos_analysis_prep.py} # Execute the JJOB -$HOMEgfs/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL +$HOMEgfs/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_PREP status=$? exit $status diff --git a/jobs/rocoto/atmensanalrun.sh b/jobs/rocoto/atmensanalrun.sh index f3aa2639ee..ddb3bb1432 100755 --- a/jobs/rocoto/atmensanalrun.sh +++ b/jobs/rocoto/atmensanalrun.sh @@ -7,10 +7,7 @@ status=$? [[ $status -ne 0 ]] && exit $status ############################################################### -# set JJOB variables -export configs="base atmensanal atmensanalrun" -export EXSCRIPT=${GDASRUNSH:-$HOMEgfs/sorc/gdas.cd/scripts/exgdas_global_atmos_ensanal_run.sh} # Execute the JJOB -$HOMEgfs/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL +$HOMEgfs/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_RUN status=$? exit $status diff --git a/jobs/rocoto/postanl.sh b/jobs/rocoto/postanl.sh deleted file mode 100755 index ac75d62ccc..0000000000 --- a/jobs/rocoto/postanl.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -x - -############################################################### -## NCEP post driver script -## FHRGRP : forecast hour group to post-process (e.g. 0, 1, 2 ...) -## FHRLST : forecast hourlist to be post-process (e.g. anl, f000, f000_f001_f002, ...) -############################################################### - -# Source FV3GFS workflow modules -. $HOMEgfs/ush/load_fv3gfs_modules.sh -status=$? -[[ $status -ne 0 ]] && exit $status - -export COMPONENT=${COMPONENT:-atmos} - -if [ $FHRGRP = 'anl' ]; then - fhrlst="anl" - restart_file=$ROTDIR/${CDUMP}.${PDY}/${cyc}/$COMPONENT/${CDUMP}.t${cyc}z.atm -else - echo "Invalid FHRGRP= $FHRGRP for postanl.sh. exit 99" - exit 99 -fi - - -#--------------------------------------------------------------- -for fhr in $fhrlst; do - - if [ ! -f $restart_file${fhr}.nemsio -a ! -f $restart_file${fhr}.nc -a ! -f $restart_file${fhr}.txt ]; then - echo "Nothing to process for FHR = $fhr, cycle, wait for 5 minutes" - sleep 300 - fi - if [ ! -f $restart_file${fhr}.nemsio -a ! -f $restart_file${fhr}.nc -a ! -f $restart_file${fhr}.txt ]; then - echo "Nothing to process for FHR = $fhr, cycle, skip" - continue - fi - - #master=$ROTDIR/${CDUMP}.${PDY}/${cyc}/$COMPONENT/${CDUMP}.t${cyc}z.master.grb2f${fhr} - pgb0p25=$ROTDIR/${CDUMP}.${PDY}/${cyc}/$COMPONENT/${CDUMP}.t${cyc}z.pgrb2.0p25.f${fhr} - if [ ! -s $pgb0p25 ]; then - export post_times=$fhr - $HOMEgfs/jobs/JGLOBAL_ATMOS_NCEPPOST - status=$? - [[ $status -ne 0 ]] && exit $status - fi - -done - -############################################################### -# Exit out cleanly -exit 0 diff --git a/parm/config/config.postanl b/parm/config/config.postanl deleted file mode 100755 index 8e32f191e4..0000000000 --- a/parm/config/config.postanl +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/ksh -x - -########## config.postanl ########## -# Postanl specific - -echo "BEGIN: config.postanl" - -# Get task specific resources -. $EXPDIR/config.resources postanl - -# Convert nemsio files to grib files using postanl job -#------------------------------------------- - -# No. of concurrent postanl jobs [0 implies sequential] -export NPOSTGRP=2 -export OUTTYP=4 -export MODEL_OUT_FORM=binarynemsiompiio -if [ $OUTPUT_FILE = "netcdf" ]; then - export MODEL_OUT_FORM=netcdfpara -fi - -# Post driver job that calls gfs_nceppost.sh and downstream jobs -export POSTJJOBSH="$HOMEpost/jobs/JGLOBAL_NCEPPOST" -export GFSDOWNSH="$HOMEpost/ush/fv3gfs_downstream_nems.sh" -export GFSDWNSH="$HOMEpost/ush/fv3gfs_dwn_nems.sh" - -export POSTGPSH="$HOMEpost/ush/gfs_nceppost.sh" -export POSTGPEXEC="$HOMEpost/exec/gfs_ncep_post" -export GOESF=NO # goes image -export FLXF=YES # grib2 flux file written by post - -export GFS_DOWNSTREAM="YES" -export downset=2 - -export GRIBVERSION='grib2' -export SENDCOM="YES" - -echo "END: config.postanl" diff --git a/parm/config/config.resources b/parm/config/config.resources index d4c52d52fd..49507e3e00 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -11,7 +11,7 @@ if [ $# -ne 1 ]; then echo "getic init coupled_ic aerosol_init" echo "atmanalprep atmanalrun atmanalpost" echo "atmensanalprep atmensanalrun atmensanalpost" - echo "anal sfcanl analcalc analdiag gldas fcst postanl post vrfy metp arch echgres" + echo "anal sfcanl analcalc analdiag gldas fcst post vrfy metp arch echgres" echo "eobs ediag eomg eupd ecen esfc efcs epos earc" echo "init_chem mom6ic ocnpost" echo "waveinit waveprep wavepostsbs wavepostbndpnt wavepostbndpntbll wavepostpnt" @@ -327,21 +327,6 @@ elif [ $step = "ocnpost" ]; then export nth_ocnpost=1 export memory_ocnpost="96G" -elif [ $step = "postanl" ]; then - - export wtime_postanl="00:30:00" - export wtime_postanl_gfs="06:00:00" - export npe_postanl=112 - export nth_postanl=1 - export npe_node_postanl=40 - export npe_dwn=24 - export npe_node_dwn=$npe_node_max - if [[ "$machine" = "WCOSS_DELL_P3" ]]; then - export npe_dwn=28 - export npe_node_postanl=28 - fi - if [[ "$machine" == "WCOSS_C" ]]; then export memory_postanl="3072M"; fi - elif [ $step = "post" ]; then export wtime_post="02:00:00" @@ -473,10 +458,6 @@ elif [ $step = "coupled_ic" ]; then elif [ $step = "atmensanalprep" ]; then - # make below case dependent later - export layout_x=2 - export layout_y=3 - export wtime_atmensanalprep="00:10:00" export npe_atmensanalprep=1 export nth_atmensanalprep=1 diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 26ae9f80e3..cf3f0209dd 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -174,8 +174,12 @@ cd ${pwd}/../ush ||exit 8 #--add GDASApp files #------------------------------ cd ${pwd}/../jobs ||exit 8 - $LINK ../sorc/gdas.cd/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS . - $LINK ../sorc/gdas.cd/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL . + $LINK ../sorc/gdas.cd/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_PREP . + $LINK ../sorc/gdas.cd/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_RUN . + $LINK ../sorc/gdas.cd/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_POST . + $LINK ../sorc/gdas.cd/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_PREP . + $LINK ../sorc/gdas.cd/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_RUN . + $LINK ../sorc/gdas.cd/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_POST . cd ${pwd}/../ush ||exit 8 $LINK ../sorc/gdas.cd/ush/ufsda . diff --git a/ush/rocoto/setup_workflow.py b/ush/rocoto/setup_workflow.py index a25a3cd792..bfacc5bc94 100755 --- a/ush/rocoto/setup_workflow.py +++ b/ush/rocoto/setup_workflow.py @@ -42,7 +42,7 @@ def main(): print(f'input arg: --expdir = {repr(args.expdir)}') sys.exit(1) - gfs_steps = ['prep', 'sfcanl', 'analcalc', 'gldas', 'fcst', 'postsnd', 'postanl', 'post', 'vrfy', 'arch'] + gfs_steps = ['prep', 'sfcanl', 'analcalc', 'gldas', 'fcst', 'postsnd', 'post', 'vrfy', 'arch'] gfs_steps_gempak = ['gempak'] gfs_steps_awips = ['awips'] gfs_steps_wafs = ['wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25'] @@ -272,7 +272,7 @@ def get_gdasgfs_resources(dict_configs, cdump='gdas'): #tasks += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostpnt', 'wavestat'] tasks += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt'] - tasks += ['fcst', 'postanl', 'post', 'vrfy', 'arch'] + tasks += ['fcst', 'post', 'vrfy', 'arch'] if cdump in ['gfs'] and do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']: #tasks += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostbndpnt', 'wavepostpnt', 'wavestat'] @@ -657,25 +657,6 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'): dict_tasks[f'{cdump}fcst'] = task - # postanl - deps = [] - data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.loganl.txt' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{cdump}analcalc'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) - fhrgrp = rocoto.create_envar(name='FHRGRP', value='anl') - fhrlst = rocoto.create_envar(name='FHRLST', value='anl') - ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') - postenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] - varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = get_postgroups_anl(dict_configs['postanl'], cdump=cdump) - vardict = {varname2: varval2, varname3: varval3} - task = wfu.create_wf_task('postanl', cdump=cdump, envar=postenvars, dependency=dependencies) - - dict_tasks[f'{cdump}postanl'] = task - # post deps = [] data = f'&ROTDIR;/{cdump}.@Y@m@d/@H/atmos/{cdump}.t@Hz.log#dep#.txt' @@ -1345,14 +1326,6 @@ def get_workflow_footer(): return ''.join(strings) -def get_postgroups_anl(post, cdump='gdas'): - - fhrgrp = 'anl' - fhrdep = 'anl' - fhrlst = 'anl' - - return fhrgrp, fhrdep, fhrlst - def get_postgroups(post, cdump='gdas'): fhmin = post['FHMIN'] @@ -1377,9 +1350,9 @@ def get_postgroups(post, cdump='gdas'): fhrs = np.array_split(fhrs, ngrps) fhrs = [f.tolist() for f in fhrs] - fhrgrp = ' '.join([f'_{f[0]}-{f[-1]}' for f in fhrs]) - fhrdep = ' '.join([f[-1] for f in fhrs]) - fhrlst = ' '.join(['_'.join(f) for f in fhrs]) + fhrgrp = ' '.join(['anl'] + [f'_{f[0]}-{f[-1]}' for f in fhrs]) + fhrdep = ' '.join(['anl'] + [f[-1] for f in fhrs]) + fhrlst = ' '.join(['anl'] + ['_'.join(f) for f in fhrs]) return fhrgrp, fhrdep, fhrlst From fa42b0b4c48192bbc21d363e89711f36caa8c135 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 1 Jul 2022 16:02:30 +0000 Subject: [PATCH 38/45] add JEDI-VAR and JEDI-ENS j-jobs to .gitignore --- .gitignore | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index bc7fd60990..e8559d3885 100644 --- a/.gitignore +++ b/.gitignore @@ -142,8 +142,12 @@ jobs/JGDAS_ENKF_POST jobs/JGDAS_ENKF_SELECT_OBS jobs/JGDAS_ENKF_SFC jobs/JGDAS_ENKF_UPDATE -jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS -jobs/JGDAS_GLOBAL_ATMOS_ENSANAL +jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_POST +jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_PREP +jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_RUN +jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_POST +jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_PREP +jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_RUN jobs/JGFS_ATMOS_VMINMON jobs/JGFS_ATMOS_WAFS jobs/JGFS_ATMOS_WAFS_BLENDING From fbbc582c90d7a8d2757ac936580d97e9e0a48216 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 21 Jul 2022 16:16:52 +0000 Subject: [PATCH 39/45] Update xml generation to include JEDI_VAR and JEDI_ENS --- workflow/applications.py | 44 +++++++++--- workflow/rocoto/workflow_tasks.py | 116 ++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+), 8 deletions(-) diff --git a/workflow/applications.py b/workflow/applications.py index e91475e36f..f6889ac72f 100644 --- a/workflow/applications.py +++ b/workflow/applications.py @@ -105,6 +105,8 @@ def __init__(self, configuration: Configuration) -> None: self.do_wafs = _base.get('DO_WAFS', False) self.do_vrfy = _base.get('DO_VRFY', True) self.do_metp = _base.get('DO_METP', False) + self.do_jedivar = _base.get('DO_JEDIVAR', False) + self.do_jediens = _base.get('DO_JEDIENS', False) self.do_hpssarch = _base.get('HPSSARCH', False) @@ -170,15 +172,27 @@ def _cycled_configs(self): Returns the config_files that are involved in the cycled app """ - configs = ['prep', - 'anal', 'sfcanl', 'analdiag', 'analcalc', - 'fcst', 'post', 'vrfy', 'arch'] + configs = ['prep'] +## 'sfcanl', 'analcalc', +## 'fcst', 'post', 'vrfy', 'arch'] + if self.do_jedivar: + configs += ['atmanalprep', 'atmanalrun', 'atmanalpost'] + else: + configs += ['anal', 'analdiag'] + + configs += ['sfcanl', 'analcalc', 'fcst', 'post', 'vrfy', 'arch'] + + if self.do_gldas: configs += ['gldas'] if self.do_hybvar: - configs += ['eobs', 'eomg', 'ediag', 'eupd', 'ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc'] + if self.do_jediens: + configs += ['atmensanalprep', 'atmensanalrun', 'atmensanalpost'] + else: + configs += ['eobs', 'eomg', 'ediag', 'eupd'] + configs += ['ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc'] if self.do_metp: configs += ['metp'] @@ -316,10 +330,17 @@ def _get_cycled_task_names(self): This is the place where that order is set. """ - gdas_gfs_common_tasks_before_fcst = ['prep', 'anal', 'sfcanl', 'analcalc'] + gdas_gfs_common_tasks_before_fcst = ['prep'] gdas_gfs_common_tasks_after_fcst = ['post', 'vrfy'] gdas_gfs_common_cleanup_tasks = ['arch'] + if self.do_jedivar: + gdas_gfs_common_tasks_before_fcst += ['atmanalprep', 'atmanalrun', 'atmanalpost'] + else: + gdas_gfs_common_tasks_before_fcst += ['anal'] + + gdas_gfs_common_tasks_before_fcst += ['sfcanl', 'analcalc'] + gldas_tasks = ['gldas'] wave_prep_tasks = ['waveinit', 'waveprep'] wave_bndpnt_tasks = ['wavepostbndpnt', 'wavepostbndpntbll'] @@ -328,12 +349,19 @@ def _get_cycled_task_names(self): hybrid_gdas_or_gfs_tasks = [] hybrid_gdas_tasks = [] if self.do_hybvar: - hybrid_gdas_or_gfs_tasks += ['eobs', 'eupd', 'echgres'] - hybrid_gdas_or_gfs_tasks += ['ediag'] if self.lobsdiag_forenkf else ['eomg'] + if self.do_jediens: + hybrid_gdas_or_gfs_tasks += ['atmensanalprep', 'atmensanalrun', 'atmensanalpost', 'echgres'] + else: + hybrid_gdas_or_gfs_tasks += ['eobs', 'eupd', 'echgres'] + hybrid_gdas_or_gfs_tasks += ['ediag'] if self.lobsdiag_forenkf else ['eomg'] hybrid_gdas_tasks += ['ecen', 'esfc', 'efcs', 'epos', 'earc'] # Collect all "gdas" cycle tasks - gdas_tasks = gdas_gfs_common_tasks_before_fcst + ['analdiag'] + gdas_tasks = [] + if self.do_jedivar: + gdas_tasks += gdas_gfs_common_tasks_before_fcst + else: + gdas_tasks += gdas_gfs_common_tasks_before_fcst + ['analdiag'] if self.do_gldas: gdas_tasks += gldas_tasks diff --git a/workflow/rocoto/workflow_tasks.py b/workflow/rocoto/workflow_tasks.py index 8e356a2a52..e10f5f6779 100644 --- a/workflow/rocoto/workflow_tasks.py +++ b/workflow/rocoto/workflow_tasks.py @@ -12,8 +12,10 @@ class Tasks: SERVICE_TASKS = ['arch', 'earc', 'getic'] VALID_TASKS = ['aerosol_init', 'coupled_ic', 'getic', 'init', 'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'gldas', 'arch', + 'atmanalprep', 'atmanalrun', 'atmanalpost', 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', + 'atmensanalprep', 'atmensanalrun', 'atmensanalpost', 'fcst', 'post', 'ocnpost', 'vrfy', 'metp', 'postsnd', 'awips', 'gempak', 'wafs', 'wafsblending', 'wafsblending0p25', @@ -376,6 +378,65 @@ def analdiag(self): return task + def atmanalprep(self): + + suffix = self._base["SUFFIX"] + dump_suffix = self._base["DUMP_SUFFIX"] + gfs_cyc = self._base["gfs_cyc"] + dmpdir = self._base["DMPDIR"] + gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.eupd_cdumps else False + + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{"gdas"}post', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.atmf009{suffix}' + dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'{dmpdir}/{self.cdump}{dump_suffix}.@Y@m@d/@H/{self.cdump}.t@Hz.updated.status.tm00.bufr_d' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + cycledef = self.cdump + if self.cdump in ['gfs'] and gfs_enkf and gfs_cyc != 4: + cycledef = 'gdas' + + resources = self.get_resource('atmanalprep') + task = create_wf_task('atmanalprep', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, + cycledef=cycledef) + return task + + def atmanalrun(self): + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}atmanalprep'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_hybvar: + dep_dict = {'type': 'metatask', 'name': f'{"gdas"}epmn', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + else: + dependencies = rocoto.create_dependency(dep=deps) + + resources = self.get_resource('atmanalrun') + task = create_wf_task('atmanalrun', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + + def atmanalpost(self): + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}atmanalrun'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'cycleexist', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + resources = self.get_resource('atmanalpost') + task = create_wf_task('atmanalpost', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + def gldas(self): deps = [] @@ -880,6 +941,61 @@ def eupd(self): return task + def atmensanalprep(self): + + suffix = self._base["SUFFIX"] + dump_suffix = self._base["DUMP_SUFFIX"] + gfs_cyc = self._base["gfs_cyc"] + dmpdir = self._base["DMPDIR"] + gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.eupd_cdumps else False + + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{"gdas"}post', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'&ROTDIR;/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.atmf009{suffix}' + dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + data = f'{dmpdir}/{self.cdump}{dump_suffix}.@Y@m@d/@H/{self.cdump}.t@Hz.updated.status.tm00.bufr_d' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + cycledef = self.cdump + if self.cdump in ['gfs'] and gfs_enkf and gfs_cyc != 4: + cycledef = 'gdas' + + resources = self.get_resource('atmensanalprep') + task = create_wf_task('atmensanalprep', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, + cycledef=cycledef) + + return task + + def atmensanalrun(self): + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanalprep'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': f'{"gdas"}epmn', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + resources = self.get_resource('atmensanalrun') + task = create_wf_task('atmensanalrun', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + + def atmensanalpost(self): + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanalrun'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + resources = self.get_resource('atmensanalpost') + task = create_wf_task('atmensanalpost', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + + return task + def ecen(self): self._is_this_a_gdas_task(self.cdump, 'ecen') From abb15aba906e6774265441a9ad854517347bb0a2 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 22 Jul 2022 00:16:11 +0000 Subject: [PATCH 40/45] Update JEDI_VAR dependencies for sfcanl and analcalc --- workflow/rocoto/workflow_tasks.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/workflow/rocoto/workflow_tasks.py b/workflow/rocoto/workflow_tasks.py index e10f5f6779..bfa319727d 100644 --- a/workflow/rocoto/workflow_tasks.py +++ b/workflow/rocoto/workflow_tasks.py @@ -338,7 +338,10 @@ def anal(self): def sfcanl(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump}anal'} + if self.app_config.do_jedivar: + dep_dict = {'type': 'task', 'name': f'{self.cdump}atmanalrun'} + else: + dep_dict = {'type': 'task', 'name': f'{self.cdump}anal'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) @@ -350,7 +353,10 @@ def sfcanl(self): def analcalc(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump}anal'} + if self.app_config.do_jedivar: + dep_dict = {'type': 'task', 'name': f'{self.cdump}atmanalrun'} + else: + dep_dict = {'type': 'task', 'name': f'{self.cdump}anal'} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.cdump}sfcanl'} deps.append(rocoto.add_dependency(dep_dict)) From b1631905c6b859cfcbcfd615ab76094e762bcff3 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 22 Jul 2022 14:37:23 +0000 Subject: [PATCH 41/45] Update JEDI_ENS dependencies for ecen and esfc --- workflow/rocoto/workflow_tasks.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/workflow/rocoto/workflow_tasks.py b/workflow/rocoto/workflow_tasks.py index bfa319727d..512c3b738d 100644 --- a/workflow/rocoto/workflow_tasks.py +++ b/workflow/rocoto/workflow_tasks.py @@ -1034,7 +1034,10 @@ def _get_ecengroups(): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}analcalc'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{eupd_cdump}eupd'} + if self.app_config.do_jediens: + dep_dict = {'type': 'task', 'name': f'{eupd_cdump}atmensanalrun'} + else: + dep_dict = {'type': 'task', 'name': f'{eupd_cdump}eupd'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -1062,7 +1065,10 @@ def esfc(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}analcalc'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{eupd_cdump}eupd'} + if self.app_config.do_jediens: + dep_dict = {'type': 'task', 'name': f'{eupd_cdump}atmensanalrun'} + else: + dep_dict = {'type': 'task', 'name': f'{eupd_cdump}eupd'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) From 594f68db15d744f195b33a11170cce4fb4173ef6 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 22 Jul 2022 14:54:48 +0000 Subject: [PATCH 42/45] Update workflow python scripts as recommended by reviewer --- workflow/applications.py | 2 -- workflow/rocoto/workflow_tasks.py | 14 +++++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/workflow/applications.py b/workflow/applications.py index 468329fa20..b6c4ba8c65 100644 --- a/workflow/applications.py +++ b/workflow/applications.py @@ -173,8 +173,6 @@ def _cycled_configs(self): """ configs = ['prep'] -## 'sfcanl', 'analcalc', -## 'fcst', 'post', 'vrfy', 'arch'] if self.do_jedivar: configs += ['atmanalprep', 'atmanalrun', 'atmanalpost'] diff --git a/workflow/rocoto/workflow_tasks.py b/workflow/rocoto/workflow_tasks.py index 512c3b738d..5ef13d3aab 100644 --- a/workflow/rocoto/workflow_tasks.py +++ b/workflow/rocoto/workflow_tasks.py @@ -390,10 +390,10 @@ def atmanalprep(self): dump_suffix = self._base["DUMP_SUFFIX"] gfs_cyc = self._base["gfs_cyc"] dmpdir = self._base["DMPDIR"] - gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.eupd_cdumps else False + do_gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.eupd_cdumps else False deps = [] - dep_dict = {'type': 'metatask', 'name': f'{"gdas"}post', 'offset': '-06:00:00'} + dep_dict = {'type': 'metatask', 'name': 'gdaspost', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) data = f'&ROTDIR;/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.atmf009{suffix}' dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'} @@ -404,7 +404,7 @@ def atmanalprep(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) cycledef = self.cdump - if self.cdump in ['gfs'] and gfs_enkf and gfs_cyc != 4: + if self.cdump in ['gfs'] and do_gfs_enkf and gfs_cyc != 4: cycledef = 'gdas' resources = self.get_resource('atmanalprep') @@ -953,10 +953,10 @@ def atmensanalprep(self): dump_suffix = self._base["DUMP_SUFFIX"] gfs_cyc = self._base["gfs_cyc"] dmpdir = self._base["DMPDIR"] - gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.eupd_cdumps else False + do_gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.eupd_cdumps else False deps = [] - dep_dict = {'type': 'metatask', 'name': f'{"gdas"}post', 'offset': '-06:00:00'} + dep_dict = {'type': 'metatask', 'name': 'gdaspost', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) data = f'&ROTDIR;/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.atmf009{suffix}' dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'} @@ -967,7 +967,7 @@ def atmensanalprep(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) cycledef = self.cdump - if self.cdump in ['gfs'] and gfs_enkf and gfs_cyc != 4: + if self.cdump in ['gfs'] and do_gfs_enkf and gfs_cyc != 4: cycledef = 'gdas' resources = self.get_resource('atmensanalprep') @@ -981,7 +981,7 @@ def atmensanalrun(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanalprep'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': f'{"gdas"}epmn', 'offset': '-06:00:00'} + dep_dict = {'type': 'metatask', 'name': 'gdasepmn', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) From 39fe8813f544713b3205e0a669a04b2060fd8b96 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 22 Jul 2022 15:02:32 +0000 Subject: [PATCH 43/45] Update atmanalrun dependency as suggested by reviewer --- workflow/rocoto/workflow_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/workflow_tasks.py b/workflow/rocoto/workflow_tasks.py index 5ef13d3aab..ea90ee37aa 100644 --- a/workflow/rocoto/workflow_tasks.py +++ b/workflow/rocoto/workflow_tasks.py @@ -418,7 +418,7 @@ def atmanalrun(self): dep_dict = {'type': 'task', 'name': f'{self.cdump}atmanalprep'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_hybvar: - dep_dict = {'type': 'metatask', 'name': f'{"gdas"}epmn', 'offset': '-06:00:00'} + dep_dict = {'type': 'metatask', 'name': 'gdasepmn', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) else: From 866467e49b97cb7036f1366193adeac1985fe39c Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 22 Jul 2022 16:19:38 +0000 Subject: [PATCH 44/45] Update initialization of gdas_tasks as suggested by reviewers --- workflow/applications.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/workflow/applications.py b/workflow/applications.py index b6c4ba8c65..8256cd9699 100644 --- a/workflow/applications.py +++ b/workflow/applications.py @@ -358,11 +358,9 @@ def _get_cycled_task_names(self): hybrid_gdas_tasks += ['ecen', 'esfc', 'efcs', 'epos', 'earc'] # Collect all "gdas" cycle tasks - gdas_tasks = [] - if self.do_jedivar: - gdas_tasks += gdas_gfs_common_tasks_before_fcst - else: - gdas_tasks += gdas_gfs_common_tasks_before_fcst + ['analdiag'] + gdas_tasks = gdas_gfs_common_tasks_before_fcst.copy() + if not self.do_jedivar: + gdas_tasks += ['analdiag'] if self.do_gldas: gdas_tasks += gldas_tasks From b8f639c20d2d3bae9c056da8905cc013a53a0cbf Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 22 Jul 2022 19:38:57 +0000 Subject: [PATCH 45/45] Add flag for JEDI_ENS to config.base.nco.static --- parm/config/config.base.nco.static | 1 + 1 file changed, 1 insertion(+) diff --git a/parm/config/config.base.nco.static b/parm/config/config.base.nco.static index ce841f08c2..48db9dd020 100755 --- a/parm/config/config.base.nco.static +++ b/parm/config/config.base.nco.static @@ -217,6 +217,7 @@ export imp_physics=8 # Shared parameters # DA engine export DO_JEDIVAR="NO" +export DO_JEDIENS="NO" # Hybrid related export DOHYBVAR="YES"