From 8ae1a8308ebf2155fbfbe6d99d36fa0caacf88bb Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Mon, 8 May 2023 16:39:17 -0400 Subject: [PATCH 01/18] Commit to test on Hera --- prototypes/configs/primordial_config.sh | 4 +++ prototypes/gen_prototype.sh | 41 +++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 prototypes/configs/primordial_config.sh create mode 100644 prototypes/gen_prototype.sh diff --git a/prototypes/configs/primordial_config.sh b/prototypes/configs/primordial_config.sh new file mode 100644 index 000000000..808fc486f --- /dev/null +++ b/prototypes/configs/primordial_config.sh @@ -0,0 +1,4 @@ +GWHASH=6838c0b +GDASHASH=58d0b9b +PSLOT=apple +PROTOROOT=/scratch1/NCEPDEV/da/Cory.R.Martin/GDAS_prototypes diff --git a/prototypes/gen_prototype.sh b/prototypes/gen_prototype.sh new file mode 100644 index 000000000..21a45a5d8 --- /dev/null +++ b/prototypes/gen_prototype.sh @@ -0,0 +1,41 @@ +#!/bin/bash +set -e + +# ============================================================================== +usage() { + set +x + echo + echo "Usage: $0 -c -h" + echo + echo " -c Configuration for prototype defined in shell script " + echo " -h display this message and quit" + echo + exit 1 +} + +# ============================================================================== +while getopts "c:h" opt; do + case $opt in + c) + config=$OPTARG + ;; + h|\?|:) + usage + ;; + esac +done + +# source the input configuration +source $config + +# create directories +mkdir -p $PROTOROOT/$PSLOT + +# clone/build/link workflow and GDASApp +cd $PROTOROOT/$PSLOT +git clone -b $GWHASH https://github.com/NOAA-EMC/global-workflow.git +cd global-workflow/sorc +git clone -b $GDASHASH https://github.com/NOAA-EMC/GDASApp.git gdas.cd +./checkout.sh -g +./build_all.sh +./link_workflow.sh From 4222119beccea7f946aa07c73477f17043f8d3b4 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Mon, 8 May 2023 20:45:10 +0000 Subject: [PATCH 02/18] Update to clone/checkout properly --- prototypes/gen_prototype.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) mode change 100644 => 100755 prototypes/gen_prototype.sh diff --git a/prototypes/gen_prototype.sh b/prototypes/gen_prototype.sh old mode 100644 new mode 100755 index 21a45a5d8..029f3e4f2 --- a/prototypes/gen_prototype.sh +++ b/prototypes/gen_prototype.sh @@ -33,9 +33,13 @@ mkdir -p $PROTOROOT/$PSLOT # clone/build/link workflow and GDASApp cd $PROTOROOT/$PSLOT -git clone -b $GWHASH https://github.com/NOAA-EMC/global-workflow.git +git clone https://github.com/NOAA-EMC/global-workflow.git cd global-workflow/sorc -git clone -b $GDASHASH https://github.com/NOAA-EMC/GDASApp.git gdas.cd +git checkout $GWHASH +git clone https://github.com/NOAA-EMC/GDASApp.git gdas.cd +cd gdas.cd +git checkout $GDASHASH +cd ../ ./checkout.sh -g ./build_all.sh ./link_workflow.sh From 3e40d64e366d16ea2980b571e113297a652b9450 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Tue, 9 May 2023 15:14:54 -0400 Subject: [PATCH 03/18] Incremental commit --- modulefiles/GDAS/hera.lua | 1 + modulefiles/GDAS/orion.lua | 3 ++ prototypes/configs/primordial_config.sh | 2 + prototypes/gen_prototype.sh | 63 +++++++++++++++++++++---- 4 files changed, 60 insertions(+), 9 deletions(-) diff --git a/modulefiles/GDAS/hera.lua b/modulefiles/GDAS/hera.lua index 6bc27d9c3..3485fd503 100644 --- a/modulefiles/GDAS/hera.lua +++ b/modulefiles/GDAS/hera.lua @@ -64,6 +64,7 @@ load("netcdf-cxx4/4.3.1") load("json/3.10.5") load("py-pybind11/2.8.1") --load("crtm/v2.4_jedi") +load("rocoto/1.3.3") load("hpc/1.2.0") load("miniconda3/4.6.14") diff --git a/modulefiles/GDAS/orion.lua b/modulefiles/GDAS/orion.lua index 790ea3020..c9c6248a4 100644 --- a/modulefiles/GDAS/orion.lua +++ b/modulefiles/GDAS/orion.lua @@ -58,6 +58,9 @@ load("ncview/2.1.8") load("netcdf-cxx4/4.3.1") load("py-pybind11/2.8.1") --load("crtm/v2.4_jedi") +load("contrib/0.1") +load("noaatools/2.0") +load("rocoto/1.3.3") load("hpc/1.2.0") load("miniconda3/4.6.14") diff --git a/prototypes/configs/primordial_config.sh b/prototypes/configs/primordial_config.sh index 808fc486f..95f37b9e4 100644 --- a/prototypes/configs/primordial_config.sh +++ b/prototypes/configs/primordial_config.sh @@ -2,3 +2,5 @@ GWHASH=6838c0b GDASHASH=58d0b9b PSLOT=apple PROTOROOT=/scratch1/NCEPDEV/da/Cory.R.Martin/GDAS_prototypes +GWDIR=$PROTOROOT/$PSLOT +BUILD="YES" diff --git a/prototypes/gen_prototype.sh b/prototypes/gen_prototype.sh index 21a45a5d8..96be04371 100644 --- a/prototypes/gen_prototype.sh +++ b/prototypes/gen_prototype.sh @@ -5,26 +5,31 @@ set -e usage() { set +x echo - echo "Usage: $0 -c -h" + echo "Usage: $0 -c -t -h" echo echo " -c Configuration for prototype defined in shell script " + echo " -t Supported platform script is running on " echo " -h display this message and quit" echo exit 1 } # ============================================================================== -while getopts "c:h" opt; do +while getopts "c:t:h" opt; do case $opt in c) config=$OPTARG ;; + t) + MACHINE=${MACHINE:-$OPTARG} h|\?|:) usage ;; esac done +dir_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../ >/dev/null 2>&1 && pwd )" + # source the input configuration source $config @@ -32,10 +37,50 @@ source $config mkdir -p $PROTOROOT/$PSLOT # clone/build/link workflow and GDASApp -cd $PROTOROOT/$PSLOT -git clone -b $GWHASH https://github.com/NOAA-EMC/global-workflow.git -cd global-workflow/sorc -git clone -b $GDASHASH https://github.com/NOAA-EMC/GDASApp.git gdas.cd -./checkout.sh -g -./build_all.sh -./link_workflow.sh +if [[ $BUILD == 'YES' ]]; then + cd $GWDIR + git clone -b $GWHASH https://github.com/NOAA-EMC/global-workflow.git + cd global-workflow/sorc + ./checkout.sh -ug + cd gdas.cd + git checkout $GDASHASH + cd .. + ./build_all.sh + ./link_workflow.sh +fi + +# load modules to then generate experiment directory and rocoto XML +module use $dir_root/modulefiles +module load GDAS/$MACHINE + +# move expdir if it exists, delete backup if it exists +if [[ -d $expdir/$PSLOT ]]; then + [[ -d $expdir/${PSLOT}.bak]] && rm -rf $expdir/${PSLOT}.bak + mv $expdir/$PSLOT $expdir/${PSLOT}.bak +fi + +# create YAML to override workflow config defaults +cat > $expdir/config_${PSLOT}.yaml << EOF +base: + ACCOUNT: da-cpu + HPSS_PROJECT: emc-da + HOMEDIR: "/scratch1/NCEPDEV/da/${USER}" +EOF + + + +# setup experiment +cd $GWDIR/workflow +./setup_expt.py cycled --idate $idate \ + --edate $edate \ + --app $app \ + --start $starttype \ + --gfs_cyc $gfscyc \ + --resdet $resdet \ + --resens $resens \ + --nens $nens \ + --pslot $pslot \ + --configdir $GWDIR/parm/config \ + --comrot $comrot \ + --expdir $expdir \ + --yaml $expdir/config_${PSLOT}.yaml From 0caf649b9791afc015f56e265caf36f60f381320 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 10 May 2023 15:21:11 +0000 Subject: [PATCH 04/18] incremental commit --- prototypes/configs/primordial_config.sh | 15 ++++++++- prototypes/gen_prototype.sh | 45 +++++++++++++++---------- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/prototypes/configs/primordial_config.sh b/prototypes/configs/primordial_config.sh index 95f37b9e4..10b80e5fd 100644 --- a/prototypes/configs/primordial_config.sh +++ b/prototypes/configs/primordial_config.sh @@ -1,6 +1,19 @@ -GWHASH=6838c0b +GWHASH=e55f090 GDASHASH=58d0b9b PSLOT=apple PROTOROOT=/scratch1/NCEPDEV/da/Cory.R.Martin/GDAS_prototypes GWDIR=$PROTOROOT/$PSLOT +DUMPDIR=/scratch1/NCEPDEV/da/role.jedipara/dump/ +ICSDIR=/scratch1/NCEPDEV/stmp4/Cory.R.Martin/tmp_ics +expdir=$PROTOROOT/expdirs +comrot=$PROTOROOT/rotdirs BUILD="YES" +idate=2021073118 +edate=2021080118 +app="ATM" +starttype="cold" +gfscyc=0 +resdet=96 +resens=48 +nens=80 + diff --git a/prototypes/gen_prototype.sh b/prototypes/gen_prototype.sh index 7901bd731..a249ea125 100755 --- a/prototypes/gen_prototype.sh +++ b/prototypes/gen_prototype.sh @@ -22,6 +22,7 @@ while getopts "c:t:h" opt; do ;; t) MACHINE=${MACHINE:-$OPTARG} + ;; h|\?|:) usage ;; @@ -56,32 +57,42 @@ module load GDAS/$MACHINE # move expdir if it exists, delete backup if it exists if [[ -d $expdir/$PSLOT ]]; then - [[ -d $expdir/${PSLOT}.bak]] && rm -rf $expdir/${PSLOT}.bak + [[ -d $expdir/${PSLOT}.bak ]] && rm -rf $expdir/${PSLOT}.bak mv $expdir/$PSLOT $expdir/${PSLOT}.bak fi +# move rotdir if it exists, delete backup if it exists +if [[ -d $comrot/$PSLOT ]]; then + [[ -d $comrot/${PSLOT}.bak ]] && rm -rf $comrot/${PSLOT}.bak + mv $comrot/$PSLOT $comrot/${PSLOT}.bak +fi + # create YAML to override workflow config defaults +mkdir -p $expdir cat > $expdir/config_${PSLOT}.yaml << EOF base: - ACCOUNT: da-cpu - HPSS_PROJECT: emc-da + ACCOUNT: "da-cpu" + HPSS_PROJECT: "emc-da" HOMEDIR: "/scratch1/NCEPDEV/da/${USER}" + DMPDIR: "${DUMPDIR}" +atmanl: + OBS_LIST: "/dev/null" EOF # setup experiment -cd $GWDIR/workflow -./setup_expt.py cycled --idate $idate \ - --edate $edate \ - --app $app \ - --start $starttype \ - --gfs_cyc $gfscyc \ - --resdet $resdet \ - --resens $resens \ - --nens $nens \ - --pslot $pslot \ - --configdir $GWDIR/parm/config \ - --comrot $comrot \ - --expdir $expdir \ - --yaml $expdir/config_${PSLOT}.yaml +cd $GWDIR/global-workflow/workflow +./setup_expt.py gfs cycled --idate $idate \ + --edate $edate \ + --app $app \ + --start $starttype \ + --gfs_cyc $gfscyc \ + --resdet $resdet \ + --resens $resens \ + --nens $nens \ + --pslot $PSLOT \ + --configdir $GWDIR/global-workflow/parm/config \ + --comrot $comrot \ + --expdir $expdir \ + --yaml $expdir/config_${PSLOT}.yaml From af866cd9b6f7b21bbad26eb9c0324afd42ad8d52 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 10 May 2023 15:56:10 +0000 Subject: [PATCH 05/18] fix for CI with workflow change --- test/atm/global-workflow/setup_workflow_exp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/atm/global-workflow/setup_workflow_exp.sh b/test/atm/global-workflow/setup_workflow_exp.sh index 8d18fc6e5..06d8e519b 100755 --- a/test/atm/global-workflow/setup_workflow_exp.sh +++ b/test/atm/global-workflow/setup_workflow_exp.sh @@ -37,7 +37,7 @@ ln -sf $srcdir/../../workflow/setup_expt.py . echo "Running global-workflow experiment generation script" -$srcdir/../../workflow/setup_expt.py cycled --idate $idate \ +$srcdir/../../workflow/setup_expt.py gfs cycled --idate $idate \ --edate $edate \ --app $app \ --start $starttype \ From c54c4a04d083d8bad00682fc4bf904cb11239264 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 10 May 2023 15:59:46 +0000 Subject: [PATCH 06/18] actually fix CI --- test/setup_workflow_exp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/setup_workflow_exp.sh b/test/setup_workflow_exp.sh index bd7a84494..274cbac05 100755 --- a/test/setup_workflow_exp.sh +++ b/test/setup_workflow_exp.sh @@ -24,6 +24,6 @@ rm -rf "${comrot}" "${expdir}" cd $srcdir/../../workflow echo "Running global-workflow experiment generation script" -./setup_expt.py cycled --idate $idate --edate $edate --app $app --start $starttype --gfs_cyc $gfscyc --resdet $resdet --resens $resens --nens $nens --pslot $pslot --configdir $configdir --comrot $comrot --expdir $expdir +./setup_expt.py gfs cycled --idate $idate --edate $edate --app $app --start $starttype --gfs_cyc $gfscyc --resdet $resdet --resens $resens --nens $nens --pslot $pslot --configdir $configdir --comrot $comrot --expdir $expdir exit $? From 40cf983537aa838e2928d48760ae112d2b47f3e2 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 10 May 2023 16:09:55 +0000 Subject: [PATCH 07/18] another attempt to fix CI --- test/atm/global-workflow/jjob_ens_init.sh | 2 +- test/atm/global-workflow/jjob_var_init.sh | 2 +- test/atm/global-workflow/setup_workflow_exp.sh | 2 +- test/setup_workflow_exp.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/atm/global-workflow/jjob_ens_init.sh b/test/atm/global-workflow/jjob_ens_init.sh index 8a1aec8f4..bd04fe82d 100755 --- a/test/atm/global-workflow/jjob_ens_init.sh +++ b/test/atm/global-workflow/jjob_ens_init.sh @@ -26,7 +26,7 @@ export COM_TOP=$ROTDIR # Set GFS COM paths source "${HOMEgfs}/ush/preamble.sh" -source "${HOMEgfs}/parm/config/config.com" +source "${HOMEgfs}/parm/config/gfs/config.com" # Set python path for workflow utilities and tasks pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" diff --git a/test/atm/global-workflow/jjob_var_init.sh b/test/atm/global-workflow/jjob_var_init.sh index ac0011e75..a0a56f893 100755 --- a/test/atm/global-workflow/jjob_var_init.sh +++ b/test/atm/global-workflow/jjob_var_init.sh @@ -26,7 +26,7 @@ export COM_TOP=$ROTDIR # Set GFS COM paths source "${HOMEgfs}/ush/preamble.sh" -source "${HOMEgfs}/parm/config/config.com" +source "${HOMEgfs}/parm/config/gfs/config.com" # Set python path for workflow utilities and tasks pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" diff --git a/test/atm/global-workflow/setup_workflow_exp.sh b/test/atm/global-workflow/setup_workflow_exp.sh index 06d8e519b..2b51bbd3f 100755 --- a/test/atm/global-workflow/setup_workflow_exp.sh +++ b/test/atm/global-workflow/setup_workflow_exp.sh @@ -14,7 +14,7 @@ resdet='48' resens='48' nens=3 pslot='gdas_test' -configdir=$srcdir/../../parm/config +configdir=$srcdir/../../parm/config/gfs comrot=$bindir/test/atm/global-workflow/testrun/ROTDIRS expdir=$bindir/test/atm/global-workflow/testrun/experiments diff --git a/test/setup_workflow_exp.sh b/test/setup_workflow_exp.sh index 274cbac05..544f7e77a 100755 --- a/test/setup_workflow_exp.sh +++ b/test/setup_workflow_exp.sh @@ -13,7 +13,7 @@ resdet='48' resens='24' nens=0 pslot='gdas_test' -configdir=$srcdir/../../parm/config +configdir=$srcdir/../../parm/config/gfs comrot=$bindir/test/testrun/ROTDIRS expdir=$bindir/test/testrun/experiments From 08e30ed2136668978b0149c41b6c180b60eff239 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 10 May 2023 16:21:32 +0000 Subject: [PATCH 08/18] Small change --- prototypes/configs/primordial_config.sh | 2 +- prototypes/gen_prototype.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/prototypes/configs/primordial_config.sh b/prototypes/configs/primordial_config.sh index 10b80e5fd..fa552c647 100644 --- a/prototypes/configs/primordial_config.sh +++ b/prototypes/configs/primordial_config.sh @@ -7,7 +7,7 @@ DUMPDIR=/scratch1/NCEPDEV/da/role.jedipara/dump/ ICSDIR=/scratch1/NCEPDEV/stmp4/Cory.R.Martin/tmp_ics expdir=$PROTOROOT/expdirs comrot=$PROTOROOT/rotdirs -BUILD="YES" +BUILD="NO" idate=2021073118 edate=2021080118 app="ATM" diff --git a/prototypes/gen_prototype.sh b/prototypes/gen_prototype.sh index a249ea125..51451ce7c 100755 --- a/prototypes/gen_prototype.sh +++ b/prototypes/gen_prototype.sh @@ -92,7 +92,7 @@ cd $GWDIR/global-workflow/workflow --resens $resens \ --nens $nens \ --pslot $PSLOT \ - --configdir $GWDIR/global-workflow/parm/config \ + --configdir $GWDIR/global-workflow/parm/config/gfs \ --comrot $comrot \ --expdir $expdir \ --yaml $expdir/config_${PSLOT}.yaml From 3593ab212612fbffb12ae7ddd904bb98f76c7069 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Thu, 11 May 2023 21:05:04 +0000 Subject: [PATCH 09/18] end of day commit --- .../configs/{primordial_config.sh => cp0.sh} | 14 ++++++++++---- prototypes/gen_prototype.sh | 13 +++++++++---- 2 files changed, 19 insertions(+), 8 deletions(-) rename prototypes/configs/{primordial_config.sh => cp0.sh} (72%) diff --git a/prototypes/configs/primordial_config.sh b/prototypes/configs/cp0.sh similarity index 72% rename from prototypes/configs/primordial_config.sh rename to prototypes/configs/cp0.sh index fa552c647..19d89c5d1 100644 --- a/prototypes/configs/primordial_config.sh +++ b/prototypes/configs/cp0.sh @@ -1,5 +1,7 @@ +# top level options GWHASH=e55f090 GDASHASH=58d0b9b +BUILD="NO" PSLOT=apple PROTOROOT=/scratch1/NCEPDEV/da/Cory.R.Martin/GDAS_prototypes GWDIR=$PROTOROOT/$PSLOT @@ -7,13 +9,17 @@ DUMPDIR=/scratch1/NCEPDEV/da/role.jedipara/dump/ ICSDIR=/scratch1/NCEPDEV/stmp4/Cory.R.Martin/tmp_ics expdir=$PROTOROOT/expdirs comrot=$PROTOROOT/rotdirs -BUILD="NO" idate=2021073118 edate=2021080118 app="ATM" starttype="cold" gfscyc=0 resdet=96 -resens=48 -nens=80 - +resens=96 +nens=0 +# config.* options +DO_JEDIVAR="YES" +DO_JEDIENS="NO" +DO_JEDIOCNVAR="NO" +DO_JEDILANDDA="NO" +DO_MERGENSST="NO" diff --git a/prototypes/gen_prototype.sh b/prototypes/gen_prototype.sh index 51451ce7c..43aa66c1d 100755 --- a/prototypes/gen_prototype.sh +++ b/prototypes/gen_prototype.sh @@ -75,12 +75,13 @@ base: HPSS_PROJECT: "emc-da" HOMEDIR: "/scratch1/NCEPDEV/da/${USER}" DMPDIR: "${DUMPDIR}" -atmanl: - OBS_LIST: "/dev/null" + DO_JEDIVAR: "${DO_JEDIVAR}" + DO_JEDIENS: "${DO_JEDIENS}" + DO_JEDIOCNVAR: "${DO_JEDIOCNVAR}" + DO_JEDILANDDA: "${DO_JEDILANDDA}" + DO_MERGENSST: "${DO_MERGENSST}" EOF - - # setup experiment cd $GWDIR/global-workflow/workflow ./setup_expt.py gfs cycled --idate $idate \ @@ -95,4 +96,8 @@ cd $GWDIR/global-workflow/workflow --configdir $GWDIR/global-workflow/parm/config/gfs \ --comrot $comrot \ --expdir $expdir \ + --icsdir $ICSDIR \ --yaml $expdir/config_${PSLOT}.yaml + +# setup XML for workflow +./setup_xml.py $expdir/$PSLOT From 7e8c5100029dda5acc832e6aa07de04caeab3569 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 12 May 2023 15:19:17 +0000 Subject: [PATCH 10/18] Changes to be consistent with g-w --- prototypes/gen_prototype.sh | 4 ++-- test/atm/global-workflow/config.base.emc.dyn | 4 ++-- test/atm/global-workflow/config.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/prototypes/gen_prototype.sh b/prototypes/gen_prototype.sh index 43aa66c1d..4c6493bb4 100755 --- a/prototypes/gen_prototype.sh +++ b/prototypes/gen_prototype.sh @@ -75,8 +75,8 @@ base: HPSS_PROJECT: "emc-da" HOMEDIR: "/scratch1/NCEPDEV/da/${USER}" DMPDIR: "${DUMPDIR}" - DO_JEDIVAR: "${DO_JEDIVAR}" - DO_JEDIENS: "${DO_JEDIENS}" + DO_JEDIATMVAR: "${DO_JEDIATMVAR}" + DO_JEDIATMENS: "${DO_JEDIATMENS}" DO_JEDIOCNVAR: "${DO_JEDIOCNVAR}" DO_JEDILANDDA: "${DO_JEDILANDDA}" DO_MERGENSST: "${DO_MERGENSST}" diff --git a/test/atm/global-workflow/config.base.emc.dyn b/test/atm/global-workflow/config.base.emc.dyn index 0f295e8d6..fe86e804c 100755 --- a/test/atm/global-workflow/config.base.emc.dyn +++ b/test/atm/global-workflow/config.base.emc.dyn @@ -302,8 +302,8 @@ export imp_physics=@IMP_PHYSICS@ # Shared parameters # DA engine -export DO_JEDIVAR="@DO_JEDIVAR@" -export DO_JEDIENS="@DO_JEDIENS@" +export DO_JEDIATMVAR="@DO_JEDIATMVAR@" +export DO_JEDIATMENS="@DO_JEDIATMENS@" export DO_JEDIOCNVAR="NO" export DO_JEDILANDDA="NO" export DO_MERGENSST="NO" diff --git a/test/atm/global-workflow/config.yaml b/test/atm/global-workflow/config.yaml index f3c699f8b..db7eba063 100644 --- a/test/atm/global-workflow/config.yaml +++ b/test/atm/global-workflow/config.yaml @@ -1,7 +1,7 @@ base: NCDUMP: $CONDA_PREFIX/ncdump - DO_JEDIVAR: "YES" - DO_JEDIENS: "YES" + DO_JEDIATMVAR: "YES" + DO_JEDIATMENS: "YES" UTILROOT: /apps/contrib/NCEP/libs/hpc-stack/intel-2018.4/prod_util/1.2.2 DATAPATH: "@bindir@/test/atm/global-workflow/testrun" DUMPDIR: "@dumpdir@" From 3685fcfbd1f48d72e739dc48dd7a7622ea10653d Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 12 May 2023 15:29:58 +0000 Subject: [PATCH 11/18] typo fix --- prototypes/configs/cp0.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prototypes/configs/cp0.sh b/prototypes/configs/cp0.sh index 19d89c5d1..71754ee43 100644 --- a/prototypes/configs/cp0.sh +++ b/prototypes/configs/cp0.sh @@ -18,8 +18,8 @@ resdet=96 resens=96 nens=0 # config.* options -DO_JEDIVAR="YES" -DO_JEDIENS="NO" +DO_JEDIATMVAR="YES" +DO_JEDIATMENS="NO" DO_JEDIOCNVAR="NO" DO_JEDILANDDA="NO" DO_MERGENSST="NO" From 89ffbc879396ca48faac329c3295738d0d2adba8 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 12 May 2023 15:36:24 +0000 Subject: [PATCH 12/18] Change hashes --- prototypes/configs/cp0.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/prototypes/configs/cp0.sh b/prototypes/configs/cp0.sh index 71754ee43..257ed5f7a 100644 --- a/prototypes/configs/cp0.sh +++ b/prototypes/configs/cp0.sh @@ -1,8 +1,8 @@ # top level options -GWHASH=e55f090 -GDASHASH=58d0b9b +GWHASH=255d99b +GDASHASH=ab6f06d BUILD="NO" -PSLOT=apple +PSLOT=cp0 PROTOROOT=/scratch1/NCEPDEV/da/Cory.R.Martin/GDAS_prototypes GWDIR=$PROTOROOT/$PSLOT DUMPDIR=/scratch1/NCEPDEV/da/role.jedipara/dump/ From ac06b56de4e2d86c65a81c1e4e27b7c14946dcee Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Fri, 12 May 2023 11:57:17 -0400 Subject: [PATCH 13/18] added gfs arg (#471) --- test/soca/gw/run_jjobs.yaml.test | 6 ++++++ ush/soca/run_jjobs.py | 8 ++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/test/soca/gw/run_jjobs.yaml.test b/test/soca/gw/run_jjobs.yaml.test index 47b448d04..5136b2ac8 100644 --- a/test/soca/gw/run_jjobs.yaml.test +++ b/test/soca/gw/run_jjobs.yaml.test @@ -30,6 +30,12 @@ gw environement: OMP_NUM_THREADS: 1 setup_expt config: + base: + DO_JEDIATMVAR: "NO" + DO_JEDIATMENS: "NO" + DO_JEDIOCNVAR: "YES" + DO_JEDILANDDA: "NO" + DO_MERGENSST: "NO" ocnanal: SOCA_INPUT_FIX_DIR: @HOMEgfs@/sorc/gdas.cd/build/soca_static CASE_ANL: C48 diff --git a/ush/soca/run_jjobs.py b/ush/soca/run_jjobs.py index cbb4a3567..19022498d 100755 --- a/ush/soca/run_jjobs.py +++ b/ush/soca/run_jjobs.py @@ -89,7 +89,7 @@ def setupexpt(self): """ # Make a copy of the configs - origconfig = "${HOMEgfs}/parm/config" + origconfig = "${HOMEgfs}/parm/config/gfs" self.f.write("\n") self.f.write("# Make a copy of config\n") self.f.write(f"cp -r {origconfig} .\n") @@ -102,7 +102,7 @@ def setupexpt(self): self.f.write("\n") self.f.write("# Setup the experiment\n") - setupexpt = "${HOMEgfs}/workflow/setup_expt.py cycled " + setupexpt = "${HOMEgfs}/workflow/setup_expt.py gfs cycled " # Most of the args keys are not used to run the jjobs but are needed to run setup_expt.py args = { "idate": "${PDY}${cyc}", @@ -114,7 +114,7 @@ def setupexpt(self): "resens": "24", "nens": "0", "pslot": "${PSLOT}", - "configdir": "${PWD}/config", + "configdir": "${PWD}/config/gfs", "comrot": self.rotdir, "expdir": "${EXPDIRS}", "yaml": "overwrite_defaults.yaml"} @@ -155,7 +155,7 @@ def copy_bkgs(self): print(f"RUN: {self.RUN}") # setup COM variables - self.f.write("source ${HOMEgfs}/parm/config/config.com\n") + self.f.write("source ${HOMEgfs}/parm/config/gfs/config.com\n") self.f.write("source ${HOMEgfs}/ush/preamble.sh\n") self.precom('COM_OCEAN_HISTORY_PREV', 'COM_OCEAN_HISTORY_TMPL') self.precom('COM_ICE_HISTORY_PREV', 'COM_ICE_HISTORY_TMPL') From d6f8eb4d8692a66c007a4aebe61e763e4721e453 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 12 May 2023 16:48:04 +0000 Subject: [PATCH 14/18] Run rocotorun and rocotostat --- prototypes/gen_prototype.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/prototypes/gen_prototype.sh b/prototypes/gen_prototype.sh index 4c6493bb4..d58492e4b 100755 --- a/prototypes/gen_prototype.sh +++ b/prototypes/gen_prototype.sh @@ -101,3 +101,9 @@ cd $GWDIR/global-workflow/workflow # setup XML for workflow ./setup_xml.py $expdir/$PSLOT + +# run rocotorun one time +rocotorun -w $expdir/$PSLOT/${PSLOT}.xml -d $expdir/$PSLOT/${PSLOT}.db + +# run rocotostat on the first cycle to see if things were submitted +rocotostat -w $expdir/$PSLOT/${PSLOT}.xml -d $expdir/$PSLOT/${PSLOT}.db -c ${idate}00 From 753578163fcb7f7862e5018672601cc317bcc6de Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 12 May 2023 18:17:58 +0000 Subject: [PATCH 15/18] comment out bad test --- test/CMakeLists.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 80e63ed7a..220f79c70 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -110,9 +110,10 @@ add_test(NAME test_gdasapp_check_yaml_keys WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) # test to ensure all YAML in repo is valid YAML -add_test(NAME test_gdasapp_check_valid_yaml - COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/test/check_valid_yaml.py ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) +#add_test(NAME test_gdasapp_check_valid_yaml +# COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/test/check_valid_yaml.py ${PROJECT_SOURCE_DIR} +# WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) +# commented out above test because it does not really do as advertised with all these templates which are not valid yaml # test for ush/jediinc2fv3.py add_test(NAME test_gdasapp_jedi_increment_to_fv3 From 95615d463fbf3d8b254b3a201ed2d410408e6d0a Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 12 May 2023 20:19:17 +0000 Subject: [PATCH 16/18] Fix issues russ pointed out in issue --- parm/atm/berror/hybvar_bump.yaml | 2 +- parm/atm/berror/hybvar_gsibec.yaml | 2 +- parm/atm/lgetkf/lgetkf.yaml | 2 +- test/atm/global-workflow/config.base.emc.dyn | 4 ++-- test/atm/global-workflow/jjob_ens_final.sh | 2 +- test/atm/global-workflow/jjob_ens_init.sh | 2 +- test/atm/global-workflow/jjob_ens_run.sh | 2 +- ush/run_jedi_exe.py | 6 +++--- ush/ufsda/stage.py | 4 ++-- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/parm/atm/berror/hybvar_bump.yaml b/parm/atm/berror/hybvar_bump.yaml index 879441e46..11398dccd 100644 --- a/parm/atm/berror/hybvar_bump.yaml +++ b/parm/atm/berror/hybvar_bump.yaml @@ -105,7 +105,7 @@ components: filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc' filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res' pattern: '%mem%' - nmembers: $(NMEM_ENKF) + nmembers: $(NMEM_ENS) zero padding: 3 # localization: # localization method: SABER diff --git a/parm/atm/berror/hybvar_gsibec.yaml b/parm/atm/berror/hybvar_gsibec.yaml index 63c8516f3..c8dfb54b9 100644 --- a/parm/atm/berror/hybvar_gsibec.yaml +++ b/parm/atm/berror/hybvar_gsibec.yaml @@ -42,7 +42,7 @@ components: filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc' filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res' pattern: '%mem%' - nmembers: $(NMEM_ENKF) + nmembers: $(NMEM_ENS) zero padding: 3 localization: localization method: SABER diff --git a/parm/atm/lgetkf/lgetkf.yaml b/parm/atm/lgetkf/lgetkf.yaml index b8a6f9886..d4ee73bf1 100644 --- a/parm/atm/lgetkf/lgetkf.yaml +++ b/parm/atm/lgetkf/lgetkf.yaml @@ -30,7 +30,7 @@ background: filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc' filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res' pattern: '%mem%' - nmembers: $(NMEM_ENKF) + nmembers: $(NMEM_ENS) zero padding: 3 observations: !INC ${OBS_LIST} diff --git a/test/atm/global-workflow/config.base.emc.dyn b/test/atm/global-workflow/config.base.emc.dyn index fe86e804c..bd234a561 100755 --- a/test/atm/global-workflow/config.base.emc.dyn +++ b/test/atm/global-workflow/config.base.emc.dyn @@ -162,7 +162,7 @@ export OPS_RES="C768" # Do not change export LEVS=128 export CASE="@CASECTL@" export CASE_ANL="$CASE" -export CASE_ENKF="@CASEENS@" +export CASE_ENS="@CASEENS@" # TODO: This should not depend on $CASE or $CASE_ENKF # These are the currently available grid-combinations @@ -310,7 +310,7 @@ export DO_MERGENSST="NO" # Hybrid related export DOHYBVAR="@DOHYBVAR@" -export NMEM_ENKF=@NMEM_ENKF@ +export NMEM_ENS=@NMEM_ENS@ export NMEM_EFCS=30 export SMOOTH_ENKF="NO" export l4densvar=".true." diff --git a/test/atm/global-workflow/jjob_ens_final.sh b/test/atm/global-workflow/jjob_ens_final.sh index b1197631d..1835d728a 100755 --- a/test/atm/global-workflow/jjob_ens_final.sh +++ b/test/atm/global-workflow/jjob_ens_final.sh @@ -19,7 +19,7 @@ export COMIN_GES=${bindir}/test/atm/bkg export pid=${pid:-$$} export jobid=$pid export COMROOT=$DATAROOT -export NMEM_ENKF=3 +export NMEM_ENS=3 export ACCOUNT=da-cpu export DO_JEDIENS="YES" diff --git a/test/atm/global-workflow/jjob_ens_init.sh b/test/atm/global-workflow/jjob_ens_init.sh index bd04fe82d..6d743a374 100755 --- a/test/atm/global-workflow/jjob_ens_init.sh +++ b/test/atm/global-workflow/jjob_ens_init.sh @@ -20,7 +20,7 @@ export COMIN_GES=${bindir}/test/atm/bkg export pid=${pid:-$$} export jobid=$pid export COMROOT=$DATAROOT -export NMEM_ENKF=3 +export NMEM_ENS=3 export ACCOUNT=da-cpu export COM_TOP=$ROTDIR diff --git a/test/atm/global-workflow/jjob_ens_run.sh b/test/atm/global-workflow/jjob_ens_run.sh index d3a5899ab..0c5a564b4 100755 --- a/test/atm/global-workflow/jjob_ens_run.sh +++ b/test/atm/global-workflow/jjob_ens_run.sh @@ -22,7 +22,7 @@ export COMIN_GES=${bindir}/test/atm/bkg export pid=${pid:-$$} export jobid=$pid export COMROOT=$DATAROOT -export NMEM_ENKF=3 +export NMEM_ENS=3 export ACCOUNT=da-cpu export DO_JEDIENS="YES" diff --git a/ush/run_jedi_exe.py b/ush/run_jedi_exe.py index 4c4693f6e..62c0d4a76 100755 --- a/ush/run_jedi_exe.py +++ b/ush/run_jedi_exe.py @@ -124,10 +124,10 @@ def run_jedi_exe(yamlconfig): 'COMOUT': os.path.join(workdir, 'obs'), 'CASE': executable_subconfig['case'], 'CASE_ANL': executable_subconfig.get('case_anl', executable_subconfig['case']), - 'CASE_ENKF': executable_subconfig.get('case_enkf', executable_subconfig['case']), + 'CASE_ENS': executable_subconfig.get('case_enkf', executable_subconfig['case']), 'DOHYBVAR': executable_subconfig.get('dohybvar', False), 'LEVS': str(executable_subconfig['levs']), - 'NMEM_ENKF': executable_subconfig.get('nmem', 0), + 'NMEM_ENS': executable_subconfig.get('nmem', 0), 'COMIN_GES_ENS': f"{comin_ges_ens}", 'forecast_steps': calc_fcst_steps(executable_subconfig.get('forecast_step', 'PT6H'), executable_subconfig['atm_window_length']), @@ -153,7 +153,7 @@ def run_jedi_exe(yamlconfig): 'npx_anl': f"{int(os.environ['CASE_ANL'][1:]) + 1}", 'npy_anl': f"{int(os.environ['CASE_ANL'][1:]) + 1}", 'npz_anl': f"{int(os.environ['LEVS']) - 1}", - 'NMEM_ENKF': f"{int(os.environ['NMEM_ENKF'])}", + 'NMEM_ENS': f"{int(os.environ['NMEM_ENS'])}", 'ATM_WINDOW_BEGIN': window_begin, 'ATM_WINDOW_LENGTH': f"PT{assim_freq}H", 'BKG_TSTEP': executable_subconfig.get('forecast_step', 'PT6H'), diff --git a/ush/ufsda/stage.py b/ush/ufsda/stage.py index 32b4d5009..e9be7dfff 100644 --- a/ush/ufsda/stage.py +++ b/ush/ufsda/stage.py @@ -36,7 +36,7 @@ def gdas_fix(input_fix_dir, working_dir, config): # error checking dohybvar = config['DOHYBVAR'] case = config['CASE'] - case_enkf = config['CASE_ENKF'] + case_enkf = config['CASE_ENS'] case_anl = config['CASE_ANL'] if dohybvar and not case_enkf == case_anl: raise ValueError(f"dohybvar is '{dohybvar}' but case_enkf= '{case_enkf}' does not equal case_anl= '{case_anl}'") @@ -430,7 +430,7 @@ def background_ens(config): if not config['DOHYBVAR']: bkgdir = 'bkg' - for imem in range(1, config['NMEM_ENKF']+1): + for imem in range(1, config['NMEM_ENS']+1): memchar = f"mem{imem:03d}" logging.info(f'Stage background_ens {memchar}') rst_dir = os.path.join(config['COMIN_GES_ENS'], memchar, 'atmos', 'RESTART') From 6c72f5a600ae2e82f5c335fd383ccb995cc17d58 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Sat, 13 May 2023 01:28:17 +0000 Subject: [PATCH 17/18] replace NMEM_ENKF with NMEM_ENS in jjobs_ens_init.sh (#467) --- test/atm/global-workflow/jjob_ens_init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/atm/global-workflow/jjob_ens_init.sh b/test/atm/global-workflow/jjob_ens_init.sh index 6d743a374..cd8770574 100755 --- a/test/atm/global-workflow/jjob_ens_init.sh +++ b/test/atm/global-workflow/jjob_ens_init.sh @@ -70,7 +70,7 @@ done # Link tiled ges and atmf006 files to ROTDIR dpath=enkfgdas.$gPDY/$gcyc -for imem in $(seq 1 $NMEM_ENKF); do +for imem in $(seq 1 $NMEM_ENS); do memchar="mem"$(printf %03i $imem) MEMDIR=${memchar} RUN=${RUN} YMD=${gPDY} HH=${gcyc} generate_com -x \ From b06683a53a75246ca0b57df36fdb0aa252616e5d Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Mon, 15 May 2023 19:05:17 +0000 Subject: [PATCH 18/18] modify run_jjobs --- ush/soca/run_jjobs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/soca/run_jjobs.py b/ush/soca/run_jjobs.py index 19022498d..2f4d6c7ac 100755 --- a/ush/soca/run_jjobs.py +++ b/ush/soca/run_jjobs.py @@ -92,7 +92,8 @@ def setupexpt(self): origconfig = "${HOMEgfs}/parm/config/gfs" self.f.write("\n") self.f.write("# Make a copy of config\n") - self.f.write(f"cp -r {origconfig} .\n") + self.f.write(f"mkdir -p config\n") + self.f.write(f"cp -r {origconfig} config/\n") # Dump the configs in a separate yaml file with open("overwrite_defaults.yaml", "w") as f: