From c961e9a44084d6bfc7042bd8e2190bcf9091d256 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Tue, 6 Dec 2022 11:21:36 -0500 Subject: [PATCH] JEDI based Marine DA tasks (#1134) This PR adds the relevant scripts/steps to exercise the JEDI based marine DA eventually needed for GFSv17. See issue #1072 for more details. The branch used for this PR was extracted from a ["rogue" branch](https://github.com/guillaumevernieres/global-workflow/tree/feature/add-soca) that included changes to the model related scripts to allow for WCDA cycling. The testing was done with the coupled `C48/5deg` atmos/ocean model, using the `GSI` for the amos initialization and the JEDI based marine DA system (SOCA) for the ocean initialization. It is probably step 1 of at least 2 steps to get this work to a functional state in develop. I just don't know yet who's responsibility it is to fix the coupled model scripting to make it work in cycled mode. #### A few notes: - Copying some of the `SOCA` fixed files has yet to be coordinated: An environment variable in `config.ocnanal` is pointing to `/scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/72x35x25/`. This needs to be copied elsewhere. - This work as is does not cycle and will require more work on the model side as pointed above - The testing of that ["rogue" branch](https://github.com/guillaumevernieres/global-workflow/tree/feature/add-soca) from which this PR is derived, was only done on `Hera`. Fixes #1072 --- .gitignore | 3 + env/HERA.env | 10 +++ env/ORION.env | 10 +++ jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_POST | 23 ++++++ jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP | 101 ++++++++++++++++++++++++++ jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN | 90 +++++++++++++++++++++++ jobs/rocoto/ocnanalpost.sh | 15 ++++ jobs/rocoto/ocnanalprep.sh | 15 ++++ jobs/rocoto/ocnanalrun.sh | 15 ++++ parm/config/config.base.emc.dyn | 1 + parm/config/config.ocnanal | 25 +++++++ parm/config/config.ocnanalpost | 10 +++ parm/config/config.ocnanalprep | 10 +++ parm/config/config.ocnanalrun | 11 +++ parm/config/config.resources | 83 +++++++++++++++++---- sorc/link_workflow.sh | 28 +++++-- workflow/applications.py | 1 + workflow/rocoto/workflow_tasks.py | 58 +++++++++++++++ 18 files changed, 488 insertions(+), 21 deletions(-) create mode 100755 jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_POST create mode 100755 jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP create mode 100755 jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN create mode 100755 jobs/rocoto/ocnanalpost.sh create mode 100755 jobs/rocoto/ocnanalprep.sh create mode 100755 jobs/rocoto/ocnanalrun.sh create mode 100644 parm/config/config.ocnanal create mode 100644 parm/config/config.ocnanalpost create mode 100644 parm/config/config.ocnanalprep create mode 100644 parm/config/config.ocnanalrun diff --git a/.gitignore b/.gitignore index 926e68c807..9ae382b28b 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,9 @@ __pycache__ .DS_Store .idea/ +# Ignore editor generated backup files +#------------------------------------- +*~ # Ignore folders #------------------- exec/ diff --git a/env/HERA.env b/env/HERA.env index 0b46505554..0f903611c5 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -83,6 +83,16 @@ elif [[ "${step}" = "aeroanlrun" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}" +elif [ ${step} = "ocnanalrun" ]; then + + export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + + nth_max=$((npe_node_max / npe_node_ocnanalrun)) + + export NTHREADS_OCNANAL=${nth_ocnanalrun:-${nth_max}} + [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max} + export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalrun}" + elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then export MKL_NUM_THREADS=4 diff --git a/env/ORION.env b/env/ORION.env index 46c7eb31a6..f15594e1cc 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -83,6 +83,16 @@ elif [[ "${step}" = "aeroanlrun" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}" +elif [ ${step} = "ocnanalrun" ]; then + + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + + nth_max=$((npe_node_max / npe_node_ocnanalrun)) + + export NTHREADS_OCNANAL=${nth_ocnanalrun:-${nth_max}} + [[ $NTHREADS_OCNANAL -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max} + export APRUN_OCNANAL="${launcher} -n ${npe_aeroanlrun}" + elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then export MKL_NUM_THREADS=4 diff --git a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_POST b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_POST new file mode 100755 index 0000000000..6b64e5ab8a --- /dev/null +++ b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_POST @@ -0,0 +1,23 @@ +#!/bin/bash +export STRICT="NO" +source "${HOMEgfs}/ush/preamble.sh" + +############################# +# Source relevant config files +############################# +export EXPDIR=${EXPDIR:-${HOMEgfs}/parm/config} +configs="base ocnanalpost" +config_path=${EXPDIR:-${PACKAGEROOT}/gfs.${gfs_ver}/parm/config} +for config in ${configs}; do + . "${config_path}"/config."${config}" + status=$? + [[ "${status}" -ne 0 ]] && exit "${status}" +done + +########################################## +# Remove the Temporary working directory +########################################## +cd "${DATAROOT}" || exit 1 +[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}" + +exit 0 diff --git a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP new file mode 100755 index 0000000000..9ba0a96ae5 --- /dev/null +++ b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP @@ -0,0 +1,101 @@ +#!/bin/bash +export STRICT="NO" +source "${HOMEgfs}/ush/preamble.sh" + +############################# +# Source relevant config files +############################# +export EXPDIR=${EXPDIR:-${HOMEgfs}/parm/config} +configs="base ocnanal ocnanalprep" +config_path=${EXPDIR:-${PACKAGEROOT}/gfs.${gfs_ver}/parm/config} +for config in ${configs}; do + . "${config_path}"/config."${config}" + status=$? + [[ ${status} -ne 0 ]] && exit "${status}" +done + + +########################################## +# Source machine runtime environment +########################################## +. "${HOMEgfs}"/env/"${machine}".env ocnanalprep +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + + +############################################## +# Obtain unique process id (pid) and make temp directory +############################################## +export pid=${pid:-$$} +export outid=${outid:-"LL${job}"} + +export DATA=${DATA:-${DATAROOT}/ocnanal_${CDATE}} +mkdir -p "${DATA}" +cd "${DATA}" || (echo "${DATA} does not exist. ABORT!"; exit 1) + + +############################################## +# Run setpdy and initialize PDY variables +############################################## +export cycle="t${cyc}z" +setpdy.sh +. ./PDY + + +############################################## +# Determine Job Output Name on System +############################################## +export pgmout="OUTPUT.${pid}" +export pgmerr=errfile + + +############################################## +# Set variables used in the script +############################################## +export CDATE=${CDATE:-${PDY}${cyc}} +export CDUMP=${CDUMP:-${RUN:-"gfs"}} +export COMPONENT=${COMPONENT:-ocean} + +############################################## +# Begin JOB SPECIFIC work +############################################## + +GDATE=$(date +%Y%m%d%H -d "${CDATE:0:8} ${CDATE:8:2} - ${assim_freq} hours") +export GDATE +gPDY=${GDATE:0:8} +export gcyc=${GDATE:8:2} +export GDUMP=${GDUMP:-"gdas"} + +export OPREFIX="${CDUMP}.t${cyc}z." +export GPREFIX="${GDUMP}.t${gcyc}z." +export APREFIX="${CDUMP}.t${cyc}z." +export GSUFFIX=${GSUFFIX:-${SUFFIX}} +export ASUFFIX=${ASUFFIX:-${SUFFIX}} + +export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}} + +mkdir -p "${COMOUT}" + +# COMIN_GES and COMIN_GES_ENS are used in script +export COMIN_GES="${ROTDIR}/${GDUMP}.${gPDY}/${gcyc}/${COMPONENT}" + +############################################################### +# Run relevant script + +EXSCRIPT=${GDASPREPPY:-${HOMEgfs}/sorc/gdas.cd/scripts/exgdas_global_marine_analysis_prep.py} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +exit 0 diff --git a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN new file mode 100755 index 0000000000..374d889ee6 --- /dev/null +++ b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN @@ -0,0 +1,90 @@ +#!/bin/bash +export STRICT="NO" +source "${HOMEgfs}/ush/preamble.sh" + +############################# +# Source relevant config files +############################# +export EXPDIR=${EXPDIR:-${HOMEgfs}/parm/config} +configs="base ocnanal ocnanalrun" +config_path=${EXPDIR:-${PACKAGEROOT}/gfs.${gfs_ver}/parm/config} +for config in ${configs}; do + . "${config_path}"/config."${config}" + status=$? + [[ ${status} -ne 0 ]] && exit "${status}" +done + + +########################################## +# Source machine runtime environment +########################################## +. "${HOMEgfs}"/env/"${machine}".env ocnanalrun +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + + +############################################## +# Obtain unique process id (pid) and make temp directory +############################################## +export pid=${pid:-$$} +export outid=${outid:-"LL${job}"} + +export DATA=${DATA:-${DATAROOT}/ocnanal_${CDATE}} +mkdir -p "${DATA}" +cd "${DATA}" || (echo "${DATA} does not exist. ABORT!"; exit 1) + + +############################################## +# Run setpdy and initialize PDY variables +############################################## +export cycle="t${cyc}z" +setpdy.sh +. ./PDY + + +############################################## +# Determine Job Output Name on System +############################################## +export pgmout="OUTPUT.${pid}" +export pgmerr=errfile + + +############################################## +# Set variables used in the script +############################################## + +export CDUMP=${CDUMP:-${RUN:-"gfs"}} +export COMPONENT=${COMPONENT:-ocean} + +############################################## +# Begin JOB SPECIFIC work +############################################## + +export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}} + +############################################################### +# Run relevant script + +EXSCRIPT=${GDASPREPPY:-${HOMEgfs}/sorc/gdas.cd/scripts/exgdas_global_marine_analysis_run.sh} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +########################################## +# Remove the Temporary working directory +########################################## +cd "${DATAROOT}" || exit 1 +[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}" + +exit 0 diff --git a/jobs/rocoto/ocnanalpost.sh b/jobs/rocoto/ocnanalpost.sh new file mode 100755 index 0000000000..bad01c93f2 --- /dev/null +++ b/jobs/rocoto/ocnanalpost.sh @@ -0,0 +1,15 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source GDASApp modules +module purge +module use "${HOMEgfs}/sorc/gdas.cd/modulefiles" +module load GDAS/"${machine,,}" + +############################################################### +# Execute the JJOB +"${HOMEgfs}"/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_POST +status=$? +exit "${status}" diff --git a/jobs/rocoto/ocnanalprep.sh b/jobs/rocoto/ocnanalprep.sh new file mode 100755 index 0000000000..7cf14a66ce --- /dev/null +++ b/jobs/rocoto/ocnanalprep.sh @@ -0,0 +1,15 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source GDASApp modules +module purge +module use "${HOMEgfs}/sorc/gdas.cd/modulefiles" +module load "GDAS/${machine,,}" + +############################################################### +# Execute the JJOB +"${HOMEgfs}"/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP +status=$? +exit "${status}" diff --git a/jobs/rocoto/ocnanalrun.sh b/jobs/rocoto/ocnanalrun.sh new file mode 100755 index 0000000000..460169a55f --- /dev/null +++ b/jobs/rocoto/ocnanalrun.sh @@ -0,0 +1,15 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source GDASApp modules +module purge +module use "${HOMEgfs}/sorc/gdas.cd/modulefiles" +module load GDAS/"${machine,,}" + +############################################################### +# Execute the JJOB +"${HOMEgfs}"/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN +status=$? +exit "${status}" diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn index 29bdbe9fec..62d7ff59ef 100755 --- a/parm/config/config.base.emc.dyn +++ b/parm/config/config.base.emc.dyn @@ -327,6 +327,7 @@ export imp_physics=@IMP_PHYSICS@ # DA engine export DO_JEDIVAR="NO" export DO_JEDIENS="NO" +export DO_JEDIOCNVAR="NO" # Hybrid related export DOHYBVAR="YES" diff --git a/parm/config/config.ocnanal b/parm/config/config.ocnanal new file mode 100644 index 0000000000..385db2dc86 --- /dev/null +++ b/parm/config/config.ocnanal @@ -0,0 +1,25 @@ +#!/bin/bash + +########## config.ocnanal ########## +# configuration common to all atm analysis tasks + +echo "BEGIN: config.ocnanal" + +export OBS_YAML_DIR=${HOMEgfs}/sorc/gdas.cd/parm/soca/obs/config +export OBS_YAML=${HOMEgfs}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml +export OBS_LIST=${OBS_YAML_DIR}/obs_list.yaml +export FV3JEDI_STAGE_YAML=${HOMEgfs}/sorc/gdas.cd/test/soca/testinput/dumy.yaml +export SOCA_INPUT_FIX_DIR=/scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/72x35x25/soca # TODO: Should be moved somewhere else +export SOCA_VARS=tocn,socn,ssh +export SOCA_NINNER=50 +export CASE_ANL="C48" +export DOMAIN_STACK_SIZE=2000000 +export JEDI_BIN=${HOMEgfs}/sorc/gdas.cd/build/bin + +# TODO: Move the R2D2 configuration to a common space +export R2D2_OBS_DB=shared +export R2D2_OBS_DUMP=s2s_v1 +export R2D2_OBS_SRC=gdas_marine +export R2D2_OBS_WINDOW=24 # TODO: Check if the R2D2 sampling DB window is still needed +export COMIN_OBS=/scratch2/NCEPDEV/marineda/r2d2 +echo "END: config.ocnanal" diff --git a/parm/config/config.ocnanalpost b/parm/config/config.ocnanalpost new file mode 100644 index 0000000000..a67f07de22 --- /dev/null +++ b/parm/config/config.ocnanalpost @@ -0,0 +1,10 @@ +#!/bin/bash + +########## config.ocnanalpost ########## +# Post Ocn Analysis specific + +echo "BEGIN: config.ocnanalpost" + +# Get task specific resources +. "${EXPDIR}/config.resources ocnanalpost" +echo "END: config.ocnanalpost" diff --git a/parm/config/config.ocnanalprep b/parm/config/config.ocnanalprep new file mode 100644 index 0000000000..66b41463a8 --- /dev/null +++ b/parm/config/config.ocnanalprep @@ -0,0 +1,10 @@ +#!/bin/bash + +########## config.ocnanalprep ########## +# Pre Ocn Analysis specific + +echo "BEGIN: config.ocnanalprep" + +# Get task specific resources +. "${EXPDIR}/config.resources ocnanalprep" +echo "END: config.ocnanalprep" diff --git a/parm/config/config.ocnanalrun b/parm/config/config.ocnanalrun new file mode 100644 index 0000000000..a71c3066f6 --- /dev/null +++ b/parm/config/config.ocnanalrun @@ -0,0 +1,11 @@ +#!/bin/bash + +########## config.ocnanalrun ########## +# Ocn Analysis specific + +echo "BEGIN: config.ocnanalrun" + +# Get task specific resources +. "${EXPDIR}/config.resources ocnanalrun" + +echo "END: config.ocnanalrun" diff --git a/parm/config/config.resources b/parm/config/config.resources index 875470e0c8..cdec42bc86 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -156,45 +156,51 @@ elif [ ${step} = "waveawipsgridded" ]; then export NTASKS=${npe_waveawipsgridded} export memory_waveawipsgridded_gfs="1GB" -elif [ ${step} = "atmanalprep" ]; then +elif [[ "${step}" = "atmanalprep" ]]; then export wtime_atmanalprep="00:10:00" export npe_atmanalprep=1 export nth_atmanalprep=1 - export npe_node_atmanalprep=$(echo "${npe_node_max} / ${nth_atmanalprep}" | bc) + npe_node_atmanalprep=$(echo "${npe_node_max} / ${nth_atmanalprep}" | bc) + export npe_node_atmanalprep export memory_atmanalprep="3072M" -elif [ ${step} = "atmanalrun" ]; then +elif [[ "${step}" = "atmanalrun" ]]; then # make below case dependent later export layout_x=1 export layout_y=1 export wtime_atmanalrun="00:30:00" - export npe_atmanalrun=$(echo "${layout_x} * ${layout_y} * 6" | bc) - export npe_atmanalrun_gfs=$(echo "${layout_x} * ${layout_y} * 6" | bc) + npe_atmanalrun=$(echo "${layout_x} * ${layout_y} * 6" | bc) + export npe_atmanalrun + npe_atmanalrun_gfs=$(echo "${layout_x} * ${layout_y} * 6" | bc) + export npe_atmanalrun_gfs export nth_atmanalrun=1 export nth_atmanalrun_gfs=${nth_atmanalrun} - export npe_node_atmanalrun=$(echo "${npe_node_max} / ${nth_atmanalrun}" | bc) + npe_node_atmanalrun=$(echo "${npe_node_max} / ${nth_atmanalrun}" | bc) + export npe_node_atmanalrun export is_exclusive=True -elif [ ${step} = "atmanalpost" ]; then +elif [[ "${step}" = "atmanalpost" ]]; then export wtime_atmanalpost="00:30:00" export npe_atmanalpost=${npe_node_max} export nth_atmanalpost=1 - export npe_node_atmanalpost=$(echo "${npe_node_max} / ${nth_atmanalpost}" | bc) + npe_node_atmanalpost=$(echo "${npe_node_max} / ${nth_atmanalpost}" | bc) + export npe_node_atmanalpost export is_exclusive=True -elif [ ${step} = "aeroanlinit" ]; then +elif [[ "${step}" = "aeroanlinit" ]]; then export wtime_aeroanlinit="00:10:00" export npe_aeroanlinit=1 export nth_aeroanlinit=1 - export npe_node_aeroanlinit=$(echo "${npe_node_max} / ${nth_aeroanlinit}" | bc) + npe_node_aeroanlinit=$(echo "${npe_node_max} / ${nth_aeroanlinit}" | bc) + export npe_node_aeroanlinit export memory_aeroanlinit="3072M" -elif [ ${step} = "aeroanlrun" ]; then +elif [[ "${step}" = "aeroanlrun" ]]; then case ${CASE} in C768) @@ -209,24 +215,69 @@ elif [ ${step} = "aeroanlrun" ]; then layout_x=3 layout_y=3 ;; + *) + echo "FATAL: Resolution not supported'" + exit 1 esac export wtime_aeroanlrun="00:30:00" - export npe_aeroanlrun=$(echo "${layout_x} * ${layout_y} * 6" | bc) - export npe_aeroanlrun_gfs=$(echo "${layout_x} * ${layout_y} * 6" | bc) + npe_aeroanlrun=$(echo "${layout_x} * ${layout_y} * 6" | bc) + export npe_aeroanlrun + npe_aeroanlrun_gfs=$(echo "${layout_x} * ${layout_y} * 6" | bc) + export npe_aeroanlrun_gfs export nth_aeroanlrun=1 export nth_aeroanlrun_gfs=1 - export npe_node_aeroanlrun=$(echo "${npe_node_max} / ${nth_aeroanlrun}" | bc) + npe_node_aeroanlrun=$(echo "${npe_node_max} / ${nth_aeroanlrun}" | bc) + export npe_node_aeroanlrun export is_exclusive=True -elif [ ${step} = "aeroanlfinal" ]; then +elif [[ "${step}" = "aeroanlfinal" ]]; then export wtime_aeroanlfinal="00:10:00" export npe_aeroanlfinal=1 export nth_aeroanlfinal=1 - export npe_node_aeroanlfinal=$(echo "${npe_node_max} / ${nth_aeroanlfinal}" | bc) + npe_node_aeroanlfinal=$(echo "${npe_node_max} / ${nth_aeroanlfinal}" | bc) + export npe_node_aeroanlfinal export memory_aeroanlfinal="3072M" +elif [[ "${step}" = "ocnanalprep" ]]; then + + export wtime_ocnanalprep="00:10:00" + export npe_ocnanalprep=1 + export nth_ocnanalprep=1 + npe_node_ocnanalprep=$(echo "${npe_node_max} / ${nth_ocnanalprep}" | bc) + export npe_node_ocnanalprep + export memory_ocnanalprep="3072M" + +elif [[ "${step}" = "ocnanalrun" ]]; then + npes=16 + case ${CASE} in + C384) + npes=480 + ;; + C48) + npes=16 + ;; + *) + echo "FATAL: Resolution not supported'" + exit 1 + esac + + export wtime_ocnanalrun="00:30:00" + export npe_ocnanalrun=${npes} + export nth_ocnanalrun=1 + export native_ocnanalrun="--exclusive" + npe_node_ocnanalrun=$(echo "${npe_node_max} / ${nth_ocnanalrun}" | bc) + export npe_node_ocnanalrun + +elif [[ "${step}" = "ocnanalpost" ]]; then + + export wtime_ocnanalpost="00:30:00" + export npe_ocnanalpost=${npe_node_max} + export nth_ocnanalpost=1 + npe_node_ocnanalpost=$(echo "${npe_node_max} / ${nth_ocnanalpost}" | bc) + export npe_node_ocnanalpost + elif [ ${step} = "anal" ]; then export wtime_anal="00:50:00" diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index ebee40b17e..cf730b6749 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -288,10 +288,29 @@ fi # GDASApp if [[ -d "${script_dir}/gdas.cd" ]]; then - for gdasexe in fv3jedi_addincrement.x fv3jedi_diffstates.x fv3jedi_ensvariance.x fv3jedi_hofx.x \ - fv3jedi_var.x fv3jedi_convertincrement.x fv3jedi_dirac.x fv3jedi_error_covariance_training.x \ - fv3jedi_letkf.x fv3jedi_convertstate.x fv3jedi_eda.x fv3jedi_forecast.x fv3jedi_plot_field.x \ - fv3jedi_data_checker.py fv3jedi_enshofx.x fv3jedi_hofx_nomodel.x fv3jedi_testdata_downloader.py; do + declare -a JEDI_EXE=("fv3jedi_addincrement.x" \ + "fv3jedi_diffstates.x" \ + "fv3jedi_ensvariance.x" \ + "fv3jedi_hofx.x" \ + "fv3jedi_var.x" \ + "fv3jedi_convertincrement.x" \ + "fv3jedi_dirac.x" \ + "fv3jedi_error_covariance_training.x" \ + "fv3jedi_letkf.x" \ + "fv3jedi_convertstate.x" \ + "fv3jedi_eda.x" \ + "fv3jedi_forecast.x" \ + "fv3jedi_plot_field.x" \ + "fv3jedi_data_checker.py" \ + "fv3jedi_enshofx.x" \ + "fv3jedi_hofx_nomodel.x" \ + "fv3jedi_testdata_downloader.py" \ + "soca_convertincrement.x" \ + "soca_error_covariance_training.x" \ + "soca_setcorscales.x" \ + "soca_gridgen.x" \ + "soca_var.x") + for gdasexe in "${JEDI_EXE[@]}"; do [[ -s "${gdasexe}" ]] && rm -f "${gdasexe}" ${LINK} "${script_dir}/gdas.cd/build/bin/${gdasexe}" . done @@ -427,4 +446,3 @@ fi echo "${BASH_SOURCE[0]} completed successfully" exit 0 - diff --git a/workflow/applications.py b/workflow/applications.py index d2776f7f7b..99e236b44d 100644 --- a/workflow/applications.py +++ b/workflow/applications.py @@ -108,6 +108,7 @@ def __init__(self, configuration: Configuration) -> None: self.do_metp = _base.get('DO_METP', False) self.do_jedivar = _base.get('DO_JEDIVAR', False) self.do_jediens = _base.get('DO_JEDIENS', False) + self.do_jediocnvar = _base.get('DO_JEDIOCNVAR', False) self.do_hpssarch = _base.get('HPSSARCH', False) diff --git a/workflow/rocoto/workflow_tasks.py b/workflow/rocoto/workflow_tasks.py index 6273c16491..e953ede3c9 100644 --- a/workflow/rocoto/workflow_tasks.py +++ b/workflow/rocoto/workflow_tasks.py @@ -13,6 +13,7 @@ class Tasks: VALID_TASKS = ['aerosol_init', 'coupled_ic', 'getic', 'init', 'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'gldas', 'arch', 'atmanalprep', 'atmanalrun', 'atmanalpost', + 'ocnanalprep', 'ocnanalrun', 'ocnanalpost', 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', 'atmensanalprep', 'atmensanalrun', 'atmensanalpost', @@ -496,6 +497,58 @@ def aeroanlfinal(self): return task + def ocnanalprep(self): + + suffix = self._base["SUFFIX"] + dump_suffix = self._base["DUMP_SUFFIX"] + dmpdir = self._base["DMPDIR"] + + deps = [] + data = f'&ROTDIR;/gdas.@Y@m@d/@H/ocean/gdas.t@Hz.ocnf009{suffix}' + dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + resources = self.get_resource('ocnanalprep') + task = create_wf_task('ocnanalprep', + resources, + cdump=self.cdump, + envar=self.envars, + dependency=dependencies) + + return task + + def ocnanalrun(self): + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}ocnanalprep'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + resources = self.get_resource('ocnanalrun') + task = create_wf_task('ocnanalrun', + resources, + cdump=self.cdump, + envar=self.envars, + dependency=dependencies) + + return task + + def ocnanalpost(self): + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}ocnanalrun'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + resources = self.get_resource('ocnanalpost') + task = create_wf_task('ocnanalpost', + resources, + cdump=self.cdump, + envar=self.envars, + dependency=dependencies) + + return task def gldas(self): deps = [] @@ -580,6 +633,11 @@ def _fcst_cycled(self): dep = rocoto.add_dependency(dep_dict) dependencies = rocoto.create_dependency(dep=dep) + if self.app_config.do_jediocnvar: + dep_dict = {'type': 'task', 'name': f'{self.cdump}ocnanalrun'} + dep = rocoto.add_dependency(dep_dict) + dependencies = rocoto.create_dependency(dep=dep) + if self.app_config.do_gldas and self.cdump in ['gdas']: dep_dict = {'type': 'task', 'name': f'{self.cdump}gldas'} dependencies.append(rocoto.add_dependency(dep_dict))