From f78afebc3125dbfb9a8450b47ad5c72a5a6c16a8 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Wed, 25 Jan 2023 15:11:05 -0500 Subject: [PATCH] More marine DA j-jobs (#1270) Addition of 2 j-jobs: - jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT_VRFY This job currently dumps a few files of the impulse response of B. Not meant to ever go in ops. - jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY This job will eventually generate relevant figures from the marine DA cycle. It currently points to a script that does not yet exist. I also took this PR opportunity to: - update how we load the modules, following what was done for the atmosphere. - modify ```ush/load_ufsda_modules.sh``` so we could optionally specify what to load (GDAS or EVA), default is GDAS when no argument is present - Use POST to copy what's needed in `COM` Fixes #1273 --- jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT_VRFY | 45 ++++++++++++++++++++++ jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_POST | 38 +++++++++++++++++- jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN | 11 +----- jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY | 45 ++++++++++++++++++++++ jobs/rocoto/ocnanalpost.sh | 8 ++-- jobs/rocoto/ocnanalprep.sh | 9 +++-- jobs/rocoto/ocnanalrun.sh | 8 ++-- jobs/rocoto/ocnanalvrfy.sh | 19 +++++++++ ush/load_ufsda_modules.sh | 21 +++++++++- 9 files changed, 180 insertions(+), 24 deletions(-) create mode 100755 jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT_VRFY create mode 100755 jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY create mode 100755 jobs/rocoto/ocnanalvrfy.sh diff --git a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT_VRFY b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT_VRFY new file mode 100755 index 0000000000..145d3032f8 --- /dev/null +++ b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT_VRFY @@ -0,0 +1,45 @@ +#!/bin/bash +export STRICT="NO" +source "${HOMEgfs}/ush/preamble.sh" + + +export DATA="${DATAROOT}/${RUN}ocnanal_${cyc}" +source "${HOMEgfs}/ush/jjob_header.sh" -e "ocnanalrun" -c "base ocnanal ocnanalrun" + + +############################################## +# Set variables used in the script +############################################## + + +############################################## +# Begin JOB SPECIFIC work +############################################## + +export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/ocean} + +############################################################### +# Run relevant script + +EXSCRIPT=${GDASPREPPY:-${HOMEgfs}/sorc/gdas.cd/scripts/exgdas_global_marine_analysis_bmat_vrfy.sh} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +########################################## +# Do not remove the Temporary working directory (do this in POST) +########################################## +cd "${DATAROOT}" || exit 1 + +exit 0 diff --git a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_POST b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_POST index f050f197f4..7ab3eb0028 100755 --- a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_POST +++ b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_POST @@ -6,13 +6,49 @@ DATA="${DATAROOT}/${RUN}ocnanal_${cyc}" source "${HOMEgfs}/ush/jjob_header.sh" -e "ocnanalpost" -c "base ocnanalpost" +############################################## +# Begin JOB SPECIFIC work +############################################## + +export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/ocean} +export CDATE=${CDATE:-${PDY}${cyc}} + +mkdir -p "${COMOUT}" + ############################################################### # Run relevant script ############################################################### +# Save some of the DA cycle output to COMOUT +# TODO: Move to a dedicated script + +# Make a copy the IAU increment +cp "${DATA}/inc.nc" "${COMOUT}/${CDUMP}.t${cyc}z.ocninc.nc" + +# TODO: Dump-splash of the sea-ice restart not done yet + +# Copy of the ioda output files, as is for now +cp -r "${DATA}/diags" "${COMOUT}" + +# Copy of the diagonal of the background error for the cycle +bdate=$(date -d "${CDATE:0:8} ${CDATE:8:2} - 3 hours" +"%Y-%m-%dT%H:00:00Z") +cp "${DATA}/ocn.bkgerr_stddev.incr.${bdate}.nc" "${COMOUT}/${CDUMP}.t${cyc}z.ocn.bkgerr_stddev.nc" +cp "${DATA}/ice.bkgerr_stddev.incr.${bdate}.nc" "${COMOUT}/${CDUMP}.t${cyc}z.ice.bkgerr_stddev.nc" + +# Copy the loacalization and correlation operators +cp -rL "${DATA}/bump" "${COMOUT}/bump" + +# Copy the analysis in the middle of the window +cdate=$(date -d "${CDATE:0:8} ${CDATE:8:2}" +"%Y-%m-%dT%H:00:00Z") +cp "${DATA}/Data/ocn.3dvarfgat_pseudo.an.${cdate}.nc" "${COMOUT}/${CDUMP}.t${cyc}z.ocnana.nc" -# TODO (#982) +# Copy DA grid (computed for the start of the window) +bcyc=$(((cyc - 3 + 24) % 24)) +cp "${DATA}/soca_gridspec.nc" "${COMOUT}/${CDUMP}.t${bcyc}z.ocngrid.nc" +# Copy logs +mkdir -p "${COMOUT}/logs" +cp "${DATA}/*.out" "${COMOUT}/logs" ########################################## # Remove the Temporary working directory diff --git a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN index 4ad58366e6..893719fa9f 100755 --- a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN +++ b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN @@ -2,22 +2,15 @@ export STRICT="NO" source "${HOMEgfs}/ush/preamble.sh" WIPE_DATA="NO" -DATA="${DATAROOT}/${RUN}ocnanal_${cyc}" +export DATA="${DATAROOT}/${RUN}ocnanal_${cyc}" source "${HOMEgfs}/ush/jjob_header.sh" -e "ocnanalrun" -c "base ocnanal ocnanalrun" -############################################## -# Set variables used in the script -############################################## - -export CDUMP=${CDUMP:-${RUN:-"gfs"}} -export COMPONENT="ocean" - ############################################## # Begin JOB SPECIFIC work ############################################## -export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}} +export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/ocean} ############################################################### # Run relevant script diff --git a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY new file mode 100755 index 0000000000..9c6ad5e7b9 --- /dev/null +++ b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY @@ -0,0 +1,45 @@ +#!/bin/bash +export STRICT="NO" +source "${HOMEgfs}/ush/preamble.sh" +source "${HOMEgfs}/ush/jjob_header.sh" -e "ocnanalprep" -c "base ocnanal ocnanalprep" + + +############################################## +# Set variables used in the script +############################################## + + +############################################## +# Begin JOB SPECIFIC work +############################################## + +export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/ocean} + +# Add UFSDA to PYTHONPATH +export PYTHONPATH=${HOMEgfs}/sorc/gdas.cd/ush/:${PYTHONPATH} + +############################################################### +# Run relevant script + +EXSCRIPT=${GDASPREPPY:-${HOMEgfs}/sorc/gdas.cd/scripts/exgdas_global_marine_analysis_vrfy.py} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +########################################## +# Do not remove the Temporary working directory (do this in POST) +########################################## +cd "${DATAROOT}" || exit 1 + +exit 0 diff --git a/jobs/rocoto/ocnanalpost.sh b/jobs/rocoto/ocnanalpost.sh index 5dead927bd..b99a4e05ca 100755 --- a/jobs/rocoto/ocnanalpost.sh +++ b/jobs/rocoto/ocnanalpost.sh @@ -3,10 +3,10 @@ source "${HOMEgfs}/ush/preamble.sh" ############################################################### -# Source GDASApp modules -module purge -module use "${HOMEgfs}/sorc/gdas.cd/modulefiles" -module load GDAS/"${machine,,}" +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" export job="ocnanalpost" export jobid="${job}.$$" diff --git a/jobs/rocoto/ocnanalprep.sh b/jobs/rocoto/ocnanalprep.sh index 0771872629..3830fe1c39 100755 --- a/jobs/rocoto/ocnanalprep.sh +++ b/jobs/rocoto/ocnanalprep.sh @@ -1,12 +1,13 @@ #! /usr/bin/env bash +export STRICT="NO" source "${HOMEgfs}/ush/preamble.sh" ############################################################### -# Source GDASApp modules -module purge -module use "${HOMEgfs}/sorc/gdas.cd/modulefiles" -module load "GDAS/${machine,,}" +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" export job="ocnanalprep" export jobid="${job}.$$" diff --git a/jobs/rocoto/ocnanalrun.sh b/jobs/rocoto/ocnanalrun.sh index cd57d13c44..5f998af989 100755 --- a/jobs/rocoto/ocnanalrun.sh +++ b/jobs/rocoto/ocnanalrun.sh @@ -3,10 +3,10 @@ source "${HOMEgfs}/ush/preamble.sh" ############################################################### -# Source GDASApp modules -module purge -module use "${HOMEgfs}/sorc/gdas.cd/modulefiles" -module load GDAS/"${machine,,}" +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" export job="ocnanalrun" export jobid="${job}.$$" diff --git a/jobs/rocoto/ocnanalvrfy.sh b/jobs/rocoto/ocnanalvrfy.sh new file mode 100755 index 0000000000..d8e9bbb805 --- /dev/null +++ b/jobs/rocoto/ocnanalvrfy.sh @@ -0,0 +1,19 @@ +#! /usr/bin/env bash + +export STRICT="NO" +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" --eva +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +export job="ocnanalvrfy" +export jobid="${job}.$$" + +############################################################### +# Execute the JJOB +"${HOMEgfs}/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY" +status=$? +exit "${status}" diff --git a/ush/load_ufsda_modules.sh b/ush/load_ufsda_modules.sh index 6ceba33706..1c17b073b9 100755 --- a/ush/load_ufsda_modules.sh +++ b/ush/load_ufsda_modules.sh @@ -6,6 +6,23 @@ if [[ "${DEBUG_WORKFLOW:-NO}" == "NO" ]]; then set +x fi +# Read optional module argument, default is to use GDAS +MODS="GDAS" +if [[ $# -gt 0 ]]; then + case "$1" in + --eva) + MODS="EVA" + ;; + --gdas) + MODS="GDAS" + ;; + *) + echo "Invalid option: $1" >&2 + exit 1 + ;; + esac +fi + # Setup runtime environment by loading modules ulimit_s=$( ulimit -S -s ) @@ -23,14 +40,14 @@ elif [[ -d /lfs3 ]] ; then echo WARNING: UFSDA NOT SUPPORTED ON THIS PLATFORM elif [[ -d /scratch1 ]] ; then # We are on NOAA Hera - module load GDAS/hera + module load "${MODS}/hera" if [[ "${DEBUG_WORKFLOW}" == "YES" ]] ; then module list pip list fi elif [[ -d /work ]] ; then # We are on MSU Orion - module load GDAS/orion + module load "${MODS}/orion" if [[ "${DEBUG_WORKFLOW}" == "YES" ]] ; then module list pip list