More marine DA j-jobs (#1270)

Addition of 2 j-jobs:
- jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT_VRFY
This job currently dumps a few files of the impulse response of B. Not meant to ever go in ops.
- jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY
This job will eventually generate relevant figures from the marine DA cycle. It currently points to a script that does not yet exist.

I also took this PR opportunity to:
- update how we load the modules, following what was done for the atmosphere.
- modify ```ush/load_ufsda_modules.sh``` so we could optionally specify what to load (GDAS or EVA), default is GDAS when no argument is present 
- Use POST to copy what's needed in `COM`

Fixes #1273

jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT_VRFY

@@ -0,0 +1,45 @@
+#!/bin/bash
+export STRICT="NO"
+source "${HOMEgfs}/ush/preamble.sh"
+
+
+export DATA="${DATAROOT}/${RUN}ocnanal_${cyc}"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "ocnanalrun" -c "base ocnanal ocnanalrun"
+
+
+##############################################
+# Set variables used in the script
+##############################################
+
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/ocean}
+
+###############################################################
+# Run relevant script
+
+EXSCRIPT=${GDASPREPPY:-${HOMEgfs}/sorc/gdas.cd/scripts/exgdas_global_marine_analysis_bmat_vrfy.sh}
+${EXSCRIPT}
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+##########################################
+# Do not remove the Temporary working directory (do this in POST)
+##########################################
+cd "${DATAROOT}" || exit 1
+
+exit 0

jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_POST

@@ -6,13 +6,49 @@ DATA="${DATAROOT}/${RUN}ocnanal_${cyc}"
 source "${HOMEgfs}/ush/jjob_header.sh" -e "ocnanalpost" -c "base ocnanalpost"
 
 
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/ocean}
+export CDATE=${CDATE:-${PDY}${cyc}}
+
+mkdir -p "${COMOUT}"
+
 ###############################################################
 # Run relevant script
 ###############################################################
 
+# Save some of the DA cycle output to COMOUT
+# TODO: Move to a dedicated script
+
+# Make a copy the IAU increment
+cp "${DATA}/inc.nc" "${COMOUT}/${CDUMP}.t${cyc}z.ocninc.nc"
+
+# TODO: Dump-splash of the sea-ice restart not done yet
+
+# Copy of the ioda output files, as is for now
+cp -r "${DATA}/diags" "${COMOUT}"
+
+# Copy of the diagonal of the background error for the cycle
+bdate=$(date -d "${CDATE:0:8} ${CDATE:8:2} - 3 hours" +"%Y-%m-%dT%H:00:00Z")
+cp "${DATA}/ocn.bkgerr_stddev.incr.${bdate}.nc" "${COMOUT}/${CDUMP}.t${cyc}z.ocn.bkgerr_stddev.nc"
+cp "${DATA}/ice.bkgerr_stddev.incr.${bdate}.nc" "${COMOUT}/${CDUMP}.t${cyc}z.ice.bkgerr_stddev.nc"
+
+# Copy the loacalization and correlation operators
+cp -rL "${DATA}/bump" "${COMOUT}/bump"
+
+# Copy the analysis in the middle of the window
+cdate=$(date -d "${CDATE:0:8} ${CDATE:8:2}" +"%Y-%m-%dT%H:00:00Z")
+cp "${DATA}/Data/ocn.3dvarfgat_pseudo.an.${cdate}.nc" "${COMOUT}/${CDUMP}.t${cyc}z.ocnana.nc"
 
-# TODO (#982)
+# Copy DA grid (computed for the start of the window)
+bcyc=$(((cyc - 3 + 24) % 24))
+cp "${DATA}/soca_gridspec.nc" "${COMOUT}/${CDUMP}.t${bcyc}z.ocngrid.nc"
 
+# Copy logs
+mkdir -p "${COMOUT}/logs"
+cp "${DATA}/*.out" "${COMOUT}/logs"
 
 ##########################################
 # Remove the Temporary working directory

jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN

@@ -2,22 +2,15 @@
 export STRICT="NO"
 source "${HOMEgfs}/ush/preamble.sh"
 WIPE_DATA="NO"
-DATA="${DATAROOT}/${RUN}ocnanal_${cyc}"
+export DATA="${DATAROOT}/${RUN}ocnanal_${cyc}"
 source "${HOMEgfs}/ush/jjob_header.sh" -e "ocnanalrun" -c "base ocnanal ocnanalrun"
 
 
-##############################################
-# Set variables used in the script
-##############################################
-
-export CDUMP=${CDUMP:-${RUN:-"gfs"}}
-export COMPONENT="ocean"
-
 ##############################################
 # Begin JOB SPECIFIC work
 ##############################################
 
-export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}}
+export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/ocean}
 
 ###############################################################
 # Run relevant script

jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY

@@ -0,0 +1,45 @@
+#!/bin/bash
+export STRICT="NO"
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "ocnanalprep" -c "base ocnanal ocnanalprep"
+
+
+##############################################
+# Set variables used in the script
+##############################################
+
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/ocean}
+
+# Add UFSDA to PYTHONPATH
+export PYTHONPATH=${HOMEgfs}/sorc/gdas.cd/ush/:${PYTHONPATH}
+
+###############################################################
+# Run relevant script
+
+EXSCRIPT=${GDASPREPPY:-${HOMEgfs}/sorc/gdas.cd/scripts/exgdas_global_marine_analysis_vrfy.py}
+${EXSCRIPT}
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+##########################################
+# Do not remove the Temporary working directory (do this in POST)
+##########################################
+cd "${DATAROOT}" || exit 1
+
+exit 0

jobs/rocoto/ocnanalpost.sh

@@ -3,10 +3,10 @@
 source "${HOMEgfs}/ush/preamble.sh"
 
 ###############################################################
-# Source GDASApp modules
-module purge
-module use "${HOMEgfs}/sorc/gdas.cd/modulefiles"
-module load GDAS/"${machine,,}"
+# Source UFSDA workflow modules
+. "${HOMEgfs}/ush/load_ufsda_modules.sh"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
 
 export job="ocnanalpost"
 export jobid="${job}.$$"

jobs/rocoto/ocnanalprep.sh

@@ -1,12 +1,13 @@
 #! /usr/bin/env bash
 
+export STRICT="NO"
 source "${HOMEgfs}/ush/preamble.sh"
 
 ###############################################################
-# Source GDASApp modules
-module purge
-module use "${HOMEgfs}/sorc/gdas.cd/modulefiles"
-module load "GDAS/${machine,,}"
+# Source UFSDA workflow modules
+. "${HOMEgfs}/ush/load_ufsda_modules.sh"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
 
 export job="ocnanalprep"
 export jobid="${job}.$$"

jobs/rocoto/ocnanalrun.sh

@@ -3,10 +3,10 @@
 source "${HOMEgfs}/ush/preamble.sh"
 
 ###############################################################
-# Source GDASApp modules
-module purge
-module use "${HOMEgfs}/sorc/gdas.cd/modulefiles"
-module load GDAS/"${machine,,}"
+# Source UFSDA workflow modules
+. "${HOMEgfs}/ush/load_ufsda_modules.sh"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
 
 export job="ocnanalrun"
 export jobid="${job}.$$"

jobs/rocoto/ocnanalvrfy.sh

@@ -0,0 +1,19 @@
+#! /usr/bin/env bash
+
+export STRICT="NO"
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source UFSDA workflow modules
+. "${HOMEgfs}/ush/load_ufsda_modules.sh" --eva
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+export job="ocnanalvrfy"
+export jobid="${job}.$$"
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY"
+status=$?
+exit "${status}"

ush/load_ufsda_modules.sh

@@ -6,6 +6,23 @@ if [[ "${DEBUG_WORKFLOW:-NO}" == "NO" ]]; then
     set +x
 fi
 
+# Read optional module argument, default is to use GDAS
+MODS="GDAS"
+if [[ $# -gt 0 ]]; then
+  case "$1" in
+    --eva)
+      MODS="EVA"
+      ;;
+    --gdas)
+      MODS="GDAS"
+      ;;
+    *)
+      echo "Invalid option: $1" >&2
+      exit 1
+      ;;
+  esac
+fi
+
 # Setup runtime environment by loading modules
 ulimit_s=$( ulimit -S -s )
 
@@ -23,14 +40,14 @@ elif [[ -d /lfs3 ]] ; then
   echo WARNING: UFSDA NOT SUPPORTED ON THIS PLATFORM
 elif [[ -d /scratch1 ]] ; then
   # We are on NOAA Hera
-  module load GDAS/hera
+  module load "${MODS}/hera"
   if [[ "${DEBUG_WORKFLOW}" == "YES" ]] ; then
      module list
      pip list
   fi
 elif [[ -d /work ]] ; then
   # We are on MSU Orion
-  module load GDAS/orion
+  module load "${MODS}/orion"
   if [[ "${DEBUG_WORKFLOW}" == "YES" ]] ; then
      module list
      pip list