JEDI based Marine DA tasks (#1134)

This PR adds the relevant scripts/steps to exercise the JEDI based marine DA eventually needed for GFSv17. See issue #1072 for more details. 

The branch used for this PR was extracted from a ["rogue" branch](https://github.com/guillaumevernieres/global-workflow/tree/feature/add-soca) that included changes to the model related scripts to allow for WCDA cycling. The testing was done with the coupled `C48/5deg` atmos/ocean model, using the `GSI` for the amos initialization and the JEDI based marine DA system (SOCA) for the ocean initialization.

It is probably step 1 of at least 2 steps to get this work to a functional state in develop. I just don't know yet who's responsibility it is to fix the coupled model scripting to make it work in cycled mode.

#### A few notes:
- Copying some of the `SOCA` fixed files has yet to be coordinated: An environment variable in `config.ocnanal` is pointing to `/scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/72x35x25/`. This needs to be copied elsewhere.
- This work as is does not cycle and will require more work on the model side as pointed above
- The testing of that ["rogue" branch](https://github.com/guillaumevernieres/global-workflow/tree/feature/add-soca) from which this PR is derived, was only done on `Hera`.

Fixes #1072

.gitignore

@@ -8,6 +8,9 @@ __pycache__
 .DS_Store
 .idea/
 
+# Ignore editor generated backup files
+#-------------------------------------
+*~
 # Ignore folders
 #-------------------
 exec/

env/HERA.env

@@ -83,6 +83,16 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}"
 
+elif [ ${step} = "ocnanalrun" ]; then
+
+    export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+
+    nth_max=$((npe_node_max / npe_node_ocnanalrun))
+
+    export NTHREADS_OCNANAL=${nth_ocnanalrun:-${nth_max}}
+    [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
+    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalrun}"
+
 elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
 
     export MKL_NUM_THREADS=4

env/ORION.env

@@ -83,6 +83,16 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}"
 
+elif [ ${step} = "ocnanalrun" ]; then
+
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    nth_max=$((npe_node_max / npe_node_ocnanalrun))
+
+    export NTHREADS_OCNANAL=${nth_ocnanalrun:-${nth_max}}
+    [[ $NTHREADS_OCNANAL -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
+    export APRUN_OCNANAL="${launcher} -n ${npe_aeroanlrun}"
+
 elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
 
     export MKL_NUM_THREADS=4

jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_POST

@@ -0,0 +1,23 @@
+#!/bin/bash
+export STRICT="NO"
+source "${HOMEgfs}/ush/preamble.sh"
+
+#############################
+# Source relevant config files
+#############################
+export EXPDIR=${EXPDIR:-${HOMEgfs}/parm/config}
+configs="base ocnanalpost"
+config_path=${EXPDIR:-${PACKAGEROOT}/gfs.${gfs_ver}/parm/config}
+for config in ${configs}; do
+    . "${config_path}"/config."${config}"
+    status=$?
+    [[ "${status}" -ne 0 ]] && exit "${status}"
+done
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || exit 1
+[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}"
+
+exit 0

jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP

@@ -0,0 +1,101 @@
+#!/bin/bash
+export STRICT="NO"
+source "${HOMEgfs}/ush/preamble.sh"
+
+#############################
+# Source relevant config files
+#############################
+export EXPDIR=${EXPDIR:-${HOMEgfs}/parm/config}
+configs="base ocnanal ocnanalprep"
+config_path=${EXPDIR:-${PACKAGEROOT}/gfs.${gfs_ver}/parm/config}
+for config in ${configs}; do
+    . "${config_path}"/config."${config}"
+    status=$?
+    [[ ${status} -ne 0 ]] && exit "${status}"
+done
+
+
+##########################################
+# Source machine runtime environment
+##########################################
+. "${HOMEgfs}"/env/"${machine}".env ocnanalprep
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+
+##############################################
+# Obtain unique process id (pid) and make temp directory
+##############################################
+export pid=${pid:-$$}
+export outid=${outid:-"LL${job}"}
+
+export DATA=${DATA:-${DATAROOT}/ocnanal_${CDATE}}
+mkdir -p "${DATA}"
+cd "${DATA}" || (echo "${DATA} does not exist. ABORT!"; exit 1)
+
+
+##############################################
+# Run setpdy and initialize PDY variables
+##############################################
+export cycle="t${cyc}z"
+setpdy.sh
+. ./PDY
+
+
+##############################################
+# Determine Job Output Name on System
+##############################################
+export pgmout="OUTPUT.${pid}"
+export pgmerr=errfile
+
+
+##############################################
+# Set variables used in the script
+##############################################
+export CDATE=${CDATE:-${PDY}${cyc}}
+export CDUMP=${CDUMP:-${RUN:-"gfs"}}
+export COMPONENT=${COMPONENT:-ocean}
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+GDATE=$(date +%Y%m%d%H -d "${CDATE:0:8} ${CDATE:8:2} - ${assim_freq} hours")
+export GDATE
+gPDY=${GDATE:0:8}
+export gcyc=${GDATE:8:2}
+export GDUMP=${GDUMP:-"gdas"}
+
+export OPREFIX="${CDUMP}.t${cyc}z."
+export GPREFIX="${GDUMP}.t${gcyc}z."
+export APREFIX="${CDUMP}.t${cyc}z."
+export GSUFFIX=${GSUFFIX:-${SUFFIX}}
+export ASUFFIX=${ASUFFIX:-${SUFFIX}}
+
+export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}}
+
+mkdir -p "${COMOUT}"
+
+# COMIN_GES and COMIN_GES_ENS are used in script
+export COMIN_GES="${ROTDIR}/${GDUMP}.${gPDY}/${gcyc}/${COMPONENT}"
+
+###############################################################
+# Run relevant script
+
+EXSCRIPT=${GDASPREPPY:-${HOMEgfs}/sorc/gdas.cd/scripts/exgdas_global_marine_analysis_prep.py}
+${EXSCRIPT}
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+exit 0

jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN

@@ -0,0 +1,90 @@
+#!/bin/bash
+export STRICT="NO"
+source "${HOMEgfs}/ush/preamble.sh"
+
+#############################
+# Source relevant config files
+#############################
+export EXPDIR=${EXPDIR:-${HOMEgfs}/parm/config}
+configs="base ocnanal ocnanalrun"
+config_path=${EXPDIR:-${PACKAGEROOT}/gfs.${gfs_ver}/parm/config}
+for config in ${configs}; do
+    . "${config_path}"/config."${config}"
+    status=$?
+    [[ ${status} -ne 0 ]] && exit "${status}"
+done
+
+
+##########################################
+# Source machine runtime environment
+##########################################
+. "${HOMEgfs}"/env/"${machine}".env ocnanalrun
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+
+##############################################
+# Obtain unique process id (pid) and make temp directory
+##############################################
+export pid=${pid:-$$}
+export outid=${outid:-"LL${job}"}
+
+export DATA=${DATA:-${DATAROOT}/ocnanal_${CDATE}}
+mkdir -p "${DATA}"
+cd "${DATA}" || (echo "${DATA} does not exist. ABORT!"; exit 1)
+
+
+##############################################
+# Run setpdy and initialize PDY variables
+##############################################
+export cycle="t${cyc}z"
+setpdy.sh
+. ./PDY
+
+
+##############################################
+# Determine Job Output Name on System
+##############################################
+export pgmout="OUTPUT.${pid}"
+export pgmerr=errfile
+
+
+##############################################
+# Set variables used in the script
+##############################################
+
+export CDUMP=${CDUMP:-${RUN:-"gfs"}}
+export COMPONENT=${COMPONENT:-ocean}
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}}
+
+###############################################################
+# Run relevant script
+
+EXSCRIPT=${GDASPREPPY:-${HOMEgfs}/sorc/gdas.cd/scripts/exgdas_global_marine_analysis_run.sh}
+${EXSCRIPT}
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || exit 1
+[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}"
+
+exit 0

jobs/rocoto/ocnanalpost.sh

@@ -0,0 +1,15 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source GDASApp modules
+module purge
+module use "${HOMEgfs}/sorc/gdas.cd/modulefiles"
+module load GDAS/"${machine,,}"
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}"/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_POST
+status=$?
+exit "${status}"

jobs/rocoto/ocnanalprep.sh

@@ -0,0 +1,15 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source GDASApp modules
+module purge
+module use "${HOMEgfs}/sorc/gdas.cd/modulefiles"
+module load "GDAS/${machine,,}"
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}"/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP
+status=$?
+exit "${status}"

jobs/rocoto/ocnanalrun.sh

@@ -0,0 +1,15 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source GDASApp modules
+module purge
+module use "${HOMEgfs}/sorc/gdas.cd/modulefiles"
+module load GDAS/"${machine,,}"
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}"/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN
+status=$?
+exit "${status}"

parm/config/config.base.emc.dyn

@@ -327,6 +327,7 @@ export imp_physics=@IMP_PHYSICS@
 # DA engine
 export DO_JEDIVAR="NO"
 export DO_JEDIENS="NO"
+export DO_JEDIOCNVAR="NO"
 
 # Hybrid related
 export DOHYBVAR="YES"

parm/config/config.ocnanal

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+########## config.ocnanal ##########
+# configuration common to all atm analysis tasks
+
+echo "BEGIN: config.ocnanal"
+
+export OBS_YAML_DIR=${HOMEgfs}/sorc/gdas.cd/parm/soca/obs/config
+export OBS_YAML=${HOMEgfs}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml
+export OBS_LIST=${OBS_YAML_DIR}/obs_list.yaml
+export FV3JEDI_STAGE_YAML=${HOMEgfs}/sorc/gdas.cd/test/soca/testinput/dumy.yaml
+export SOCA_INPUT_FIX_DIR=/scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/72x35x25/soca # TODO: Should be moved somewhere else
+export SOCA_VARS=tocn,socn,ssh
+export SOCA_NINNER=50
+export CASE_ANL="C48"
+export DOMAIN_STACK_SIZE=2000000
+export JEDI_BIN=${HOMEgfs}/sorc/gdas.cd/build/bin
+
+# TODO: Move the R2D2 configuration to a common space
+export R2D2_OBS_DB=shared
+export R2D2_OBS_DUMP=s2s_v1
+export R2D2_OBS_SRC=gdas_marine
+export R2D2_OBS_WINDOW=24     # TODO: Check if the R2D2 sampling DB window is still needed
+export COMIN_OBS=/scratch2/NCEPDEV/marineda/r2d2
+echo "END: config.ocnanal"

parm/config/config.ocnanalpost

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+########## config.ocnanalpost ##########
+# Post Ocn Analysis specific
+
+echo "BEGIN: config.ocnanalpost"
+
+# Get task specific resources
+. "${EXPDIR}/config.resources ocnanalpost"
+echo "END: config.ocnanalpost"

parm/config/config.ocnanalprep

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+########## config.ocnanalprep ##########
+# Pre Ocn Analysis specific
+
+echo "BEGIN: config.ocnanalprep"
+
+# Get task specific resources
+. "${EXPDIR}/config.resources ocnanalprep"
+echo "END: config.ocnanalprep"

parm/config/config.ocnanalrun

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+########## config.ocnanalrun ##########
+# Ocn Analysis specific
+
+echo "BEGIN: config.ocnanalrun"
+
+# Get task specific resources
+. "${EXPDIR}/config.resources ocnanalrun"
+
+echo "END: config.ocnanalrun"