Enable AO WCDA test (NOAA-EMC#1963)

This PR: - adds GSI + SOCA C48 5-deg ocean 3DVar test (courtesy @guillaumevernieres) - adds a toggle to optionally disable ocnanalvrfy job.
kayeekayee · Feb 8, 2024 · 43429e2 · 43429e2
1 parent f563528
commit 43429e2
Show file tree

Hide file tree

Showing 10 changed files with 75 additions and 33 deletions.
diff --git a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml
@@ -0,0 +1,22 @@
+experiment:
+  system: gfs
+  mode: cycled
+
+arguments:
+  pslot: {{ 'pslot' | getenv }}
+  app: S2S
+  resdetatmos: 48
+  resdetocean: 5.0
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
+  expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500
+  idate: 2021032412
+  edate: 2021032418
+  nens: 0
+  gfs_cyc: 0
+  start: warm
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml
+
+skip_ci_on_hosts:
+  - orion
+  - hercules
diff --git a/ci/cases/yamls/soca_gfs_defaults_ci.yaml b/ci/cases/yamls/soca_gfs_defaults_ci.yaml
@@ -0,0 +1,5 @@
+defaults:
+  !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
+base:
+  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
+  DO_JEDIOCNVAR: "YES"
diff --git a/env/HERA.env b/env/HERA.env
@@ -93,31 +93,19 @@ elif [[ "${step}" = "ocnanalbmat" ]]; then
 
     export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
 
-    nth_max=$((npe_node_max / npe_node_ocnanalbmat))
-
-    export NTHREADS_OCNANAL=${nth_ocnanalbmat:-${nth_max}}
-    [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
-    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalbmat} --cpus-per-task=${NTHREADS_OCNANAL}"
+    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalbmat}"
 
 elif [[ "${step}" = "ocnanalrun" ]]; then
 
     export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
 
-    nth_max=$((npe_node_max / npe_node_ocnanalrun))
-
-    export NTHREADS_OCNANAL=${nth_ocnanalrun:-${nth_max}}
-    [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
-    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalrun} --cpus-per-task=${NTHREADS_OCNANAL}"
+    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalrun}"
 
 elif [[ "${step}" = "ocnanalchkpt" ]]; then
 
     export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
 
-    nth_max=$((npe_node_max / npe_node_ocnanalchkpt))
-
-    export NTHREADS_OCNANAL=${nth_ocnanalchkpt:-${nth_max}}
-    [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
-    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalchkpt} --cpus-per-task=${NTHREADS_OCNANAL}"
+    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalchkpt}"
 
 elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
 

diff --git a/jobs/JGLOBAL_PREP_OCEAN_OBS b/jobs/JGLOBAL_PREP_OCEAN_OBS
@@ -15,7 +15,7 @@ YMD=${PDY} HH=${cyc} generate_com -rx COMOUT_OBS:COM_OBS_TMPL
 ##############################################
 
 # Add prep_marine_obs.py to PYTHONPATH
-export PYTHONPATH=${HOMEgfs}/sorc/gdas.cd/ush/soca:${PYTHONPATH}
+export PYTHONPATH=${HOMEgfs}/sorc/gdas.cd/ush:${PYTHONPATH}
 
 ###############################################################
 # Run relevant script
@@ -38,7 +38,7 @@ if [[ -e "${pgmout}" ]] ; then
 fi
 
 ##########################################
-# Handle the temporary working directory 
+# Handle the temporary working directory
 ##########################################
 cd "${DATAROOT}" || (echo "FATAL ERROR: ${DATAROOT} does not exist. ABORT!"; exit 1)
 [[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}"

diff --git a/parm/config/gfs/config.base.emc.dyn b/parm/config/gfs/config.base.emc.dyn
@@ -378,6 +378,7 @@ export binary_diag=".false."
 # Verification options
 export DO_METP="NO"          # Run METPLUS jobs - set METPLUS settings in config.metp; not supported with spack-stack
 export DO_FIT2OBS="YES"      # Run fit to observations package
+export DO_VRFY_OCEANDA="NO"  # Run SOCA Ocean DA verification tasks
 
 # Archiving options
 export HPSSARCH="@HPSSARCH@"        # save data to HPSS archive

diff --git a/parm/config/gfs/config.ocn b/parm/config/gfs/config.ocn
@@ -16,6 +16,14 @@ if [[ "${DO_JEDIOCNVAR}" == "YES" ]]; then
 else
   export ODA_INCUPD="False"
 fi
-export ODA_INCUPD_NHOURS="3.0"  # In MOM_input, this is time interval for applying increment
+
+# Time interval for applying the increment
+if [[ "${DOIAU}" == "YES" ]]; then
+  export ODA_INCUPD_NHOURS="6.0"
+else
+  export ODA_INCUPD_NHOURS="3.0"
+fi
+
+
 
 echo "END: config.ocn"
diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
@@ -352,7 +352,7 @@ case ${step} in
     export npe_prepoceanobs=1
     export nth_prepoceanobs=1
     export npe_node_prepoceanobs=$(( npe_node_max / nth_prepoceanobs ))
-    export memory_prepoceanobs="24GB"
+    export memory_prepoceanobs="48GB"
     ;;
 
   "ocnanalbmat")
@@ -406,13 +406,20 @@ case ${step} in
     export nth_ocnanalchkpt=1
     export npe_node_ocnanalchkpt=$(( npe_node_max / nth_ocnanalchkpt ))
     case ${CASE} in
-      "C384") memory_ocnanalchkpt="128GB";;
-      "C96")  memory_ocnanalchkpt="32GB";;
-      "C48")  memory_ocnanalchkpt="32GB";;
+      "C384")
+        memory_ocnanalchkpt="128GB"
+        npes=40;;
+      "C96")
+        memory_ocnanalchkpt="32GB"
+        npes=16;;
+      "C48")
+        memory_ocnanalchkpt="32GB"
+        npes=8;;
       *)
         echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}"
         exit 4
     esac
+    export npe_ocnanalchkpt=${npes}
     export memory_ocnanalchkpt
     ;;
 
@@ -926,7 +933,7 @@ case ${step} in
           export nth_eupd=4
         fi
         ;;
-      *)        
+      *)
         echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}"
         exit 4
         ;;

diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml
@@ -25,13 +25,14 @@ landanl:
   IO_LAYOUT_Y: 1
 
 ocnanal:
-  SOCA_INPUT_FIX_DIR: "/scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/72x35x25/soca"  # TODO: These need to go to glopara fix space. @guillaumevernieres will open an issue
-  CASE_ANL: "C48"
-  COMIN_OBS: "/scratch2/NCEPDEV/marineda/r2d2-v2-v3"  # TODO: make platform agnostic
-  SOCA_OBS_LIST: "{{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml"
+  SOCA_INPUT_FIX_DIR: "/scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/72x35x25/soca"  # TODO: These need to go to glopara fix space.
+  CASE_ANL: "C48"  # TODO: Check in gdasapp if used anywhere for SOCA
+  SOCA_OBS_LIST: "{{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml"  # TODO: This is also repeated in oceanprepobs
   SOCA_NINNER: 100
-  R2D2_OBS_SRC: "gdas_marine"
-  R2D2_OBS_DUMP: "s2s_v1"
   SABER_BLOCKS_YAML: ""
   NICAS_RESOL: 1
   NICAS_GRID_SIZE: 15000
+prepoceanobs:
+  SOCA_OBS_LIST: "{{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml"  # TODO: This is also repeated in ocnanal
+  OBSPREP_YAML: "{{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obsprep/obsprep_config.yaml"
+  DMPDIR: "/scratch1/NCEPDEV/global/glopara/data/experimental_obs"
diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py
@@ -18,6 +18,7 @@ def __init__(self, conf: Configuration):
         self.do_jediocnvar = self._base.get('DO_JEDIOCNVAR', False)
         self.do_jedilandda = self._base.get('DO_JEDILANDDA', False)
         self.do_mergensst = self._base.get('DO_MERGENSST', False)
+        self.do_vrfy_oceanda = self._base.get('DO_VRFY_OCEANDA', False)
 
         self.lobsdiag_forenkf = False
         self.eupd_cdumps = None
@@ -43,8 +44,9 @@ def _get_app_configs(self):
 
         if self.do_jediocnvar:
             configs += ['prepoceanobs', 'ocnanalprep', 'ocnanalbmat',
-                        'ocnanalrun', 'ocnanalchkpt', 'ocnanalpost',
-                        'ocnanalvrfy']
+                        'ocnanalrun', 'ocnanalchkpt', 'ocnanalpost']
+            if self.do_vrfy_oceanda:
+                configs += ['ocnanalvrfy']
 
         if self.do_ocean:
             configs += ['ocnpost']
@@ -137,8 +139,9 @@ def get_task_names(self):
         if self.do_jediocnvar:
             gdas_gfs_common_tasks_before_fcst += ['prepoceanobs', 'ocnanalprep',
                                                   'ocnanalbmat', 'ocnanalrun',
-                                                  'ocnanalchkpt', 'ocnanalpost',
-                                                  'ocnanalvrfy']
+                                                  'ocnanalchkpt', 'ocnanalpost']
+            if self.do_vrfy_oceanda:
+                gdas_gfs_common_tasks_before_fcst += ['ocnanalvrfy']
 
         gdas_gfs_common_tasks_before_fcst += ['sfcanl', 'analcalc']
 

diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py
@@ -224,6 +224,13 @@ def link_files_from_src_to_dst(src_dir, dst_dir):
         src_file = os.path.join(src_dir, fname)
         if os.path.exists(src_file):
             os.symlink(src_file, os.path.join(dst_dir, fname))
+    # First 1/2 cycle also needs a atmos increment if doing warm start
+    if inputs.start in ['warm']:
+        for ftype in ['atmi003.nc', 'atminc.nc', 'atmi009.nc']:
+            fname = f'{inputs.cdump}.t{idatestr[8:]}z.{ftype}'
+            src_file = os.path.join(src_dir, fname)
+            if os.path.exists(src_file):
+                os.symlink(src_file, os.path.join(dst_dir, fname))
 
     return