Merge remote-tracking branch 'origin/develop' into feature/issue2332

Refs NOAA-EMC#2332
* origin/develop:
  Add Hercules support for the GSI monitor (NOAA-EMC#2373)
  Reformat snowDA templates to jinja2 (NOAA-EMC#2371)

.gitignore

@@ -42,8 +42,12 @@ fix/wave
 
 # Ignore parm file symlinks
 #--------------------------
-parm/config/config.base
-parm/gdas/bufr2ioda
+parm/gdas/aero
+parm/gdas/atm
+parm/gdas/io
+parm/gdas/ioda
+parm/gdas/snow
+parm/gdas/soca
 parm/monitor
 parm/post/AEROSOL_LUTS.dat
 parm/post/nam_micro_lookup.dat

.gitmodules

@@ -1,28 +1,28 @@
 [submodule "sorc/ufs_model.fd"]
-	path = sorc/ufs_model.fd
-	url = https://github.com/ufs-community/ufs-weather-model
-	ignore = dirty
+  path = sorc/ufs_model.fd
+  url = https://github.com/ufs-community/ufs-weather-model
+  ignore = dirty
 [submodule "sorc/wxflow"]
-	path = sorc/wxflow
-	url = https://github.com/NOAA-EMC/wxflow
+  path = sorc/wxflow
+  url = https://github.com/NOAA-EMC/wxflow
 [submodule "sorc/gfs_utils.fd"]
-	path = sorc/gfs_utils.fd
-	url = https://github.com/NOAA-EMC/gfs-utils
+  path = sorc/gfs_utils.fd
+  url = https://github.com/NOAA-EMC/gfs-utils
 [submodule "sorc/ufs_utils.fd"]
-	path = sorc/ufs_utils.fd
-	url = https://github.com/ufs-community/UFS_UTILS.git
+  path = sorc/ufs_utils.fd
+  url = https://github.com/ufs-community/UFS_UTILS.git
 [submodule "sorc/verif-global.fd"]
-	path = sorc/verif-global.fd
-	url = https://github.com/NOAA-EMC/EMC_verif-global.git
+  path = sorc/verif-global.fd
+  url = https://github.com/NOAA-EMC/EMC_verif-global.git
 [submodule "sorc/gsi_enkf.fd"]
-	path = sorc/gsi_enkf.fd
-	url = https://github.com/NOAA-EMC/GSI.git
+  path = sorc/gsi_enkf.fd
+  url = https://github.com/NOAA-EMC/GSI.git
 [submodule "sorc/gdas.cd"]
-	path = sorc/gdas.cd
-	url = https://github.com/NOAA-EMC/GDASApp.git
+  path = sorc/gdas.cd
+  url = https://github.com/NOAA-EMC/GDASApp.git
 [submodule "sorc/gsi_utils.fd"]
-	path = sorc/gsi_utils.fd
-	url = https://github.com/NOAA-EMC/GSI-Utils.git
+  path = sorc/gsi_utils.fd
+  url = https://github.com/NOAA-EMC/GSI-Utils.git
 [submodule "sorc/gsi_monitor.fd"]
-	path = sorc/gsi_monitor.fd
-	url = https://github.com/NOAA-EMC/GSI-Monitor.git
+  path = sorc/gsi_monitor.fd
+  url = https://github.com/NOAA-EMC/GSI-Monitor.git

ci/cases/pr/C48mx500_3DVarAOWCDA.yaml

@@ -19,4 +19,5 @@ arguments:
 
 skip_ci_on_hosts:
   - orion
+  - hera
   - hercules

jobs/JGLOBAL_ATM_PREP_IODA_OBS

@@ -18,7 +18,7 @@ YMD=${PDY} HH=${cyc} generate_com -rx COM_OBS
 ###############################################################
 # Run relevant script
 EXSCRIPT=${BUFR2IODASH:-${USHgfs}/run_bufr2ioda.py}
-${EXSCRIPT} "${PDY}${cyc}" "${RUN}" "${DMPDIR}" "${PARMgfs}/gdas/bufr2ioda" "${COM_OBS}/"
+${EXSCRIPT} "${PDY}${cyc}" "${RUN}" "${DMPDIR}" "${PARMgfs}/gdas/ioda/bufr2ioda" "${COM_OBS}/"
 status=$?
 [[ ${status} -ne 0 ]] && (echo "FATAL ERROR: Error executing ${EXSCRIPT}, ABORT!"; exit "${status}")
 

modulefiles/module_base.hercules.lua

@@ -9,7 +9,6 @@ load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None")))
 load(pathJoin("stack-intel-oneapi-mpi", (os.getenv("stack_impi_ver") or "None")))
 load(pathJoin("intel-oneapi-mkl", (os.getenv("intel_mkl_ver") or "None")))
 load(pathJoin("python", (os.getenv("python_ver") or "None")))
-load(pathJoin("perl", (os.getenv("perl_ver") or "None")))
 
 load(pathJoin("jasper", (os.getenv("jasper_ver") or "None")))
 load(pathJoin("libpng", (os.getenv("libpng_ver") or "None")))

parm/config/gefs/config.base.emc.dyn → parm/config/gefs/config.base

@@ -109,6 +109,7 @@ export RUN="gefs"  # RUN is defined in the job-card (ecf); CDUMP is used at EMC
 # Get all the COM path templates
 source "${EXPDIR}/config.com"
 
+# shellcheck disable=SC2016
 export ERRSCRIPT=${ERRSCRIPT:-'eval [[ $err = 0 ]]'}
 export LOGSCRIPT=${LOGSCRIPT:-""}
 #export ERRSCRIPT=${ERRSCRIPT:-"err_chk"}
@@ -209,7 +210,7 @@ export gfs_cyc=@gfs_cyc@ # 0: no GFS cycle, 1: 00Z only, 2: 00Z and 12Z only, 4:
 export FHMIN_GFS=0
 export FHMIN=${FHMIN_GFS}
 export FHMAX_GFS=@FHMAX_GFS@
-export FHOUT_GFS=6 
+export FHOUT_GFS=6
 export FHMAX_HF_GFS=0
 export FHOUT_HF_GFS=1
 export FHOUT_OCNICE_GFS=6

parm/config/gfs/config.base.emc.dyn → parm/config/gfs/config.base

@@ -56,16 +56,9 @@ export DO_NPOESS="NO"      # NPOESS products
 export DO_TRACKER="YES"    # Hurricane track verification
 export DO_GENESIS="YES"    # Cyclone genesis verification
 export DO_GENESIS_FSU="NO" # Cyclone genesis verification (FSU)
-# The monitor is not yet supported on Hercules
-if [[ "${machine}" == "HERCULES" ]]; then
-   export DO_VERFOZN="NO"     # Ozone data assimilation monitoring
-   export DO_VERFRAD="NO"     # Radiance data assimilation monitoring
-   export DO_VMINMON="NO"     # GSI minimization monitoring
-else
-   export DO_VERFOZN="YES"    # Ozone data assimilation monitoring
-   export DO_VERFRAD="YES"    # Radiance data assimilation monitoring
-   export DO_VMINMON="YES"    # GSI minimization monitoring
-fi
+export DO_VERFOZN="YES"    # Ozone data assimilation monitoring
+export DO_VERFRAD="YES"    # Radiance data assimilation monitoring
+export DO_VMINMON="YES"    # GSI minimization monitoring
 export DO_MOS="NO"         # GFS Model Output Statistics - Only supported on WCOSS2
 
 # NO for retrospective parallel; YES for real-time parallel
@@ -131,6 +124,7 @@ export RUN=${RUN:-${CDUMP:-"gfs"}}  # RUN is defined in the job-card (ecf); CDUM
 # Get all the COM path templates
 source "${EXPDIR}/config.com"
 
+# shellcheck disable=SC2016
 export ERRSCRIPT=${ERRSCRIPT:-'eval [[ $err = 0 ]]'}
 export LOGSCRIPT=${LOGSCRIPT:-""}
 #export ERRSCRIPT=${ERRSCRIPT:-"err_chk"}
@@ -246,7 +240,7 @@ export gfs_cyc=@gfs_cyc@ # 0: no GFS cycle, 1: 00Z only, 2: 00Z and 12Z only, 4:
 # GFS output and frequency
 export FHMIN_GFS=0
 export FHMAX_GFS=@FHMAX_GFS@
-export FHOUT_GFS=3 
+export FHOUT_GFS=3
 export FHMAX_HF_GFS=0
 export FHOUT_HF_GFS=1
 export FHOUT_OCNICE_GFS=6

parm/config/gfs/config.prepsnowobs

@@ -8,11 +8,14 @@ echo "BEGIN: config.prepsnowobs"
 # Get task specific resources
 . "${EXPDIR}/config.resources" prepsnowobs
 
-export GTS_OBS_LIST="${HOMEgfs}/sorc/gdas.cd/parm/snow/prep/prep_gts.yaml"
+export GTS_OBS_LIST="${PARMgfs}/gdas/snow/prep/prep_gts.yaml.j2"
+export IMS_OBS_LIST="${PARMgfs}/gdas/snow/prep/prep_ims.yaml.j2"
+
 export BUFR2IODAX="${EXECgfs}/bufr2ioda.x"
-export FIMS_NML_TMPL="${HOMEgfs}/sorc/gdas.cd/parm/snow/prep/fims.nml.j2"
-export IMS_OBS_LIST="${HOMEgfs}/sorc/gdas.cd/parm/snow/prep/prep_ims.yaml"
+
 export CALCFIMSEXE="${EXECgfs}/calcfIMS.exe"
+export FIMS_NML_TMPL="${PARMgfs}/gdas/snow/prep/fims.nml.j2"
+
 export IMS2IODACONV="${USHgfs}/imsfv3_scf2ioda.py"
 
 echo "END: config.prepsnowobs"

parm/config/gfs/config.snowanl

@@ -6,27 +6,21 @@
 echo "BEGIN: config.snowanl"
 
 # Get task specific resources
-. "${EXPDIR}/config.resources" snowanl
+source "${EXPDIR}/config.resources" snowanl
 
-obs_list_name=gdas_snow_gts_only.yaml
-if [[ "${cyc}" = "18" ]]; then
-    obs_list_name=gdas_snow_prototype.yaml
-fi
-
-export OBS_YAML_DIR=${HOMEgfs}/sorc/gdas.cd/parm/snow/obs/config/
-export OBS_LIST=${HOMEgfs}/sorc/gdas.cd/parm/snow/obs/lists/${obs_list_name}
+export OBS_LIST="${PARMgfs}/gdas/snow/obs/lists/gdas_snow.yaml.j2"
 
 # Name of the JEDI executable and its yaml template
 export JEDIEXE="${EXECgfs}/fv3jedi_letkf.x"
-export JEDIYAML="${HOMEgfs}/sorc/gdas.cd/parm/snow/letkfoi/letkfoi.yaml"
+export JEDIYAML="${PARMgfs}/gdas/snow/letkfoi/letkfoi.yaml.j2"
 
 # Ensemble member properties
 export SNOWDEPTHVAR="snodl"
 export BESTDDEV="30."  # Background Error Std. Dev. for LETKFOI
 
 # Name of the executable that applies increment to bkg and its namelist template
 export APPLY_INCR_EXE="${EXECgfs}/apply_incr.exe"
-export APPLY_INCR_NML_TMPL="${HOMEgfs}/sorc/gdas.cd/parm/snow/letkfoi/apply_incr_nml.j2"
+export APPLY_INCR_NML_TMPL="${PARMgfs}/gdas/snow/letkfoi/apply_incr_nml.j2"
 
 export io_layout_x=@IO_LAYOUT_X@
 export io_layout_y=@IO_LAYOUT_Y@

parm/gdas/snow_jedi_fix.yaml.j2

@@ -0,0 +1,7 @@
+mkdir:
+- '{{ DATA }}/fv3jedi'
+copy:
+- ['{{ HOMEgfs }}/fix/gdas/fv3jedi/fv3files/akbk{{ npz }}.nc4', '{{ DATA }}/fv3jedi/akbk.nc4']
+- ['{{ HOMEgfs }}/fix/gdas/fv3jedi/fv3files/fmsmpp.nml', '{{ DATA }}/fv3jedi/fmsmpp.nml']
+- ['{{ HOMEgfs }}/fix/gdas/fv3jedi/fv3files/field_table_gfdl', '{{ DATA }}/fv3jedi/field_table']
+- ['{{ HOMEgfs }}/sorc/gdas.cd/parm/io/fv3jedi_fieldmetadata_restart.yaml', '{{ DATA }}/fv3jedi/fv3jedi_fieldmetadata_restart.yaml']

sorc/link_workflow.sh

@@ -204,7 +204,11 @@ fi
 #------------------------------
 if [[ -d "${HOMEgfs}/sorc/gdas.cd" ]]; then
   cd "${HOMEgfs}/parm/gdas" || exit 1
-  ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/parm/ioda/bufr2ioda" .
+  declare -a gdasapp_comps=("aero" "atm" "io" "ioda" "snow" "soca")
+  for comp in "${gdasapp_comps[@]}"; do
+    [[ -d "${comp}" ]] && rm -rf "${comp}"
+    ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/parm/${comp}" .
+  done
 fi
 
 #------------------------------

ush/python/pygfs/task/analysis.py

@@ -25,6 +25,8 @@ class Analysis(Task):
     def __init__(self, config: Dict[str, Any]) -> None:
         super().__init__(config)
         self.config.ntiles = 6
+        # Store location of GDASApp jinja2 templates
+        self.gdasapp_j2tmpl_dir = os.path.join(self.config.HOMEgfs, 'parm/gdas')
 
     def initialize(self) -> None:
         super().initialize()
@@ -56,7 +58,7 @@ def get_obs_dict(self) -> Dict[str, Any]:
             a dictionary containing the list of observation files to copy for FileHandler
         """
         logger.debug(f"OBS_LIST: {self.task_config['OBS_LIST']}")
-        obs_list_config = parse_j2yaml(self.task_config["OBS_LIST"], self.task_config)
+        obs_list_config = parse_j2yaml(self.task_config["OBS_LIST"], self.task_config, searchpath=self.gdasapp_j2tmpl_dir)
         logger.debug(f"obs_list_config: {obs_list_config}")
         # get observers from master dictionary
         observers = obs_list_config['observers']
@@ -88,7 +90,7 @@ def get_bias_dict(self) -> Dict[str, Any]:
             a dictionary containing the list of observation bias files to copy for FileHandler
         """
         logger.debug(f"OBS_LIST: {self.task_config['OBS_LIST']}")
-        obs_list_config = parse_j2yaml(self.task_config["OBS_LIST"], self.task_config)
+        obs_list_config = parse_j2yaml(self.task_config["OBS_LIST"], self.task_config, searchpath=self.gdasapp_j2tmpl_dir)
         logger.debug(f"obs_list_config: {obs_list_config}")
         # get observers from master dictionary
         observers = obs_list_config['observers']

ush/python/pygfs/task/snow_analysis.py

@@ -11,7 +11,7 @@
                     FileHandler,
                     to_fv3time, to_YMD, to_YMDH, to_timedelta, add_to_datetime,
                     rm_p,
-                    parse_j2yaml, parse_yamltmpl, save_as_yaml,
+                    parse_j2yaml, save_as_yaml,
                     Jinja,
                     logit,
                     Executable,
@@ -99,7 +99,7 @@ def prepare_GTS(self) -> None:
 
         def _gtsbufr2iodax(exe, yaml_file):
             if not os.path.isfile(yaml_file):
-                logger.exception(f"{yaml_file} not found")
+                logger.exception(f"FATAL ERROR: {yaml_file} not found")
                 raise FileNotFoundError(yaml_file)
 
             logger.info(f"Executing {exe}")
@@ -260,9 +260,9 @@ def initialize(self) -> None:
         FileHandler({'mkdir': dirlist}).sync()
 
         # stage fix files
-        jedi_fix_list_path = os.path.join(self.task_config.HOMEgfs, 'parm', 'gdas', 'snow_jedi_fix.yaml')
+        jedi_fix_list_path = os.path.join(self.task_config.HOMEgfs, 'parm', 'gdas', 'snow_jedi_fix.yaml.j2')
         logger.info(f"Staging JEDI fix files from {jedi_fix_list_path}")
-        jedi_fix_list = parse_yamltmpl(jedi_fix_list_path, self.task_config)
+        jedi_fix_list = parse_j2yaml(jedi_fix_list_path, self.task_config)
         FileHandler(jedi_fix_list).sync()
 
         # stage backgrounds
@@ -271,10 +271,9 @@ def initialize(self) -> None:
 
         # generate letkfoi YAML file
         logger.info(f"Generate JEDI LETKF YAML file: {self.task_config.jedi_yaml}")
-        letkfoi_yaml = parse_j2yaml(self.task_config.JEDIYAML, self.task_config)
+        letkfoi_yaml = parse_j2yaml(self.task_config.JEDIYAML, self.task_config, searchpath=self.gdasapp_j2tmpl_dir)
         save_as_yaml(letkfoi_yaml, self.task_config.jedi_yaml)
         logger.info(f"Wrote letkfoi YAML to: {self.task_config.jedi_yaml}")
-
         # need output dir for diags and anl
         logger.info("Create empty output [anl, diags] directories to receive output from executable")
         newdirs = [

versions/run.hercules.ver

@@ -3,7 +3,5 @@ export stack_impi_ver=2021.9.0
 export intel_mkl_ver=2023.1.0
 export spack_env=gsi-addon-env
 
-export perl_ver=5.36.0
-
 source "${HOMEgfs:-}/versions/run.spack.ver"
 export spack_mod_path="/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core"

workflow/setup_expt.py

@@ -252,12 +252,6 @@ def fill_EXPDIR(inputs):
     expdir = os.path.join(inputs.expdir, inputs.pslot)
 
     configs = glob.glob(f'{configdir}/config.*')
-    exclude_configs = ['base', 'base.emc.dyn', 'base.nco.static', 'fv3.nco.static']
-    for exclude in exclude_configs:
-        try:
-            configs.remove(f'{configdir}/config.{exclude}')
-        except ValueError:
-            pass
     if len(configs) == 0:
         raise IOError(f'no config files found in {configdir}')
     for config in configs:
@@ -295,7 +289,8 @@ def _update_defaults(dict_in: dict) -> dict:
 
 def edit_baseconfig(host, inputs, yaml_dict):
     """
-    Parses and populates the templated `config.base.emc.dyn` to `config.base`
+    Parses and populates the templated `HOMEgfs/parm/config/<gfs|gefs>/config.base`
+    to `EXPDIR/pslot/config.base`
     """
 
     tmpl_dict = {
@@ -347,7 +342,7 @@ def edit_baseconfig(host, inputs, yaml_dict):
     except KeyError:
         pass
 
-    base_input = f'{inputs.configdir}/config.base.emc.dyn'
+    base_input = f'{inputs.configdir}/config.base'
     base_output = f'{inputs.expdir}/{inputs.pslot}/config.base'
     edit_config(base_input, base_output, tmpl_dict)