Initial blocks in place for forecast refactor work (#1466)

This PR is a first in a series of PR for transforming the forecast job.
This PR does not affect current function of the forecast job.
This PR:
- adds initial blocks to separate task specific and model configuration for the task blocks

jobs/rocoto/fcst.sh

@@ -18,10 +18,29 @@ module load prod_util
 if [[ "${MACHINE_ID}" = "wcoss2" ]]; then
   module load cray-pals
 fi
+if [[ "${MACHINE_ID}" = "hera" ]]; then
+  module use "/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/modulefiles/core"
+  module load "miniconda3/4.6.14"
+  module load "gfs_workflow/1.0.0"
+# TODO: orion and wcoss2 will be uncommented when they are ready.  This comment block will be removed in the next PR
+#elif [[ "${MACHINE_ID}" = "orion" ]]; then
+#  module use "/home/rmahajan/opt/global-workflow/modulefiles/core"
+#  module load "python/3.7.5"
+#  module load "gfs_workflow/1.0.0"
+#elif [[ "${MACHINE_ID}" = "wcoss2" ]]; then
+#  module load "python/3.7.5"
+fi
 module list
 unset MACHINE_ID
 set_trace
 
+###############################################################
+# exglobal_forecast.py requires the following in PYTHONPATH
+# This will be moved to a module load when ready
+pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src:${HOMEgfs}/ush/python/pygfs"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}"
+export PYTHONPATH
+
 export job="fcst"
 export jobid="${job}.$$"
 

parm/config/config.fcst

@@ -71,6 +71,7 @@ fi
 #######################################################################
 
 export FORECASTSH="$HOMEgfs/scripts/exglobal_forecast.sh"
+#export FORECASTSH="$HOMEgfs/scripts/exglobal_forecast.py"  # Temp. while this is worked on
 export FCSTEXECDIR="$HOMEgfs/exec"
 export FCSTEXEC="ufs_model.x"
 

scripts/exglobal_forecast.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+
+import os
+
+from pygw.logger import Logger, logit
+from pygw.yaml_file import save_as_yaml
+from pygw.configuration import cast_strdict_as_dtypedict
+from pygfs.task.gfs_forecast import GFSForecast
+
+# initialize root logger
+logger = Logger(level=os.environ.get("LOGGING_LEVEL"), colored_log=True)
+
+
+@logit(logger)
+def main():
+
+    # instantiate the forecast
+    config = cast_strdict_as_dtypedict(os.environ)
+    save_as_yaml(config, f'{config.EXPDIR}/fcst.yaml')  # Temporarily save the input to the Forecast
+
+    fcst = GFSForecast(config)
+    fcst.initialize()
+    fcst.configure()
+
+
+if __name__ == '__main__':
+    main()

ush/python/pygfs/task/gfs_forecast.py

@@ -0,0 +1,35 @@
+import os
+import logging
+from typing import Dict, Any
+
+from pygw.logger import logit
+from pygw.task import Task
+from pygfs.ufswm.gfs import GFS
+
+logger = logging.getLogger(__name__.split('.')[-1])
+
+
+class GFSForecast(Task):
+    """
+    UFS-weather-model forecast task for the GFS
+    """
+
+    @logit(logger, name="GFSForecast")
+    def __init__(self, config: Dict[str, Any], *args, **kwargs):
+        """
+        Parameters
+        ----------
+        config : Dict
+                 dictionary object containing configuration from environment
+
+        *args : tuple
+                Additional arguments to `Task`
+
+        **kwargs : dict, optional
+                   Extra keyword arguments to `Task`
+        """
+
+        super().__init__(config, *args, **kwargs)
+
+        # Create and initialize the GFS variant of the UFS
+        self.gfs = GFS(config)

ush/python/pygfs/ufswm/gfs.py

@@ -0,0 +1,20 @@
+import copy
+import logging
+
+from pygw.logger import logit
+from pygfs.ufswm.ufs import UFS
+
+logger = logging.getLogger(__name__.split('.')[-1])
+
+
+class GFS(UFS):
+
+    @logit(logger, name="GFS")
+    def __init__(self, config):
+
+        super().__init__("GFS", config)
+
+        # Start putting fixed properties of the GFS
+        self.ntiles = 6
+
+        # Determine coupled/uncoupled from config and define as appropriate

ush/python/pygfs/ufswm/ufs.py

@@ -0,0 +1,58 @@
+import re
+import copy
+import logging
+from typing import Dict, Any
+
+from pygw.template import Template, TemplateConstants
+from pygw.logger import logit
+
+logger = logging.getLogger(__name__.split('.')[-1])
+
+UFS_VARIANTS = ['GFS']
+
+
+class UFS:
+
+    @logit(logger, name="UFS")
+    def __init__(self, model_name: str, config: Dict[str, Any]):
+        """Initialize the UFS-weather-model generic class and check if the model_name is a valid variant
+
+        Parameters
+        ----------
+        model_name: str
+            UFS variant
+        config : Dict
+            Incoming configuration dictionary
+        """
+
+        # First check if this is a valid variant
+        if model_name not in UFS_VARIANTS:
+            logger.warn(f"{model_name} is not a valid UFS variant")
+            raise NotImplementedError(f"{model_name} is not yet implemented")
+
+        # Make a deep copy of incoming config for caching purposes. _config should not be updated
+        self._config = copy.deepcopy(config)
+
+    @logit(logger)
+    def parse_ufs_templates(input_template, output_file, ctx: Dict) -> None:
+        """
+        This method parses UFS-weather-model templates of the pattern @[VARIABLE]
+        drawing the value from ctx['VARIABLE']
+        """
+
+        with open(input_template, 'r') as fhi:
+            file_in = fhi.read()
+            file_out = Template.substitute_structure(
+                file_in, TemplateConstants.AT_SQUARE_BRACES, ctx.get)
+
+        # If there are unrendered bits, find out what they are
+        pattern = r"@\[.*?\]+"
+        matches = re.findall(pattern, file_out)
+        if matches:
+            logger.warn(f"{input_template} was rendered incompletely")
+            logger.warn(f"The following variables were not substituted")
+            print(matches)  # TODO: improve the formatting of this message
+        # TODO: Should we abort here? or continue to write output_file?
+
+        with open(output_file, 'w') as fho:
+            fho.write(file_out)

ush/python/pygw/src/pygw/logger.py

@@ -2,6 +2,7 @@
 Logger
 """
 
+import os
 import sys
 from functools import wraps
 from pathlib import Path
@@ -48,7 +49,7 @@ class Logger:
     DEFAULT_FORMAT = '%(asctime)s - %(levelname)-8s - %(name)-12s: %(message)s'
 
     def __init__(self, name: str = None,
-                 level: str = None,
+                 level: str = os.environ.get("LOGGING_LEVEL"),
                  _format: str = DEFAULT_FORMAT,
                  colored_log: bool = False,
                  logfile_path: Union[str, Path] = None):
@@ -74,18 +75,15 @@ def __init__(self, name: str = None,
                        default : None
         """
 
-        if level is None:
-            level = os.environ.get("LOGGING_LEVEL", Logger.DEFAULT_LEVEL)
-
         self.name = name
-        self.level = level.upper()
+        self.level = level.upper() if level else Logger.DEFAULT_LEVEL
         self.format = _format
         self.colored_log = colored_log
 
         if self.level not in Logger.LOG_LEVELS:
-            raise LookupError('{self.level} is unknown logging level\n' +
-                              'Currently supported log levels are:\n' +
-                              f'{" | ".join(Logger.LOG_LEVELS)}')
+            raise LookupError(f"{self.level} is unknown logging level\n" +
+                              f"Currently supported log levels are:\n" +
+                              f"{' | '.join(Logger.LOG_LEVELS)}")
 
         # Initialize the root logger if no name is present
         self._logger = logging.getLogger(name) if name else logging.getLogger()