per #1910, added time util function to add field info to time info di…

…ctionary and updated wrapper logic to use this function to add this info instead of passing field info (var_info) to CommandBuilder.find_data to get level info, ci-run-all-diff

internal/tests/pytests/wrappers/command_builder/test_command_builder.py

@@ -6,7 +6,7 @@
 import datetime
 
 from metplus.wrappers.command_builder import CommandBuilder
-from metplus.util import ti_calculate
+from metplus.util import ti_calculate, add_field_info_to_time_info
 
 
 def get_data_dir(config):
@@ -27,9 +27,9 @@ def test_find_data_no_dated(metplus_config, data_type):
     config = metplus_config
 
     pcw = CommandBuilder(config)
-    v = {}
-    v['fcst_level'] = "6"
-    v['obs_level'] = "6"
+    var_info = {}
+    var_info['fcst_level'] = "6"
+    var_info['obs_level'] = "6"
     task_info = {}
     task_info['valid'] = datetime.datetime.strptime("201802010000",'%Y%m%d%H%M')
     task_info['lead'] = 0
@@ -39,7 +39,8 @@ def test_find_data_no_dated(metplus_config, data_type):
     pcw.c_dict[f'{data_type}FILE_WINDOW_END'] = 3600
     pcw.c_dict[f'{data_type}INPUT_DIR'] = get_data_dir(pcw.config)
     pcw.c_dict[f'{data_type}INPUT_TEMPLATE'] = "{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}"
-    obs_file = pcw.find_data(time_info, v, data_type)
+    add_field_info_to_time_info(time_info, var_info)
+    obs_file = pcw.find_data(time_info, data_type)
     assert obs_file == pcw.c_dict[f'{data_type}INPUT_DIR']+'/20180201_0045'
 
 
@@ -67,7 +68,7 @@ def test_find_data_not_a_path(metplus_config, data_type):
     pcw.c_dict[f'{data_type}FILE_WINDOW_END'] = 0
     pcw.c_dict[f'{data_type}INPUT_DIR'] = ''
     pcw.c_dict[f'{data_type}INPUT_TEMPLATE'] = 'G003'
-    obs_file = pcw.find_data(time_info, var_info=None, data_type=data_type)
+    obs_file = pcw.find_data(time_info, data_type=data_type)
     assert obs_file == 'G003'
 
 
@@ -76,8 +77,8 @@ def test_find_obs_no_dated(metplus_config):
     config = metplus_config
 
     pcw = CommandBuilder(config)
-    v = {}
-    v['obs_level'] = "6"
+    var_info = {}
+    var_info['obs_level'] = "6"
     task_info = {}
     task_info['valid'] = datetime.datetime.strptime("201802010000", '%Y%m%d%H%M')
     task_info['lead'] = 0
@@ -87,7 +88,8 @@ def test_find_obs_no_dated(metplus_config):
     pcw.c_dict['OBS_FILE_WINDOW_END'] = 3600
     pcw.c_dict['OBS_INPUT_DIR'] = get_data_dir(pcw.config)
     pcw.c_dict['OBS_INPUT_TEMPLATE'] = "{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}"
-    obs_file = pcw.find_obs(time_info, v)
+    add_field_info_to_time_info(time_info, var_info)
+    obs_file = pcw.find_obs(time_info)
     assert obs_file == pcw.c_dict['OBS_INPUT_DIR'] + '/20180201_0045'
 
 
@@ -96,8 +98,8 @@ def test_find_obs_dated(metplus_config):
     config = metplus_config
 
     pcw = CommandBuilder(config)
-    v = {}
-    v['obs_level'] = "6"
+    var_info = {}
+    var_info['obs_level'] = "6"
     task_info = {}
     task_info['valid'] = datetime.datetime.strptime("201802010000", '%Y%m%d%H%M')
     task_info['lead'] = 0
@@ -107,7 +109,8 @@ def test_find_obs_dated(metplus_config):
     pcw.c_dict['OBS_FILE_WINDOW_END'] = 3600
     pcw.c_dict['OBS_INPUT_DIR'] = get_data_dir(pcw.config)
     pcw.c_dict['OBS_INPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d}/{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}'
-    obs_file = pcw.find_obs(time_info, v)
+    add_field_info_to_time_info(time_info, var_info)
+    obs_file = pcw.find_obs(time_info)
     assert obs_file == pcw.c_dict['OBS_INPUT_DIR']+'/20180201/20180201_0013'
 
 
@@ -126,8 +129,8 @@ def test_find_obs_offset(metplus_config, offsets, expected_file, offset_seconds)
     config = metplus_config
 
     pcw = CommandBuilder(config)
-    v = {}
-    v['obs_level'] = "6"
+    var_info = {}
+    var_info['obs_level'] = "6"
     task_info = {}
     task_info['valid'] = datetime.datetime.strptime("2020020112", '%Y%m%d%H')
     task_info['lead'] = 0
@@ -136,7 +139,8 @@ def test_find_obs_offset(metplus_config, offsets, expected_file, offset_seconds)
     pcw.c_dict['OFFSETS'] = offsets
     pcw.c_dict['OBS_INPUT_DIR'] = get_data_dir(pcw.config)
     pcw.c_dict['OBS_INPUT_TEMPLATE'] = "{da_init?fmt=%2H}z.prepbufr.tm{offset?fmt=%2H}.{da_init?fmt=%Y%m%d}"
-    obs_file, time_info = pcw.find_obs_offset(time_info, v)
+    add_field_info_to_time_info(time_info, var_info)
+    obs_file, time_info = pcw.find_obs_offset(time_info)
 
     print(f"OBSFILE: {obs_file}")
     print(f"EXPECTED FILE: {expected_file}")
@@ -153,8 +157,8 @@ def test_find_obs_dated_previous_day(metplus_config):
     config = metplus_config
 
     pcw = CommandBuilder(config)
-    v = {}
-    v['obs_level'] = "6"
+    var_info = {}
+    var_info['obs_level'] = "6"
     task_info = {}
     task_info['valid'] = datetime.datetime.strptime("201802010000", '%Y%m%d%H%M')
     task_info['lead'] = 0
@@ -164,7 +168,8 @@ def test_find_obs_dated_previous_day(metplus_config):
     pcw.c_dict['OBS_INPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d}/{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}'
     pcw.c_dict['OBS_FILE_WINDOW_BEGIN'] = -3600
     pcw.c_dict['OBS_FILE_WINDOW_END'] = 0
-    obs_file = pcw.find_obs(time_info, v)
+    add_field_info_to_time_info(time_info, var_info)
+    obs_file = pcw.find_obs(time_info)
     assert obs_file == pcw.c_dict['OBS_INPUT_DIR']+'/20180131/20180131_2345'
 
 
@@ -173,8 +178,9 @@ def test_find_obs_dated_next_day(metplus_config):
     config = metplus_config
 
     pcw = CommandBuilder(config)
-    v = {}
-    v['obs_level'] = "6"
+    var_info = {
+        'obs_level': "6"
+    }
     task_info = {}
     task_info['valid'] = datetime.datetime.strptime("201802012345", '%Y%m%d%H%M')
     task_info['lead'] = 0
@@ -184,7 +190,8 @@ def test_find_obs_dated_next_day(metplus_config):
     pcw.c_dict['OBS_INPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d}/{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}'
     pcw.c_dict['OBS_FILE_WINDOW_BEGIN'] = 0
     pcw.c_dict['OBS_FILE_WINDOW_END'] = 3600
-    obs_file = pcw.find_obs(time_info, v)
+    add_field_info_to_time_info(time_info, var_info)
+    obs_file = pcw.find_obs(time_info)
     assert obs_file == pcw.c_dict['OBS_INPUT_DIR']+'/20180202/20180202_0013'
 
 

metplus/util/time_util.py

@@ -13,6 +13,8 @@
 from dateutil.relativedelta import relativedelta
 import re
 
+from .string_manip import split_level, format_thresh
+
 '''!@namespace TimeInfo
 @brief Utility to handle timing in METplus wrappers
 @code{.sh}
@@ -333,6 +335,8 @@ def _format_time_list(string_value, get_met_format, sort_list=True):
 
 def ti_calculate(input_dict_preserve):
     out_dict = {}
+    # copy input dictionary so valid or init can be removed to recalculate it
+    # without modifying the input to the function
     input_dict = input_dict_preserve.copy()
 
     KEYS_TO_COPY = ['custom', 'instance']
@@ -381,7 +385,6 @@ def ti_calculate(input_dict_preserve):
     else:
         out_dict['lead'] = relativedelta(seconds=0)
 
-
     # set offset to 0 if not specified
     if 'offset_hours' in input_dict.keys():
         out_dict['offset'] = datetime.timedelta(hours=input_dict['offset_hours'])
@@ -390,7 +393,6 @@ def ti_calculate(input_dict_preserve):
     else:
         out_dict['offset'] = datetime.timedelta(seconds=0)
 
-
     # if init and valid are set, check which was set first via loop_by
     # remove the other to recalculate
     if 'init' in input_dict.keys() and 'valid' in input_dict.keys():
@@ -509,3 +511,32 @@ def add_to_time_input(time_input, clock_time=None, instance=None, custom=None):
     # otherwise leave it unset so it can be set within the wrapper
     if custom:
         time_input['custom'] = custom
+
+
+def add_field_info_to_time_info(time_info, var_info):
+    """!Add field information from var_info to the time_info dictionary to use
+    in string template substitution. Sets new items in time_info.
+
+    @param time_info dictionary containing time information to substitute
+    filename template tags
+    @param var_info dictionary containing information for the fields to process
+    """
+    if var_info is None:
+        return
+
+    for key, value in var_info.items():
+        # skip index and extra field info
+        if key == 'index' or key.endswith('extra'):
+            continue
+        # format level and thresh
+        if key.endswith('level'):
+            # strip off prefix letter if it exists
+            value = split_level(value)[1]
+
+            # set level to 0 character if it is not a number, e.g. NetCDF level
+            if not value.isdigit():
+                value = '0'
+        elif key.endswith('thresh'):
+            value = format_thresh(value)
+
+        time_info[key] = value

metplus/wrappers/ascii2nc_wrapper.py

@@ -297,7 +297,7 @@ def find_input_files(self, time_info):
             return self.infiles
 
         # get list of files even if only one is found (return_list=True)
-        obs_path = self.find_obs(time_info, var_info=None, return_list=True)
+        obs_path = self.find_obs(time_info, return_list=True)
         if obs_path is None:
             return None
 

metplus/wrappers/command_builder.py

@@ -425,47 +425,41 @@ def print_env_item(self, item):
         """
         return f"{item}={self.env[item]}"
 
-    def find_model(self, time_info, var_info=None, mandatory=True,
+    def find_model(self, time_info, mandatory=True,
                    return_list=False):
         """! Finds the model file to compare
-              Args:
+
                 @param time_info dictionary containing timing information
-                @param var_info object containing variable information
                 @param mandatory if True, report error if not found, warning
                  if not, default is True
                 @rtype string
                 @return Returns the path to an model file
         """
         return self.find_data(time_info,
-                              var_info=var_info,
                               data_type="FCST",
                               mandatory=mandatory,
                               return_list=return_list)
 
-    def find_obs(self, time_info, var_info=None, mandatory=True,
+    def find_obs(self, time_info, mandatory=True,
                  return_list=False):
         """! Finds the observation file to compare
-              Args:
+
                 @param time_info dictionary containing timing information
-                @param var_info object containing variable information
                 @param mandatory if True, report error if not found, warning
                  if not, default is True
                 @rtype string
                 @return Returns the path to an observation file
         """
         return self.find_data(time_info,
-                              var_info=var_info,
                               data_type="OBS",
                               mandatory=mandatory,
                               return_list=return_list)
 
-    def find_obs_offset(self, time_info, var_info=None, mandatory=True,
-                        return_list=False):
+    def find_obs_offset(self, time_info, mandatory=True, return_list=False):
         """! Finds the observation file to compare, looping through offset
             list until a file is found
 
              @param time_info dictionary containing timing information
-             @param var_info object containing variable information
              @param mandatory if True, report error if not found, warning
               if not, default is True
              @rtype string
@@ -483,7 +477,6 @@ def find_obs_offset(self, time_info, var_info=None, mandatory=True,
             time_info['offset_hours'] = offset
             time_info = ti_calculate(time_info)
             obs_path = self.find_obs(time_info,
-                                     var_info=var_info,
                                      mandatory=is_mandatory,
                                      return_list=return_list)
 
@@ -505,12 +498,10 @@ def find_obs_offset(self, time_info, var_info=None, mandatory=True,
 
         return None, time_info
 
-    def find_data(self, time_info, var_info=None, data_type='', mandatory=True,
+    def find_data(self, time_info, data_type='', mandatory=True,
                   return_list=False, allow_dir=False):
         """! Finds the data file to compare
-              Args:
                 @param time_info dictionary containing timing information
-                @param var_info object containing variable information
                 @param data_type type of data to find (i.e. FCST_ or OBS_)
                 @param mandatory if True, report error if not found, warning
                  if not. default is True
@@ -523,22 +514,8 @@ def find_data(self, time_info, var_info=None, data_type='', mandatory=True,
         if data_type and not data_type.endswith('_'):
             data_type_fmt += '_'
 
-        if var_info is not None:
-            # set level based on input data type
-            if data_type_fmt.startswith("OBS"):
-                v_level = var_info['obs_level']
-            else:
-                v_level = var_info['fcst_level']
-
-            # separate character from beginning of numeric
-            # level value if applicable
-            level = split_level(v_level)[1]
-
-            # set level to 0 character if it is not a number
-            if not level.isdigit():
-                level = '0'
-        else:
-            level = '0'
+        # set generic 'level' to level that corresponds to data_type if set
+        level = time_info.get(f'{data_type_fmt.lower()}level', '0')
 
         # if level is a range, use the first value, i.e. if 250-500 use 250
         level = level.split('-')[0]
@@ -627,7 +604,7 @@ def find_exact_file(self, level, data_type, time_info, mandatory=True,
                 check_file_list.append(full_path)
 
         # if it was set, add level back to time_info
-        if saved_level:
+        if saved_level is not None:
             time_info['level'] = saved_level
 
         # if multiple files are not supported by the wrapper and multiple
@@ -1061,15 +1038,15 @@ def check_gempaktocf(self, gempaktocf_jar):
                              "on how to obtain the tool: parm/use_cases/met_tool_wrapper/GempakToCF/GempakToCF.py")
             self.isOK = False
 
-    def add_field_info_to_time_info(self, time_info, field_info):
-        """!Add name and level values from field info to time info dict to be used in string substitution
-            Args:
-                @param time_info time dictionary to add items to
-                @param field_info field dictionary to get values from
-        """
-        field_items = ['fcst_name', 'fcst_level', 'obs_name', 'obs_level']
-        for field_item in field_items:
-            time_info[field_item] = field_info[field_item] if field_item in field_info else ''
+    # def add_field_info_to_time_info(self, time_info, field_info):
+    #     """!Add name and level values from field info to time info dict to be used in string substitution
+    #         Args:
+    #             @param time_info time dictionary to add items to
+    #             @param field_info field dictionary to get values from
+    #     """
+    #     field_items = ['fcst_name', 'fcst_level', 'obs_name', 'obs_level']
+    #     for field_item in field_items:
+    #         time_info[field_item] = field_info[field_item] if field_item in field_info else ''
 
     def set_current_field_config(self, field_info=None):
         """! Sets config variables for current fcst/obs name/level that can be