Feature 718 pcp combine output names (#843)

docs/Users_Guide/glossary.rst

@@ -5669,7 +5669,7 @@ METplus Configuration Glossary
      | *Used by:* TCStat
 
    FCST_PCP_COMBINE_EXTRA_NAMES
-     Specify a list of any additional fields to add to the command. The number of items in this list should match :term:`FCST_PCP_COMBINE_EXTRA_LEVELS`. A corresponding variable exists for observation data called :term:`OBS_PCP_COMBINE_EXTRA_NAMES`. Example:
+     Specify a list of any additional fields to add to the command. The items in this list correspond to the list set by :term:`FCST_PCP_COMBINE_EXTRA_LEVELS`. A corresponding variable exists for observation data called :term:`OBS_PCP_COMBINE_EXTRA_NAMES`. Example:
 
      | FCST_PCP_COMBINE_EXTRA_NAMES = TMP, HGT
      | FCST_PCP_COMBINE_EXTRA_LEVELS = "(*,*)", "(*,*)"
@@ -5686,7 +5686,7 @@ METplus Configuration Glossary
      | *Used by:*  PCPCombine
 
    FCST_PCP_COMBINE_EXTRA_LEVELS
-     Specify a list of any additional fields to add to the command. The number of items in this list should match :term:`FCST_PCP_COMBINE_EXTRA_NAMES`. A corresponding variable exists for observation data called :term:`OBS_PCP_COMBINE_EXTRA_LEVELS`. See :term:`FCST_PCP_COMBINE_EXTRA_NAMES` for an example.
+     Specify a list of any additional fields to add to the command. The items in this list correspond to the list set by :term:`FCST_PCP_COMBINE_EXTRA_NAMES`. If this list has fewer items than the names list, then no level value will be specified for those names (i.e. if using Python Embedding). A corresponding variable exists for observation data called :term:`OBS_PCP_COMBINE_EXTRA_LEVELS`. See :term:`FCST_PCP_COMBINE_EXTRA_NAMES` for an example.
 
      | *Used by:*  PCPCombine
 
@@ -5695,6 +5695,16 @@ METplus Configuration Glossary
 
      | *Used by:*  PCPCombine
 
+   FCST_PCP_COMBINE_EXTRA_OUTPUT_NAMES
+     Specify a list of output names for any additional fields to add to the command. The items in this list correspond to the list set by :term:`FCST_PCP_COMBINE_EXTRA_NAMES`. A corresponding variable exists for observation data called :term:`OBS_PCP_COMBINE_EXTRA_OUTPUT_NAMES`. Example:
+
+     | *Used by:*  PCPCombine
+
+   OBS_PCP_COMBINE_EXTRA_OUTPUT_NAMES
+     See :term:`FCST_PCP_COMBINE_EXTRA_OUTPUT_NAMES`
+
+     | *Used by:*  PCPCombine
+
    ENSEMBLE_STAT_MESSAGE_TYPE
      Set the message_type option in the EnsembleStat MET config file.
 

docs/Users_Guide/wrappers.rst

@@ -2225,8 +2225,10 @@ Configuration
 | :term:`PCP_COMBINE_CUSTOM_LOOP_LIST`
 | :term:`FCST_PCP_COMBINE_EXTRA_NAMES` (optional)
 | :term:`FCST_PCP_COMBINE_EXTRA_LEVELS` (optional)
+| :term:`FCST_PCP_COMBINE_EXTRA_OUTPUT_NAMES` (optional)
 | :term:`OBS_PCP_COMBINE_EXTRA_NAMES` (optional)
 | :term:`OBS_PCP_COMBINE_EXTRA_LEVELS` (optional)
+| :term:`OBS_PCP_COMBINE_EXTRA_OUTPUT_NAMES` (optional)
 | :term:`FCST_PCP_COMBINE_OUTPUT_ACCUM` (optional)
 | :term:`FCST_PCP_COMBINE_OUTPUT_NAME` (optional)
 | :term:`OBS_PCP_COMBINE_OUTPUT_ACCUM` (optional)

internal_tests/pytests/pcp_combine/test_pcp_combine_wrapper.py

@@ -658,3 +658,69 @@ def test_pcp_combine_sum_subhourly(metplus_config):
     for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds):
         # ensure commands are generated as expected
         assert(cmd == expected_cmd)
+
+@pytest.mark.parametrize(
+    'output_name,extra_output,expected_result', [
+        (None, None, None),
+        ('out_name1', None, '"out_name1"'),
+        ('out_name1', '"out_name2"', '"out_name1","out_name2"'),
+        ('out_name1', '"out_name2","out_name3"',
+         '"out_name1","out_name2","out_name3"'),
+    ]
+)
+def test_get_output_string(metplus_config, output_name, extra_output,
+                           expected_result):
+    config = metplus_config()
+    wrapper = PCPCombineWrapper(config)
+    wrapper.output_name = output_name
+    wrapper.extra_output = extra_output
+    actual_result = wrapper.get_output_string()
+    assert(actual_result == expected_result)
+
+
+@pytest.mark.parametrize(
+    'names,levels,out_names,expected_input,expected_output', [
+        # none specified
+        ('', '', '',
+         None, None),
+        # 1 input name, no level
+        ('input1', '', '',
+         "-field 'name=\"input1\";'", None),
+        # 1 input name, 1 level
+        ('input1', 'level1', '',
+         "-field 'name=\"input1\"; level=\"level1\";'", None),
+        # 2 input names, no levels
+        ('input1,input2', '', '',
+         "-field 'name=\"input1\";' -field 'name=\"input2\";'", None),
+        # 2 input names, 2 levels
+        ('input1,input2', 'level1,level2', '',
+         ("-field 'name=\"input1\"; level=\"level1\";' "
+          "-field 'name=\"input2\"; level=\"level2\";'"), None),
+        # 2 input names, 1 level
+        ('input1,input2', 'level1', '',
+         ("-field 'name=\"input1\"; level=\"level1\";' "
+          "-field 'name=\"input2\";'"),
+         None),
+        # 1 input name, 1 level, 1 output
+        ('input1', 'level1', 'output1',
+         "-field 'name=\"input1\"; level=\"level1\";'", '"output1"'),
+        # 2 input names, 2 levels, 2 outputs
+        ('input1,input2', 'level1,level2', 'output1,output2',
+         ("-field 'name=\"input1\"; level=\"level1\";' "
+          "-field 'name=\"input2\"; level=\"level2\";'"),
+         '"output1","output2"'),
+    ]
+)
+def test_get_extra_fields(metplus_config, names, levels, out_names,
+                          expected_input, expected_output):
+    config = metplus_config()
+    config.set('config', 'FCST_PCP_COMBINE_RUN', True)
+    config.set('config', 'FCST_PCP_COMBINE_EXTRA_NAMES', names)
+    config.set('config', 'FCST_PCP_COMBINE_EXTRA_LEVELS', levels)
+    config.set('config', 'FCST_PCP_COMBINE_EXTRA_OUTPUT_NAMES', out_names)
+
+    wrapper = PCPCombineWrapper(config)
+
+    actual_input, actual_output = wrapper.get_extra_fields('FCST')
+    assert(actual_input == expected_input)
+    assert (actual_output == expected_output)

metplus/wrappers/pcp_combine_wrapper.py

@@ -19,8 +19,8 @@
 from . import ReformatGriddedWrapper
 
 '''!@namespace PCPCombineWrapper
-@brief Wraps the MET tool pcp_combine to combine or divide
-precipitation accumulations
+@brief Wraps the MET tool pcp_combine to combine/divide
+precipitation accumulations or derive additional fields
 Call as follows:
 @code{.sh}
 Cannot be called directly. Must use child classes.
@@ -66,7 +66,6 @@ def create_c_dict(self):
 
         if not fcst_run and not obs_run:
             self.log_error("Must set either FCST_PCP_COMBINE_RUN or OBS_PCP_COMBINE_RUN")
-            self.isOK = False
         else:
             if fcst_run:
                 c_dict = self.set_fcst_or_obs_dict_items('FCST', c_dict)
@@ -95,6 +94,7 @@ def set_fcst_or_obs_dict_items(self, d_type, c_dict):
                                                               d_type+'_PCP_COMBINE_INPUT_TEMPLATE', '')
         if not c_dict[d_type+'_INPUT_TEMPLATE']:
             self.log_error(d_type + "_PCP_COMBINE_INPUT_TEMPLATE required to run")
+
         c_dict[d_type+'_OUTPUT_DIR'] = self.config.getdir(d_type+'_PCP_COMBINE_OUTPUT_DIR', '')
         c_dict[d_type+'_OUTPUT_TEMPLATE'] = self.config.getraw('filename_templates',
                                      d_type+'_PCP_COMBINE_OUTPUT_TEMPLATE')
@@ -136,31 +136,37 @@ def set_fcst_or_obs_dict_items(self, d_type, c_dict):
             self.config.getraw('config',
                                d_type+'_PCP_COMBINE_EXTRA_LEVELS', '')
         )
+        # fill in missing extra level values with None
+        fill_num = (len(c_dict[f'{d_type}_EXTRA_NAMES']) -
+                    len(c_dict[f'{d_type}_EXTRA_LEVELS']))
+        if fill_num > 0:
+            for num in range(fill_num):
+                c_dict[d_type + '_EXTRA_LEVELS'].append(None)
+
+        c_dict[d_type+'_EXTRA_OUTPUT_NAMES'] = util.getlist(
+            self.config.getraw('config',
+                               d_type+'_PCP_COMBINE_EXTRA_OUTPUT_NAMES', '')
+        )
 
         if run_method not in self.valid_run_methods:
             self.log_error(f"Invalid value for {d_type}_PCP_COMBINE_METHOD: "
                            f"{run_method}. Valid options are "
                            f"{','.join(self.valid_run_methods)}.")
-            self.isOK = False
 
         if run_method == 'DERIVE' and not c_dict[d_type+'_STAT_LIST']:
             self.log_error('Statistic list is empty. ' + \
               'Must set ' + d_type + '_PCP_COMBINE_STAT_LIST if running ' +\
                               'derive mode')
-            self.isOK = False
 
         if not c_dict[d_type+'_INPUT_TEMPLATE'] and c_dict[d_type+'_RUN_METHOD'] != 'SUM':
             self.log_error(f"Must set {d_type}_PCP_COMBINE_INPUT_TEMPLATE unless using SUM method")
-            self.isOK = False
 
         if not c_dict[d_type+'_OUTPUT_TEMPLATE']:
             self.log_error(f"Must set {d_type}_PCP_COMBINE_OUTPUT_TEMPLATE")
-            self.isOK = False
 
         if run_method == 'DERIVE' or run_method == 'ADD':
             if not c_dict[d_type+'_ACCUMS']:
                 self.log_error(f'{d_type}_PCP_COMBINE_INPUT_ACCUMS must be specified.')
-                self.isOK = False
 
             # name list should either be empty or the same length as accum list
             if c_dict[d_type+'_NAMES'] and \
@@ -169,15 +175,13 @@ def set_fcst_or_obs_dict_items(self, d_type, c_dict):
                       'either empty or the same length as ' +\
                       f'{d_type}_PCP_COMBINE_INPUT_ACCUMS list.'
                 self.log_error(msg)
-                self.isOK = False
 
             if c_dict[d_type+'_LEVELS'] and \
               len(c_dict[d_type+'_ACCUMS']) != len(c_dict[d_type+'_LEVELS']):
                 msg = f'{d_type}_PCP_COMBINE_INPUT_LEVELS list should be ' +\
                       'either empty or the same length as ' +\
                       f'{d_type}_PCP_COMBINE_INPUT_ACCUMS list.'
                 self.log_error(msg)
-                self.isOK = False
 
         return c_dict
 
@@ -198,7 +202,8 @@ def clear(self):
         self.name = ""
         self.compress = -1
         self.user_command = ''
-        self.extra_fields = ''
+        self.extra_fields = None
+        self.extra_output = None
 
     def add_input_file(self, filename, addon):
         self.infiles.append(filename)
@@ -571,8 +576,9 @@ def get_command(self):
         if self.extra_fields:
             cmd += self.extra_fields + ' '
 
-        if self.output_name:
-            cmd += f'-name "{self.output_name}" '
+        output_string = self.get_output_string()
+        if output_string:
+            cmd += f'-name {output_string} '
 
         if not self.outfile:
             self.log_error("No output filename specified")
@@ -604,15 +610,46 @@ def get_command(self):
     def get_extra_fields(self, data_src):
         extra_names = self.c_dict.get(data_src + '_EXTRA_NAMES')
         if not extra_names:
-            return
+            return None, None
 
         extra_list = []
 
         extra_levels = self.c_dict.get(data_src + '_EXTRA_LEVELS')
         for name, level in zip(extra_names, extra_levels):
-            extra_list.append(f"-field 'name=\"{name}\"; level=\"{level}\";'")
+            field_fmt = f"-field 'name=\"{name}\";"
+            if level:
+                field_fmt += f" level=\"{level}\";"
+            field_fmt += "'"
+            extra_list.append(field_fmt)
+
+        extra_input_fmt = ' '.join(extra_list)
+
+        # handle extra output names if specified
+        extra_output_names = self.c_dict.get(data_src + '_EXTRA_OUTPUT_NAMES')
+        if not extra_output_names:
+            extra_output_fmt = None
+        else:
+            extra_output_fmt = '","'.join(extra_output_names)
+            extra_output_fmt = f'"{extra_output_fmt}"'
+
+        return extra_input_fmt, extra_output_fmt
+
+    def get_output_string(self):
+        """! If self.output_name is set, add quotes and return the string. If
+        self.extra_output is also set, add the additional names separated by
+        commas inside the quotes.
+
+        @returns formatted string if output name(s) is specified, None if not
+        """
+        if not self.output_name:
+            return None
+
+        output_string = f'"{self.output_name}"'
+        # add extra output field names
+        if self.extra_output:
+            output_string = f'{output_string},{self.extra_output}'
 
-        return ' '.join(extra_list)
+        return output_string
 
     def run_at_time_once(self, time_info, var_list, data_src):
 
@@ -627,7 +664,7 @@ def run_at_time_one_field(self, time_info, var_info, data_src):
         self.clear()
 
         # read additional names/levels to add to command if set
-        self.extra_fields = self.get_extra_fields(data_src)
+        self.extra_fields, self.extra_output = self.get_extra_fields(data_src)
 
         cmd = None
         self.method = self.c_dict[data_src+'_RUN_METHOD']

parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_derive.conf

@@ -109,6 +109,11 @@ FCST_PCP_COMBINE_OUTPUT_ACCUM = 18H
 # number of derivation specified in FCST_PCP_COMBINE_STAT_LIST
 FCST_PCP_COMBINE_OUTPUT_NAME =
 
+# set the following to add additional fields to add to command
+#FCST_PCP_COMBINE_EXTRA_NAMES =
+#FCST_PCP_COMBINE_EXTRA_LEVELS =
+#FCST_PCP_COMBINE_EXTRA_OUTPUT_NAMES =
+
 # If running a MET tool comparison tool after PCPCombine, one can instead set FCST_VAR1_[NAME/LEVELS] to
 # a value starting with A that corresponds to the desired accumulation to use in the comparison
 # this value will be used to determine the accumulation to build with PCPCombine as well