Update develop-ref after dtcenter/MET#2241 (#1770)

Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com>
Co-authored-by: Hank Fisher <fisherh@seneca.rap.ucar.edu>
Co-authored-by: Mrinal Biswas <biswas@seneca.rap.ucar.edu>
Co-authored-by: jprestop <jpresto@ucar.edu>
Co-authored-by: johnhg <johnhg@ucar.edu>
Co-authored-by: Mrinal Biswas <biswas@ucar.edu>
Co-authored-by: Julie Prestopnik <jpresto@seneca.rap.ucar.edu>
Co-authored-by: j-opatz <jopatz@ucar.edu>
Co-authored-by: Kathryn Newman <knewman@seneca.rap.ucar.edu>
Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com>
Co-authored-by: Lisa Goodrich <lisag@ucar.edu>
Co-authored-by: Julie Prestopnik <jpresto@ucar.edu>
Co-authored-by: Christina Kalb <kalb@mohawk.rap.ucar.edu>
Co-authored-by: Christina Kalb <kalb@ucar.edu>
Co-authored-by: Hank Fisher <fisherh@ucar.edu>
Co-authored-by: mrinalbiswas <biswas@ucar.edu>
Co-authored-by: bikegeek <3753118+bikegeek@users.noreply.github.com>
Co-authored-by: lisagoodrich <33230218+lisagoodrich@users.noreply.github.com>
Co-authored-by: Howard Soh <hsoh@seneca.rap.ucar.edu>
Co-authored-by: Molly Smith <molly.b.smith@noaa.gov>
Co-authored-by: hsoh-u <hsoh@ucar.edu>
Co-authored-by: bikegeek <minnawin@ucar.edu>

.github/labels/README.md

@@ -31,7 +31,8 @@ Sample commands for processing all METplus repos:
 ```
 # List of METplus repositories
 REPO_LIST="metplus met metplotpy metcalcpy metdataio metviewer \
-                      metexpress metplus-training metplus-internal";
+                      metexpress metplus-training metplus-internal \
+                      metbaseimage";
 
 # Build commands to add/update common labels
 for repo in ${REPO_LIST}; do

.github/labels/post_patch_labels.sh

@@ -8,7 +8,7 @@
 
 if [[ $# -ne 4 ]]; then
   echo "ERROR: `basename $0` ... must specify the GitHub username, authorization key, repository name, and label file."
-  echo "ERROR:   repo names: metplus, met, metdataio, metcalcpy, metplotpy, metviewer, metexpress, metplus-training, metplus-internal"
+  echo "ERROR:   repo names: metplus, met, metdataio, metcalcpy, metplotpy, metviewer, metexpress, metplus-training, metplus-internal, metbaseimage"
   exit 1
 else
   user=$1

docs/Contributors_Guide/user_support.rst

@@ -48,7 +48,7 @@ Review the `unanswered discussions <https://github.com/dtcenter/METplus/discussi
   * Otherwise, leave the discussion unanswered until the next day to allow the
     user community time to respond.
 
-* For new discussions that have 0 repsonses and were posted the previous day:
+* For new discussions that have 0 responses and were posted the previous day:
 
   * Take action. See :ref:`actions_new_discussions`.
 

metplus/util/config_metplus.py

@@ -251,8 +251,7 @@ def launch(config_list):
 
     # create final conf directory if it doesn't already exist
     final_conf_dir = os.path.dirname(final_conf)
-    if not os.path.exists(final_conf_dir):
-        os.makedirs(final_conf_dir)
+    util.mkdir_p(final_conf_dir)
 
     # set METPLUS_BASE/PARM_BASE conf so they can be referenced in other confs
     config.set('config', 'METPLUS_BASE', METPLUS_BASE)
@@ -372,10 +371,8 @@ def get_logger(config, sublog=None):
     log_dir = config.getdir('LOG_DIR')
     log_level = config.getstr('config', 'LOG_LEVEL')
 
-    # Check if the directory path for the log file exists, if
-    # not create it.
-    if not os.path.exists(log_dir):
-        util.mkdir_p(log_dir)
+    # Create the log directory if it does not exist
+    util.mkdir_p(log_dir)
 
     if sublog is not None:
         logger = config.log(sublog)

metplus/util/met_util.py

@@ -148,15 +148,14 @@ def run_metplus(config, process_list):
                          "Options are processes, times")
             return 1
 
-        # write out all commands and environment variables to file
-        if not write_all_commands(all_commands, config):
-            # if process list contains any wrapper that should run commands,
-            # report an error if no commands were generated
-            if any([item[0] not in NO_COMMAND_WRAPPERS
-                    for item in process_list]):
+        # if process list contains any wrapper that should run commands
+        if any([item[0] not in NO_COMMAND_WRAPPERS for item in process_list]):
+            # write out all commands and environment variables to file
+            if not write_all_commands(all_commands, config):
+                # report an error if no commands were generated
                 total_errors += 1
 
-       # compute total number of errors that occurred and output results
+        # compute total number of errors that occurred and output results
         for process in processes:
             if process.errors != 0:
                 process_name = process.__class__.__name__.replace('Wrapper', '')
@@ -252,9 +251,7 @@ def handle_tmp_dir(config):
     # create temp dir if it doesn't exist already
     # this will fail if TMP_DIR is not set correctly and
     # env MET_TMP_DIR was not set
-    tmp_dir = config.getdir('TMP_DIR')
-    if not os.path.exists(tmp_dir):
-        os.makedirs(tmp_dir)
+    mkdir_p(config.getdir('TMP_DIR'))
 
 def handle_env_var_config(config, env_var_name, config_name):
     """! If environment variable is set, use that value
@@ -1230,9 +1227,7 @@ def preprocess_file(filename, data_type, config, allow_dir=False):
                 return stagefile
             # if it does not exist, run GempakToCF and return staged nc file
             # Create staging area if it does not exist
-            outdir = os.path.dirname(stagefile)
-            if not os.path.exists(outdir):
-                os.makedirs(outdir, mode=0o0775)
+            mkdir_p(os.path.dirname(stagefile))
 
             # only import GempakToCF if needed
             from ..wrappers import GempakToCFWrapper
@@ -1263,9 +1258,7 @@ def preprocess_file(filename, data_type, config, allow_dir=False):
     # Create staging area directory only if file has compression extension
     if any([os.path.isfile(f'{filename}{ext}')
             for ext in COMPRESSION_EXTENSIONS]):
-        outdir = os.path.dirname(outpath)
-        if not os.path.exists(outdir):
-            os.makedirs(outdir, mode=0o0775)
+        mkdir_p(os.path.dirname(outpath))
 
     # uncompress gz, bz2, or zip file
     if os.path.isfile(filename+".gz"):

metplus/util/time_looping.py

@@ -159,6 +159,9 @@ def _get_time_prefix(config):
      valid time, or None if not enough information was found in the config
     """
     loop_by = config.getstr('config', 'LOOP_BY', '').upper()
+    if not loop_by:
+        return None
+
     if loop_by in ['INIT', 'RETRO']:
         return 'INIT'
 

metplus/wrappers/ascii2nc_wrapper.py

@@ -228,15 +228,6 @@ def get_command(self):
         out_path = self.get_output_path()
         cmd += ' ' + out_path
 
-        parent_dir = os.path.dirname(out_path)
-        if parent_dir == '':
-            self.log_error('Must specify path to output file')
-            return None
-
-        # create full output dir if it doesn't already exist
-        if not os.path.exists(parent_dir):
-            os.makedirs(parent_dir)
-
         # add arguments
         cmd += ''.join(self.args)
 

metplus/wrappers/command_builder.py

@@ -912,8 +912,7 @@ def write_list_file(self, filename, file_list, output_dir=None):
 
         list_path = os.path.join(list_dir, filename)
 
-        if not os.path.exists(list_dir):
-            os.makedirs(list_dir, mode=0o0775)
+        util.mkdir_p(list_dir)
 
         self.logger.debug("Writing list of filenames...")
         with open(list_path, 'w') as file_handle:
@@ -1008,7 +1007,7 @@ def find_and_check_output_file(self, time_info=None,
         if (not os.path.exists(parent_dir) and
                 not self.c_dict.get('DO_NOT_RUN_EXE', False)):
             self.logger.debug(f"Creating output directory: {parent_dir}")
-            os.makedirs(parent_dir)
+            util.mkdir_p(parent_dir)
 
         if not output_exists or not skip_if_output_exists:
             return True
@@ -1195,7 +1194,7 @@ def get_command(self):
         """
         if self.app_path is None:
             self.log_error('No app path specified. '
-                              'You must use a subclass')
+                           'You must use a subclass')
             return None
 
         cmd = '{} -v {}'.format(self.app_path, self.c_dict['VERBOSITY'])
@@ -1222,8 +1221,7 @@ def get_command(self):
             self.log_error('Must specify path to output file')
             return None
 
-        if not os.path.exists(parent_dir):
-            os.makedirs(parent_dir)
+        util.mkdir_p(parent_dir)
 
         cmd += " " + out_path
 

metplus/wrappers/compare_gridded_wrapper.py

@@ -312,33 +312,6 @@ def process_fields(self, time_info):
         # run the MET command
         self.build()
 
-    def create_and_set_output_dir(self, time_info):
-        """! Builds the full output dir path with valid or init time
-              Creates output directory if it doesn't already exist
-              Args:
-                @param time_info dictionary with time information
-        """
-        out_dir = self.c_dict['OUTPUT_DIR']
-
-        # use output template if it is set
-        # if not set, do not add any extra directories to path
-        out_template_name = '{}_OUTPUT_TEMPLATE'.format(self.app_name.upper())
-        if self.config.has_option('config',
-                                  out_template_name):
-            template = self.config.getraw('config',
-                                          out_template_name)
-            # perform string substitution to get full path
-            extra_path = do_string_sub(template,
-                                       **time_info)
-            out_dir = os.path.join(out_dir, extra_path)
-
-        # create full output dir if it doesn't already exist
-        if not os.path.exists(out_dir):
-            os.makedirs(out_dir)
-
-        # set output dir for wrapper
-        self.outdir = out_dir
-
     def get_command(self):
         """! Builds the command to run the MET application
            @rtype string

metplus/wrappers/cyclone_plotter_wrapper.py

@@ -37,8 +37,6 @@
     WRAPPER_CANNOT_RUN = True
     EXCEPTION_ERR = err_msg
 
-import produtil.setup
-
 from ..util import met_util as util
 from ..util import do_string_sub
 from ..util import time_generator, add_to_time_input

metplus/wrappers/gen_vx_mask_wrapper.py

@@ -12,9 +12,7 @@
 
 import os
 
-from ..util import getlist
-from ..util import met_util as util
-from ..util import time_util
+from ..util import getlist, get_lead_sequence, skip_time, ti_calculate, mkdir_p
 from . import CommandBuilder
 from ..util import do_string_sub
 
@@ -117,15 +115,6 @@ def get_command(self):
         out_path = self.get_output_path()
         cmd += ' ' + out_path
 
-        parent_dir = os.path.dirname(out_path)
-        if not parent_dir:
-            self.log_error('Must specify path to output file')
-            return None
-
-        # create full output dir if it doesn't already exist
-        if not os.path.exists(parent_dir):
-            os.makedirs(parent_dir)
-
         # add arguments
         cmd += ' ' + self.args
 
@@ -141,14 +130,14 @@ def run_at_time(self, input_dict):
                 @param input_dict dictionary containing timing information
                 @returns None
         """
-        lead_seq = util.get_lead_sequence(self.config, input_dict)
+        lead_seq = get_lead_sequence(self.config, input_dict)
         for lead in lead_seq:
             self.clear()
             input_dict['lead'] = lead
 
-            time_info = time_util.ti_calculate(input_dict)
+            time_info = ti_calculate(input_dict)
 
-            if util.skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})):
+            if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})):
                 self.logger.debug('Skipping run time')
                 continue
 
@@ -193,7 +182,8 @@ def run_at_time_all(self, time_info):
             temp_file = os.path.join(self.config.getdir('STAGING_DIR'),
                                      'gen_vx_mask',
                                      f'temp_{index}.nc')
-            self.set_output_path(temp_file)
+            self.find_and_check_output_file(time_info,
+                                            output_path_template=temp_file)
 
             # run GenVxMask
             self.build()

metplus/wrappers/gfdl_tracker_wrapper.py

@@ -272,11 +272,9 @@ def run_at_time_once(self, input_dict):
             self.log_error("TCVitals file not found")
             return False
 
-        # create output directory if it doesn't exist
-        output_dir = self.c_dict.get('OUTPUT_DIR')
-        if not os.path.exists(output_dir):
-            self.logger.debug(f"Creating output directory: {output_dir}")
-            os.makedirs(output_dir)
+        # get output path
+        if not self.find_and_check_output_file(input_dict):
+            return False
 
         # create sym link to output directory for all files (including tcvit)
         all_output_files, tc_vitals_out = (
@@ -656,11 +654,6 @@ def rename_fort_to_output_path(self, time_info):
                                    self.c_dict.get('OUTPUT_TEMPLATE'))
         output_path = do_string_sub(output_path, **time_info)
 
-        # create parent directory of output path if it does not exist
-        parent_dir = os.path.dirname(output_path)
-        if not os.path.exists(parent_dir):
-            self.logger.debug(f"Creating output directory: {parent_dir}")
-
         # copy fort.64/66 file to new file name
         self.logger.debug(f"Copying {fort_file} file to: {output_path}")
         try:

metplus/wrappers/point2grid_wrapper.py

@@ -12,8 +12,8 @@
 
 import os
 
-from ..util import met_util as util
-from ..util import time_util
+from ..util import get_lead_sequence
+from ..util import ti_calculate
 from ..util import do_string_sub
 from ..util import remove_quotes
 from . import CommandBuilder
@@ -138,15 +138,6 @@ def get_command(self):
         out_path = self.get_output_path()
         cmd += ' ' + out_path
 
-        parent_dir = os.path.dirname(out_path)
-        if parent_dir == '':
-            self.log_error('Must specify path to output file')
-            return None
-
-        # create full output dir if it doesn't already exist
-        if not os.path.exists(parent_dir):
-            os.makedirs(parent_dir)
-
         # add arguments
         cmd += ' ' + ' '.join(self.args)
 
@@ -161,12 +152,12 @@ def run_at_time(self, input_dict):
               Args:
                 @param input_dict dictionary containing timing information
         """
-        lead_seq = util.get_lead_sequence(self.config, input_dict)
+        lead_seq = get_lead_sequence(self.config, input_dict)
         for lead in lead_seq:
             self.clear()
             input_dict['lead'] = lead
 
-            time_info = time_util.ti_calculate(input_dict)
+            time_info = ti_calculate(input_dict)
             for custom_string in self.c_dict['CUSTOM_LOOP_LIST']:
                 if custom_string:
                     self.logger.info(f"Processing custom string: {custom_string}")

metplus/wrappers/series_analysis_wrapper.py

@@ -22,7 +22,7 @@
     WRAPPER_CANNOT_RUN = True
     EXCEPTION_ERR = err_msg
 
-from ..util import getlist, get_storms
+from ..util import getlist, get_storms, mkdir_p
 from ..util import do_string_sub, parse_template, get_tags
 from ..util import get_lead_sequence, get_lead_sequence_groups
 from ..util import ti_get_hours_from_lead, ti_get_seconds_from_lead
@@ -1044,8 +1044,7 @@ def generate_animations(self):
 
         animate_dir = os.path.join(self.c_dict['OUTPUT_DIR'],
                                    'series_animate')
-        if not os.path.exists(animate_dir):
-            os.makedirs(animate_dir)
+        mkdir_p(animate_dir)
 
         for group, files in self.c_dict['PNG_FILES'].items():
             # write list of files to a text file

metplus/wrappers/tc_gen_wrapper.py

@@ -329,15 +329,6 @@ def get_command(self):
         out_path = self.get_output_path()
         cmd += ' -out ' + out_path
 
-        parent_dir = os.path.dirname(out_path)
-        if not parent_dir:
-            self.log_error('Must specify path to output file')
-            return None
-
-        # create full output dir if it doesn't already exist
-        if not os.path.exists(parent_dir):
-            os.makedirs(parent_dir)
-
         return cmd
 
     def run_all_times(self):