Merge 9ed6d50 into 0eac3df

epregressions/builds/base.py

@@ -40,8 +40,30 @@ def __init__(self):
     def get_idfs_in_dir(idf_dir: Path) -> Set[Path]:
         idf_path = Path(idf_dir)
         all_idfs_absolute_path = list(idf_path.rglob('*.idf'))
+        all_idfs_absolute_path.extend(list(idf_path.rglob('*.imf')))
         all_idfs_relative_path = set([idf.relative_to(idf_path) for idf in all_idfs_absolute_path])
-        return all_idfs_relative_path
+        known_ignore_list = [
+            # these files are for running EnergyPlus _as an FMU_ and we aren't doing that
+            '_ExternalInterface-actuator.idf',
+            '_ExternalInterface-schedule.idf',
+            '_ExternalInterface-variable.idf',
+            # these files are macro resource files, imported by AbsorptionChiller_Macro.imf
+            'HVAC3ZoneGeometry.imf',
+            'HVAC3ZoneMat-Const.imf',
+            'HVAC3ZoneChillerSpec.imf',
+            'HVAC3Zone-IntGains-Def.imf',
+        ]
+
+        def should_keep(file_path):
+            should_ignore = False
+            for i in known_ignore_list:
+                if i in str(file_path):
+                    should_ignore = True
+                    break
+            return not should_ignore
+
+        filtered_list = filter(should_keep, all_idfs_relative_path)
+        return set(filtered_list)
 
     def set_build_directory(self, build_directory):
         raise NotImplementedError('Must implement set_build_directory(str) in derived classes')

epregressions/diffs/ci_compare_script.py

@@ -191,6 +191,18 @@ def main_function(file_name, base_dir, mod_dir, base_sha, mod_sha, make_public,
             has_small_diffs = True
             print_message("Table small diffs.")
 
+    if entry.idf_diffs and (entry.idf_diffs.diff_type != TextDifferences.EQUAL):
+        has_small_diffs = True
+        print_message("IDF diffs.")
+
+    if entry.stdout_diffs and (entry.stdout_diffs.diff_type != TextDifferences.EQUAL):
+        has_small_diffs = True
+        print_message("StdOut diffs.")
+
+    if entry.stderr_diffs and (entry.stderr_diffs.diff_type != TextDifferences.EQUAL):
+        has_small_diffs = True
+        print_message("StdErr diffs.")
+
     if has_small_diffs:
         print("[decent_ci:test_result:warn]")
 

epregressions/energyplus.py

@@ -38,13 +38,17 @@ def execute_energyplus(e_args: ExecutionArguments):
 
     # Save the current path so we can go back here
     start_path = os.getcwd()
+
+    std_out = b""
+    std_err = b""
+
     try:
 
         new_idd_path = os.path.join(e_args.test_run_directory, 'Energy+.idd')
         shutil.copy(idd_path, new_idd_path)
 
         # Copy the weather file into the simulation directory
-        if e_args.run_type != ForceRunType.DD:
+        if e_args.weather_file_name:
             shutil.copy(e_args.weather_file_name, os.path.join(e_args.test_run_directory, 'in.epw'))
 
         # Switch to the simulation directory
@@ -67,7 +71,9 @@ def execute_energyplus(e_args: ExecutionArguments):
             macro_run = subprocess.Popen(
                 epmacro, shell=True, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE
             )
-            macro_run.communicate()
+            o, e = macro_run.communicate()
+            std_out += o
+            std_err += e
             os.rename('out.idf', 'in.idf')
 
         # Run Preprocessor -- after EPMacro?
@@ -76,21 +82,25 @@ def execute_energyplus(e_args: ExecutionArguments):
                 parametric + ' in.idf', shell=True, stdin=subprocess.DEVNULL,
                 stdout=subprocess.PIPE, stderr=subprocess.PIPE
             )
-            parametric_run.communicate()
+            o, e = parametric_run.communicate()
+            std_out += o
+            std_err += e
             candidate_files = glob.glob('in-*.idf')
             if len(candidate_files) > 0:
                 file_to_run_here = sorted(candidate_files)[0]
                 if os.path.exists('in.idf'):
                     os.remove('in.idf')
                 os.rename(file_to_run_here, 'in.idf')
             else:
-                return [e_args.build_tree['build_dir'], e_args.entry_name, False, False]
+                return [e_args.build_tree['build_dir'], e_args.entry_name, False, False, "Issue with Parametrics"]
 
         # Run ExpandObjects and process as necessary
         expand_objects_run = subprocess.Popen(
             expandobjects, shell=True, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE
         )
-        expand_objects_run.communicate()
+        o, e = expand_objects_run.communicate()
+        std_out += o
+        std_err += e
         if os.path.exists('expanded.idf'):
             if os.path.exists('in.idf'):
                 os.remove('in.idf')
@@ -104,7 +114,9 @@ def execute_energyplus(e_args: ExecutionArguments):
                     basement, shell=True, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
                     stderr=subprocess.PIPE, env=basement_environment
                 )
-                basement_run.communicate()
+                o, e = basement_run.communicate()
+                std_out += o
+                std_err += e
                 with open('EPObjects.TXT') as f:
                     append_text = f.read()
                 with open('in.idf', 'a') as f:
@@ -121,7 +133,9 @@ def execute_energyplus(e_args: ExecutionArguments):
                 slab_run = subprocess.Popen(
                     slab, shell=True, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE
                 )
-                slab_run.communicate()
+                o, e = slab_run.communicate()
+                std_out += o
+                std_err += e
                 with open('SLABSurfaceTemps.TXT') as f:
                     append_text = f.read()
                 with open('in.idf', 'a') as f:
@@ -157,14 +171,14 @@ def execute_energyplus(e_args: ExecutionArguments):
 
         # Execute EnergyPlus
         try:
-            subprocess.check_call(
-                energyplus, shell=True, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+            std_out += subprocess.check_output(
+                energyplus, shell=True, stdin=subprocess.DEVNULL, stderr=subprocess.PIPE
             )
-        except Exception:  # pragma: no cover
+        except subprocess.CalledProcessError as e:  # pragma: no cover
             ...
             # so I can verify that I hit this during the test_case_b_crash test, but if I just have the return in
             #  here alone, it shows as missing on the coverage...wonky
-            return [e_args.build_tree['build_dir'], e_args.entry_name, False, False]
+            return [e_args.build_tree['build_dir'], e_args.entry_name, False, False, str(e)]
 
         # Execute readvars
         if os.path.exists('in.rvi'):
@@ -175,7 +189,9 @@ def execute_energyplus(e_args: ExecutionArguments):
         else:
             csv_run = subprocess.Popen(
                 readvars, shell=True, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        csv_run.communicate()
+        o, e = csv_run.communicate()
+        std_out += o
+        std_err += e
         if os.path.exists('in.mvi'):
             mtr_run = subprocess.Popen(
                 readvars + ' in.mvi', shell=True, stdin=subprocess.DEVNULL,
@@ -189,13 +205,22 @@ def execute_energyplus(e_args: ExecutionArguments):
                 readvars + ' in.mvi', shell=True, stdin=subprocess.DEVNULL,
                 stdout=subprocess.PIPE, stderr=subprocess.PIPE
             )
-        mtr_run.communicate()
+        o, e = mtr_run.communicate()
+        std_out += o
+        std_err += e
+
+        if len(std_out) > 0:
+            with open('eplusout.stdout', 'w') as f:
+                f.write(std_out.decode('utf-8'))
+        if len(std_err) > 0:
+            with open('eplusout.stderr', 'w') as f:
+                f.write(std_err.decode('utf-8'))
 
         os.remove(new_idd_path)
         return [e_args.build_tree['build_dir'], e_args.entry_name, True, False]
 
-    except Exception:
-        return [e_args.build_tree['build_dir'], e_args.entry_name, False, False]
+    except Exception as e:
+        return [e_args.build_tree['build_dir'], e_args.entry_name, False, False, str(e)]
 
     finally:
         os.chdir(start_path)

epregressions/runtests.py

@@ -49,11 +49,12 @@ def __init__(self, force_run_type, num_threads, report_freq, build_a, build_b, s
 
 
 class TestCaseCompleted:
-    def __init__(self, run_directory, case_name, run_status, error_msg_reported_already):
+    def __init__(self, run_directory, case_name, run_status, error_msg_reported_already, extra_message=""):
         self.run_directory = run_directory
         self.case_name = case_name
         self.run_success = run_status
         self.muffle_err_msg = error_msg_reported_already
+        self.extra_message = extra_message
 
 
 # the actual main test suite run class
@@ -258,6 +259,22 @@ def run_build(self, build_tree):
                     os.path.join(test_run_directory, 'HybridZoneModel_TemperatureData.csv')
                 )
 
+            if 'LookupTable.csv' in idf_text:
+                shutil.copy(
+                    os.path.join(build_tree['test_files_dir'], 'LookupTable.csv'),
+                    os.path.join(test_run_directory, 'LookupTable.csv')
+                )
+
+            if 'HybridModel' in this_entry.basename:
+                shutil.copy(
+                    os.path.join(build_tree['test_files_dir'], 'HybridModel_Measurements_with_HVAC.csv'),
+                    os.path.join(test_run_directory, 'HybridModel_Measurements_with_HVAC.csv')
+                )
+                shutil.copy(
+                    os.path.join(build_tree['test_files_dir'], 'HybridModel_Measurements_no_HVAC.csv'),
+                    os.path.join(test_run_directory, 'HybridModel_Measurements_no_HVAC.csv')
+                )
+
             if 'SolarShadingTest_Shading_Data.csv' in idf_text:
                 shutil.copy(
                     os.path.join(build_tree['test_files_dir'], 'SolarShadingTest_Shading_Data.csv'),
@@ -279,19 +296,20 @@ def run_build(self, build_tree):
             # if the file requires the FMUs data set file, bring it
             #  into the test run directory, right now I think it's broken
             if 'ExternalInterface:' in idf_text:
-                self.my_print('Skipping an FMU based file as this is not set up to run yet')
-                continue
-                # os.mkdir(os.path.join(test_run_directory, 'datasets'))
-                # os.mkdir(os.path.join(test_run_directory, 'datasets', 'FMUs'))
-                # source_dir = os.path.join('datasets', 'FMUs')
-                # src_files = os.listdir(source_dir)
-                # for file_name in src_files:
-                #     full_file_name = os.path.join(source_dir, file_name)
-                #     if os.path.isfile(full_file_name):
-                #         shutil.copy(
-                #             full_file_name,
-                #             os.path.join(test_run_directory, 'datasets', 'FMUs')
-                #         )
+                # self.my_print('Skipping an FMU based file as this is not set up to run yet')
+                # continue
+                os.mkdir(os.path.join(test_run_directory, 'datasets'))
+                os.mkdir(os.path.join(test_run_directory, 'datasets', 'FMUs'))
+                fmu_dir = os.path.join(build_tree['data_sets_dir'], 'FMUs')
+                src_files = os.listdir(fmu_dir)
+                for file_name in src_files:
+                    full_file_name = os.path.join(fmu_dir, file_name)
+                    if os.path.isfile(full_file_name):
+                        shutil.copy(
+                            full_file_name,
+                            os.path.join(test_run_directory, 'datasets', 'FMUs')
+                        )
+                idf_text = idf_text.replace('..\\datasets', 'datasets')
 
             # rewrite the idf with the (potentially) modified idf text
             with io.open(
@@ -353,7 +371,7 @@ def run_build(self, build_tree):
         # class and add some extra stuff in there, but I could not figure out how to integrate that along with the
         # `apply_async` approach I am using.  Blech.  Once again, on Windows, this means it will partially not be
         # multithreaded.
-        if frozen and system() in ['Windows', 'Darwin']:  # pragma: no cover -- not covering frozen apps in unit tests
+        if self.number_of_threads == 1 or frozen and system() in ['Windows', 'Darwin']:  # pragma: no cover
             self.my_print("Ignoring num_threads on frozen Windows/Mac instance, just running with one thread.")
             for run in energy_plus_runs:
                 ep_return = self.ep_wrapper(run)
@@ -367,7 +385,7 @@ def run_build(self, build_tree):
 
     def ep_wrapper(self, run_args):  # pragma: no cover -- this is being skipped by coverage?
         if self.id_like_to_stop_now:
-            return
+            return ["", "Cancelled", False, False]
         return execute_energyplus(run_args)
 
     def ep_done(self, results):
@@ -398,10 +416,17 @@ def diff_text_files(file_a, file_b, diff_file):
             "(user input)=",
             "(input file)=",
             "(IDF Directory)=",
+            "(Current Working Directory)=",
             "(Current Working Directory)\"=",
             "ReadVars Run Time",
             "EnergyPlus Program Version",
-            "PythonPlugin: Class"
+            "PythonPlugin: Class",
+            "ExpandObjects Finished. Time:",
+            "EnergyPlus, Version",
+            "EnergyPlus Run Time=",
+            "ParametricPreprocessor Finished. Time:",
+            "ExpandObjects Finished with Error(s). Time:",
+            "Elapsed time: ",
         ]
         for line in txt1:
             if any([x in line for x in skip_strings]):
@@ -784,6 +809,16 @@ def process_diffs_for_one_case(self, this_entry, ci_mode=False):
                 join(case_result_dir_1, 'in.idf'),
                 join(case_result_dir_2, 'in.idf'),
                 join(out_dir, 'in.idf.diff'))), TextDifferences.IDF)
+        if self.both_files_exist(case_result_dir_1, case_result_dir_2, 'eplusout.stdout'):
+            this_entry.add_text_differences(TextDifferences(self.diff_text_files(
+                join(case_result_dir_1, 'eplusout.stdout'),
+                join(case_result_dir_2, 'eplusout.stdout'),
+                join(out_dir, 'eplusout.stdout.diff'))), TextDifferences.STDOUT)
+        if self.both_files_exist(case_result_dir_1, case_result_dir_2, 'eplusout.stderr'):
+            this_entry.add_text_differences(TextDifferences(self.diff_text_files(
+                join(case_result_dir_1, 'eplusout.stderr'),
+                join(case_result_dir_2, 'eplusout.stderr'),
+                join(out_dir, 'eplusout.stderr.diff'))), TextDifferences.STDERR)
         if self.both_files_exist(case_result_dir_1, case_result_dir_2, 'eplusout.audit'):
             this_entry.add_text_differences(TextDifferences(self.diff_text_files(
                 join(case_result_dir_1, 'eplusout.audit'),

epregressions/structures.py

@@ -68,6 +68,8 @@ class TextDifferences:
     SCREEN = 19
     GLHE = 20
     IDF = 21
+    STDOUT = 22
+    STDERR = 23
     # diff types
     EQUAL = 1
     DIFFS = 2
@@ -211,6 +213,8 @@ def __init__(self, name_relative_to_testfiles_dir, epw):
         self.glhe_diffs = None
         self.json_diffs = None
         self.idf_diffs = None
+        self.stdout_diffs = None
+        self.stderr_diffs = None
 
     def add_summary_result(self, end_err_summary):
         self.summary_result = end_err_summary
@@ -270,6 +274,10 @@ def add_text_differences(self, diffs, diff_type):
             self.glhe_diffs = diffs
         elif diff_type == TextDifferences.IDF:
             self.idf_diffs = diffs
+        elif diff_type == TextDifferences.STDOUT:
+            self.stdout_diffs = diffs
+        elif diff_type == TextDifferences.STDERR:
+            self.stderr_diffs = diffs
 
     def add_table_differences(self, diffs):
         self.table_diffs = diffs
@@ -336,6 +344,10 @@ def to_dict(self):
                 response['json_diffs'] = self.json_diffs.to_dict()
             if self.idf_diffs:
                 response['idf_diffs'] = self.idf_diffs.to_dict()
+            if self.stdout_diffs:
+                response['stdout_diffs'] = self.stdout_diffs.to_dict()
+            if self.stderr_diffs:
+                response['stderr_diffs'] = self.stderr_diffs.to_dict()
         return response
 
 
@@ -425,6 +437,8 @@ def add_test_entry(self, this_entry):
             this_entry.dfs_diffs: "dfs",
             this_entry.screen_diffs: "screen",
             this_entry.glhe_diffs: "glhe",
+            this_entry.stdout_diffs: "stdout",
+            this_entry.stderr_diffs: "stderr",
         }
         for diff in text_diff_hash:
             file_type = text_diff_hash[diff]

epregressions/tests/builds/test_base.py

@@ -61,5 +61,8 @@ def test_get_idfs(self):
         open(os.path.join(temp_idf_dir, 'file1.idf'), 'w').write('hi')
         open(os.path.join(temp_idf_dir, 'file2.iQQ'), 'w').write('he')
         open(os.path.join(temp_idf_dir, 'file3.idf'), 'w').write('ha')
+        open(os.path.join(temp_idf_dir, 'file4.imf'), 'w').write('ha')  # macro
+        open(os.path.join(temp_idf_dir, '_ExternalInterface-actuator.idf'), 'w').write('ha')  # ext interface as FMU
+        open(os.path.join(temp_idf_dir, 'HVAC3ZoneGeometry.imf'), 'w').write('ha')  # macro resource file
         # TODO: Modify the test to expect relevant IMF files as well and fix the function
-        self.assertEqual(2, len(self.base_build.get_idfs_in_dir(temp_idf_dir)))
+        self.assertEqual(3, len(self.base_build.get_idfs_in_dir(temp_idf_dir)))