Merge pull request #1039 from NREL/run-analysis-tests

Improve run_analysis tests

measures/ServerDirectoryCleanup/measure.xml

@@ -3,8 +3,8 @@
   <schema_version>3.0</schema_version>
   <name>server_directory_cleanup</name>
   <uid>ec7d04ad-0b7b-495b-825a-e1b6d28d1d3f</uid>
-  <version_id>c158e249-51d2-4197-8dec-9f2ea57222f3</version_id>
-  <version_modified>20230207T212145Z</version_modified>
+  <version_id>ae93524c-0520-46e6-a723-2a6169062ac6</version_id>
+  <version_modified>20230306T204324Z</version_modified>
   <xml_checksum>5F1EDF75</xml_checksum>
   <class_name>ServerDirectoryCleanup</class_name>
   <display_name>Server Directory Cleanup</display_name>

test/test_run_analysis.rb

@@ -39,6 +39,17 @@ def _test_measure_order(osw)
     assert_equal(expected_order, actual_order)
   end
 
+  def _assert_and_puts(output, msg, expect_error = true)
+    includes = output.include?(msg)
+    if !includes && expect_error
+      puts output
+      assert(includes)
+    elsif includes && !expect_error
+      puts output
+      assert(!includes)
+    end
+  end
+
   def test_version
     @command += ' -v'
 
@@ -53,7 +64,7 @@ def test_errors_wrong_path
 
     cli_output = `#{@command}`
 
-    assert(cli_output.include?("Error: YML file does not exist at 'test/yml_bad_value/testing_baseline.yml'."))
+    _assert_and_puts(cli_output, "Error: YML file does not exist at 'test/yml_bad_value/testing_baseline.yml'.")
   end
 
   def test_errors_bad_value
@@ -62,11 +73,11 @@ def test_errors_bad_value
 
     cli_output = `#{@command}`
 
-    assert(cli_output.include?('Failures detected for: 1, 2.'))
+    _assert_and_puts(cli_output, 'Failures detected for: 1, 2.')
 
     cli_output_log = File.read(File.join(@testing_baseline, 'cli_output.log'))
-    assert(cli_output_log.include?('ERROR'))
-    assert(cli_output_log.include?('Run Period End Day of Month (32) must be one of'))
+    _assert_and_puts(cli_output_log, 'ERROR')
+    _assert_and_puts(cli_output_log, 'Run Period End Day of Month (32) must be one of')
   end
 
   def test_errors_already_exists
@@ -76,7 +87,7 @@ def test_errors_already_exists
     `#{@command}`
     cli_output = `#{@command}`
 
-    assert(cli_output.include?("Output directory 'testing_baseline' already exists."))
+    _assert_and_puts(cli_output, "Error: Output directory 'testing_baseline' already exists.")
   end
 
   def test_errors_downselect_resample
@@ -85,7 +96,7 @@ def test_errors_downselect_resample
 
     cli_output = `#{@command}`
 
-    assert(cli_output.include?("Not supporting residential_quota_downselect's 'resample' at this time."))
+    _assert_and_puts(cli_output, "Error: Not supporting residential_quota_downselect's 'resample' at this time.")
   end
 
   def test_errors_weather_files
@@ -96,7 +107,7 @@ def test_errors_weather_files
     assert(!File.exist?(File.join(File.dirname(__FILE__), '../weather')))
     cli_output = `#{@command}`
 
-    assert(cli_output.include?("Must include 'weather_files_url' or 'weather_files_path' in yml."))
+    _assert_and_puts(cli_output, "Error: Must include 'weather_files_url' or 'weather_files_path' in yml.")
     assert(!File.exist?(File.join(File.dirname(__FILE__), '../weather')))
   end
 
@@ -106,7 +117,7 @@ def test_errors_downsampler
 
     cli_output = `#{@command}`
 
-    assert(cli_output.include?("Sampler type 'residential_quota_downsampler' is invalid or not supported."))
+    _assert_and_puts(cli_output, "Error: Sampler type 'residential_quota_downsampler' is invalid or not supported.")
   end
 
   def test_errors_missing_key
@@ -115,27 +126,27 @@ def test_errors_missing_key
 
     cli_output = `#{@command}`
 
-    assert(cli_output.include?("Both 'build_existing_model' and 'simulation_output_report' must be included in yml."))
+    _assert_and_puts(cli_output, "Error: Both 'build_existing_model' and 'simulation_output_report' must be included in yml.")
   end
 
   def test_errors_precomputed_outdated_missing_parameter
     yml = ' -y test/tests_yml_files/yml_precomputed_outdated/testing_baseline_missing.yml'
     @command += yml
 
-    cli_output = `#{@command}` # rubocop:disable Lint/UselessAssignment
+    `#{@command}`
     cli_output = File.read(File.join(@testing_baseline, 'cli_output.log'))
 
-    assert(cli_output.include?('Mismatch between buildstock.csv and options_lookup.tsv. Missing parameters: HVAC Cooling Partial Space Conditioning.'))
+    _assert_and_puts(cli_output, 'Mismatch between buildstock.csv and options_lookup.tsv. Missing parameters: HVAC Cooling Partial Space Conditioning.')
   end
 
   def test_errors_precomputed_outdated_extra_parameter
     yml = ' -y test/tests_yml_files/yml_precomputed_outdated/testing_baseline_extra.yml'
     @command += yml
 
-    cli_output = `#{@command}` # rubocop:disable Lint/UselessAssignment
+    `#{@command}`
     cli_output = File.read(File.join(@testing_baseline, 'cli_output.log'))
 
-    assert(cli_output.include?('Mismatch between buildstock.csv and options_lookup.tsv. Extra parameters: Extra Parameter.'))
+    _assert_and_puts(cli_output, 'Mismatch between buildstock.csv and options_lookup.tsv. Extra parameters: Extra Parameter.')
   end
 
   def test_measures_only
@@ -226,10 +237,7 @@ def test_testing_baseline
     cli_output_log = File.join(@testing_baseline, 'cli_output.log')
     assert(File.exist?(cli_output_log))
     cli_output = File.read(cli_output_log)
-    if cli_output.include?('ERROR')
-      puts cli_output
-      assert(!cli_output.include?('ERROR'))
-    end
+    _assert_and_puts(cli_output, 'ERROR', false)
 
     _test_measure_order(File.join(@testing_baseline, 'testing_baseline-Baseline.osw'))
     results_baseline = File.join(@testing_baseline, 'results-Baseline.csv')
@@ -262,10 +270,7 @@ def test_national_baseline
     cli_output_log = File.join(@national_baseline, 'cli_output.log')
     assert(File.exist?(cli_output_log))
     cli_output = File.read(cli_output_log)
-    if cli_output.include?('ERROR')
-      puts cli_output
-      assert(!cli_output.include?('ERROR'))
-    end
+    _assert_and_puts(cli_output, 'ERROR', false)
 
     _test_measure_order(File.join(@national_baseline, 'national_baseline-Baseline.osw'))
     results_baseline = File.join(@national_baseline, 'results-Baseline.csv')
@@ -299,10 +304,7 @@ def test_testing_upgrades
     cli_output_log = File.join(@testing_upgrades, 'cli_output.log')
     assert(File.exist?(cli_output_log))
     cli_output = File.read(cli_output_log)
-    if cli_output.include?('ERROR')
-      puts cli_output
-      assert(!cli_output.include?('ERROR'))
-    end
+    _assert_and_puts(cli_output, 'ERROR', false)
 
     _test_measure_order(File.join(@testing_upgrades, 'testing_upgrades-Baseline.osw'))
     results_baseline = File.join(@testing_upgrades, 'results-Baseline.csv')
@@ -355,10 +357,7 @@ def test_national_upgrades
     cli_output_log = File.join(@national_upgrades, 'cli_output.log')
     assert(File.exist?(cli_output_log))
     cli_output = File.read(cli_output_log)
-    if cli_output.include?('ERROR')
-      puts cli_output
-      assert(!cli_output.include?('ERROR'))
-    end
+    _assert_and_puts(cli_output, 'ERROR', false)
 
     _test_measure_order(File.join(@national_upgrades, 'national_upgrades-Baseline.osw'))
     results_baseline = File.join(@national_upgrades, 'results-Baseline.csv')

test/tests_yml_files/yml_precomputed_outdated/README.md

@@ -1,6 +1,6 @@
 If the set of testing project TSV files gets changed, the "outdated" sample test files will need to be re-generated. For generating buildstock_extra.csv and buildstock_missing.csv precomputed sample files:
 1. Change n_datapoints for testing_baseline.yml to 2
-2. Run ``openstudio workflow/run_analysis.rb -y project_testing/project_testing.yml -s``
+2. Run ``openstudio workflow/run_analysis.rb -y project_testing/testing_baseline.yml -s``
 3. Copy the buildstock.csv, and either rename to:
    - buildstock_extra.csv; then add column "Extra Parameter" with options of "1" and "2"
    - buildstock_missing.csv; then remove column "HVAC Cooling Partial Space Conditioning"