Merge pull request #1042 from NREL/check-buildstock

Check a buildstock csv against an options_lookup tsv

CHANGELOG.md

@@ -1,7 +1,7 @@
 ## ResStock v3.1.0 (pending)
 
 Features
-- 
+- Ability to check buildstock csv against an options lookup as a command line utility ([#1042](https://github.com/NREL/resstock/pull/1042))
 
 Fixes
 - 

measures/ServerDirectoryCleanup/measure.xml

@@ -3,8 +3,8 @@
   <schema_version>3.0</schema_version>
   <name>server_directory_cleanup</name>
   <uid>ec7d04ad-0b7b-495b-825a-e1b6d28d1d3f</uid>
-  <version_id>ae93524c-0520-46e6-a723-2a6169062ac6</version_id>
-  <version_modified>20230306T204324Z</version_modified>
+  <version_id>0fb92be4-f6da-42f4-bca6-ca79a284a210</version_id>
+  <version_modified>20230307T014103Z</version_modified>
   <xml_checksum>5F1EDF75</xml_checksum>
   <class_name>ServerDirectoryCleanup</class_name>
   <display_name>Server Directory Cleanup</display_name>

test/check_buildstock.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+# Load helper file and sampling file
+resources_dir = File.join(File.dirname(__FILE__), '../resources')
+require File.join(resources_dir, 'buildstock')
+require 'optparse'
+require 'csv'
+require_relative 'integrity_checks'
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{File.basename(__FILE__)} -o outfile -l lookup_file\n e.g., #{File.basename(__FILE__)} -o /path/to/buildstock.csv -l /path/to/options_lookup.tsv"
+
+  opts.on('-o', '--output <STRING>', 'Output file name') do |t|
+    options[:outfile] = t
+  end
+
+  opts.on('-l', '--lookup_file <STRING>', 'Lookup file name') do |t|
+    options[:lookup_file] = t
+  end
+
+  opts.on_tail('-h', '--help', 'Display help') do
+    puts opts
+    exit!
+  end
+end.parse!
+
+if (not options[:outfile]) || (not options[:lookup_file])
+  fail "ERROR: All 2 arguments are required. Call #{File.basename(__FILE__)} -h for usage."
+end
+
+outfile = options[:outfile]
+lookup_file = options[:lookup_file]
+
+t0 = Time.now
+check_buildstock(outfile, lookup_file)
+t1 = Time.now
+
+puts "Checking took: #{((t1 - t0) / 60.0).round(1)} minutes."

test/integrity_checks.rb

@@ -143,56 +143,8 @@ def integrity_check(project_dir_name, housing_characteristics_dir = 'housing_cha
   r = RunSampling.new
   output_file = r.run(project_dir_name, 10000, "#{project_dir_name}.csv", housing_characteristics_dir, lookup_file)
 
-  # Cache {parameter => options}
-  parameters_options = {}
-  CSV.foreach(output_file, headers: true).each do |row|
-    row.each do |parameter_name, option_name|
-      next if parameter_name == 'Building'
-
-      unless parameters_options.keys.include? parameter_name
-        parameters_options[parameter_name] = []
-      end
-
-      unless parameters_options[parameter_name].include? option_name
-        parameters_options[parameter_name] << option_name
-      end
-    end
-  end
-
-  # Cache {parameter => {option => {measure => {arg => value}}}}
-  parameters_options_measure_args = {}
-  parameters_options.each do |parameter_name, option_names|
-    parameters_options_measure_args[parameter_name] = get_measure_args_from_option_names(lookup_csv_data, option_names, parameter_name, lookup_file)
-  end
-
-  # Check that measure arguments aren't getting overwritten
-  err = ''
-  CSV.foreach(output_file, headers: true).each do |row|
-    args_map = {}
-    row.each do |parameter_name, option_name|
-      next if parameter_name == 'Building'
-
-      parameters_options_measure_args[parameter_name][option_name].each do |measure_name, args|
-        args.keys.each do |arg|
-          args_map[[measure_name, arg]] = [] if args_map[[measure_name, arg]].nil?
-          args_map[[measure_name, arg]] << parameter_name
-        end
-      end
-    end
-    args_map.each do |k, v|
-      next unless v.size > 1
-
-      param_names = v.join('", "')
-      measure_name = k[0]
-      arg_name = k[1]
-      next if err.include?(param_names) && err.include?(measure_name) && err.include?(arg_name)
-
-      err += "ERROR: Duplicate measure argument assignment(s) across [\"#{param_names}\"] parameters. #{measure_name} => \"#{arg_name}\" already assigned.\n"
-    end
-  end
-  if not err.empty?
-    raise err
-  end
+  # Check outfile
+  check_buildstock(output_file, lookup_file, lookup_csv_data)
 
   if File.exist?(output_file)
     if project_dir_name == 'project_national'
@@ -321,6 +273,64 @@ def integrity_check_options_lookup_tsv(project_dir_name, housing_characteristics
   end
 end
 
+def check_buildstock(output_file, lookup_file, lookup_csv_data = nil)
+  require 'csv'
+
+  lookup_csv_data = CSV.open(lookup_file, col_sep: "\t").each.to_a if lookup_csv_data.nil?
+
+  # Cache {parameter => options}
+  parameters_options = {}
+  CSV.foreach(output_file, headers: true).each do |row|
+    row.each do |parameter_name, option_name|
+      next if parameter_name == 'Building'
+
+      unless parameters_options.keys.include? parameter_name
+        parameters_options[parameter_name] = []
+      end
+
+      unless parameters_options[parameter_name].include? option_name
+        parameters_options[parameter_name] << option_name
+      end
+    end
+  end
+
+  # Cache {parameter => {option => {measure => {arg => value}}}}
+  parameters_options_measure_args = {}
+  parameters_options.each do |parameter_name, option_names|
+    parameters_options_measure_args[parameter_name] = get_measure_args_from_option_names(lookup_csv_data, option_names, parameter_name, lookup_file)
+  end
+
+  # Check that measure arguments aren't getting overwritten
+  err = ''
+  CSV.foreach(output_file, headers: true).each do |row|
+    args_map = {}
+    row.each do |parameter_name, option_name|
+      next if parameter_name == 'Building'
+
+      parameters_options_measure_args[parameter_name][option_name].each do |measure_name, args|
+        args.keys.each do |arg|
+          args_map[[measure_name, arg]] = [] if args_map[[measure_name, arg]].nil?
+          args_map[[measure_name, arg]] << parameter_name
+        end
+      end
+    end
+    args_map.each do |k, v|
+      next unless v.size > 1
+
+      param_names = v.join('", "')
+      measure_name = k[0]
+      arg_name = k[1]
+      next if err.include?(param_names) && err.include?(measure_name) && err.include?(arg_name)
+
+      err += "ERROR: Duplicate measure argument assignment(s) across [\"#{param_names}\"] parameters. #{measure_name} => \"#{arg_name}\" already assigned.\n"
+    end
+  end
+
+  if not err.empty?
+    raise err
+  end
+end
+
 def check_for_illegal_chars(name, name_type)
   # Check for illegal characters in parameter/option names. These characters are
   # reserved for use in the apply upgrade logic.

test/test_integrity_checks.rb

@@ -2,6 +2,7 @@
 
 require_relative '../resources/hpxml-measures/HPXMLtoOpenStudio/resources/minitest_helper'
 require_relative 'integrity_checks'
+require_relative '../resources/buildstock'
 
 class TestResStockErrors < MiniTest::Test
   def before_setup
@@ -182,4 +183,46 @@ def test_housing_characteristics_missing_lookup_option
       flunk "Should have caused an error but didn't."
     end
   end
+
+  def test_buildstock_csv_valid
+    outfile = File.join(File.dirname(__FILE__), '..', 'test/tests_buildstock_csvs/buildstock_csv_valid/buildstock.csv')
+    lookup_file = File.join(File.dirname(__FILE__), '..', 'resources', 'test_options_lookup.tsv')
+
+    # Method 1: call the method
+    check_buildstock(outfile, lookup_file)
+
+    # Method 2: call from command line
+    cli_path = OpenStudio.getOpenStudioCLI
+    command = "\"#{cli_path}\" test/check_buildstock.rb"
+    command += " -o #{outfile}"
+    command += " -l #{lookup_file}"
+    cli_output = `#{command}`
+    assert(cli_output.include?('Checking took:'))
+  end
+
+  def test_buildstock_csv_bad_parameter
+    begin
+      outfile = File.join(File.dirname(__FILE__), '..', 'test/tests_buildstock_csvs/buildstock_csv_bad_parameter/buildstock.csv')
+      lookup_file = File.join(File.dirname(__FILE__), '..', 'resources', 'test_options_lookup.tsv')
+      check_buildstock(outfile, lookup_file)
+    rescue Exception => e
+      puts e.message
+      assert(e.message.include? "ERROR: Could not find parameter 'Location2' and option 'AL_Birmingham.Muni.AP.722280' in")
+    else
+      flunk "Should have caused an error but didn't."
+    end
+  end
+
+  def test_buildstock_csv_bad_option
+    begin
+      outfile = File.join(File.dirname(__FILE__), '..', 'test/tests_buildstock_csvs/buildstock_csv_bad_option/buildstock.csv')
+      lookup_file = File.join(File.dirname(__FILE__), '..', 'resources', 'test_options_lookup.tsv')
+      check_buildstock(outfile, lookup_file)
+    rescue Exception => e
+      puts e.message
+      assert(e.message.include? "ERROR: Could not find parameter 'Vintage' and option '<1940s' in")
+    else
+      flunk "Should have caused an error but didn't."
+    end
+  end
 end

test/tests_buildstock_csvs/buildstock_csv_bad_option/buildstock.csv

@@ -0,0 +1,4 @@
+Building,Location,Vintage
+1,AL_Birmingham.Muni.AP.722280,<1940s
+2,AL_Huntsville.Intl.AP-Jones.Field.723230,1940s
+3,AL_Mobile-Rgnl.AP.722230,<1950

test/tests_buildstock_csvs/buildstock_csv_bad_parameter/buildstock.csv

@@ -0,0 +1,4 @@
+Building,Location2,Vintage
+1,AL_Birmingham.Muni.AP.722280,<1940
+2,AL_Huntsville.Intl.AP-Jones.Field.723230,1940s
+3,AL_Mobile-Rgnl.AP.722230,<1950

test/tests_buildstock_csvs/buildstock_csv_valid/buildstock.csv

@@ -0,0 +1,4 @@
+Building,Location,Vintage
+1,AL_Birmingham.Muni.AP.722280,<1940
+2,AL_Huntsville.Intl.AP-Jones.Field.723230,1940s
+3,AL_Mobile-Rgnl.AP.722230,<1950