Allow user to provide their own buildstock.csv file

buildstockbatch/base.py

@@ -125,23 +125,40 @@ def __init__(self, project_filename):
         # Call property to create directory and copy weather files there
         _ = self.weather_dir  # noqa: F841
 
+        if 'buildstock_csv' in self.cfg['baseline']:
+            buildstock_csv = self.path_rel_to_projectfile(self.cfg['baseline']['buildstock_csv'])
+            if not os.path.exists(buildstock_csv):
+                raise FileNotFoundError('The buildstock.csv file does not exist at {}'.format(buildstock_csv))
+            df = pd.read_csv(buildstock_csv)
+            n_datapoints = self.cfg['baseline'].get('n_datapoints', df.shape[0])
+            self.cfg['baseline']['n_datapoints'] = n_datapoints
+            if n_datapoints != df.shape[0]:
+                raise RuntimeError(
+                    'A buildstock_csv was provided, so n_datapoints for sampling should not be provided or should be '
+                    'equal to the number of rows in the buildstock.csv file. Remove or comment out '
+                    'baseline->n_datapoints from your project file.'
+                )
+            if 'downselect' in self.cfg:
+                raise RuntimeError(
+                    'A buildstock_csv was provided, which isn\'t compatible with downselecting.'
+                    'Remove or comment out the downselect key from your project file.'
+                )
+
         self.sampler = None
 
+    def path_rel_to_projectfile(self, x):
+        if os.path.isabs(x):
+            return os.path.abspath(x)
+        else:
+            return os.path.abspath(os.path.join(os.path.dirname(self.project_filename), x))
+
     def _get_weather_files(self):
         local_weather_dir = os.path.join(self.project_dir, 'weather')
         for filename in os.listdir(local_weather_dir):
             shutil.copy(os.path.join(local_weather_dir, filename), self.weather_dir)
         if 'weather_files_path' in self.cfg:
             logger.debug('Copying weather files')
-            if os.path.isabs(self.cfg['weather_files_path']):
-                weather_file_path = os.path.abspath(self.cfg['weather_files_path'])
-            else:
-                weather_file_path = os.path.abspath(
-                    os.path.join(
-                        os.path.dirname(self.project_filename),
-                        self.cfg['weather_files_path']
-                    )
-                )
+            weather_file_path = self.path_rel_to_projectfile(self.cfg['weather_files_path'])
             with zipfile.ZipFile(weather_file_path, 'r') as zf:
                 logger.debug('Extracting weather files to: {}'.format(self.weather_dir))
                 zf.extractall(self.weather_dir)
@@ -170,15 +187,7 @@ def weather_dir(self):
 
     @property
     def buildstock_dir(self):
-        if os.path.isabs(self.cfg['buildstock_directory']):
-            d = os.path.abspath(self.cfg['buildstock_directory'])
-        else:
-            d = os.path.abspath(
-                os.path.join(
-                    os.path.dirname(self.project_filename),
-                    self.cfg['buildstock_directory']
-                )
-            )
+        d = self.path_rel_to_projectfile(self.cfg['buildstock_directory'])
         # logger.debug('buildstock_dir = {}'.format(d))
         assert(os.path.isdir(d))
         return d
@@ -203,7 +212,19 @@ def output_dir(self):
     def run_sampling(self, n_datapoints=None):
         if n_datapoints is None:
             n_datapoints = self.cfg['baseline']['n_datapoints']
-        return self.sampler.run_sampling(n_datapoints)
+        if 'buildstock_csv' in self.cfg['baseline']:
+            buildstock_csv = self.path_rel_to_projectfile(self.cfg['baseline']['buildstock_csv'])
+            destination_filename = self.sampler.csv_path
+            if destination_filename != buildstock_csv:
+                if os.path.exists(destination_filename):
+                    os.remove(destination_filename)
+                shutil.copy(
+                    buildstock_csv,
+                    destination_filename
+                )
+            return destination_filename
+        else:
+            return self.sampler.run_sampling(n_datapoints)
 
     def run_batch(self):
         raise NotImplementedError

buildstockbatch/hpc.py

@@ -21,6 +21,7 @@
 import random
 import requests
 import shlex
+import shutil
 import subprocess
 import time
 
@@ -109,6 +110,13 @@ def results_dir(self):
         return results_dir
 
     def run_batch(self):
+        destination_dir = os.path.dirname(self.sampler.csv_path)
+        if os.path.exists(destination_dir):
+            shutil.rmtree(destination_dir)
+        shutil.copytree(
+            os.path.join(self.project_dir, 'housing_characteristics'),
+            destination_dir
+        )
         if 'downselect' in self.cfg:
             buildstock_csv_filename = self.downselect()
         else:

buildstockbatch/localdocker.py

@@ -136,15 +136,7 @@ def results_dir(self):
             'output_directory',
             os.path.join(self.project_dir, 'localResults')
         )
-        if os.path.isabs(results_dir):
-            results_dir = os.path.abspath(results_dir)
-        else:
-            results_dir = os.path.abspath(
-                os.path.join(
-                    os.path.dirname(self.project_filename),
-                    results_dir
-                )
-            )
+        results_dir = self.path_rel_to_projectfile(results_dir)
         if not os.path.isdir(results_dir):
             os.makedirs(results_dir)
         return results_dir

buildstockbatch/sampler/base.py

@@ -16,6 +16,8 @@
 
 class BuildStockSampler(object):
 
+    csv_path = None
+
     def __init__(self, cfg, buildstock_dir, project_dir):
         """
         Create the buildstock.csv file required for batch simulations using this class.

buildstockbatch/sampler/commercial_sobol.py

@@ -39,6 +39,10 @@ def __init__(self, output_dir, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.output_dir = output_dir
 
+    @property
+    def csv_path(self):
+        return os.path.join(self.project_dir, 'buildstock.csv')
+
     def run_sampling(self, n_datapoints):
         """
         Run the commercial sampling.
@@ -60,7 +64,7 @@ def run_sampling(self, n_datapoints):
                 tsv_hash[tsv_file.replace('.tsv', '')] = tsv_df
         dependency_hash, attr_order = self._com_order_tsvs(tsv_hash)
         sample_matrix = self._com_execute_sobol_sampling(attr_order.__len__(), n_datapoints)
-        csv_path = os.path.join(self.project_dir, 'buildstock.csv')
+        csv_path = self.csv_path
         header = 'Building,'
         for item in attr_order:
             header += str(item) + ','

buildstockbatch/sampler/residential_docker.py

@@ -32,6 +32,7 @@ def __init__(self, docker_image, *args, **kwargs):
         """
         super().__init__(*args, **kwargs)
         self.docker_image = docker_image
+        self.csv_path = os.path.join(self.project_dir, 'housing_characteristics', 'buildstock.csv')
 
     def run_sampling(self, n_datapoints):
         """
@@ -61,7 +62,7 @@ def run_sampling(self, n_datapoints):
         for line in container_output.decode('utf-8').split('\n'):
             logger.debug(line)
         logger.debug('Sampling took {:.1f} seconds'.format(tick))
-        destination_filename = os.path.join(self.project_dir, 'housing_characteristics', 'buildstock.csv')
+        destination_filename = self.csv_path
         if os.path.exists(destination_filename):
             os.remove(destination_filename)
         shutil.move(

buildstockbatch/sampler/residential_singularity.py

@@ -36,6 +36,7 @@ def __init__(self, singularity_image, output_dir, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.singularity_image = singularity_image
         self.output_dir = output_dir
+        self.csv_path = os.path.join(self.output_dir, 'housing_characteristics', 'buildstock.csv')
 
     def run_sampling(self, n_datapoints):
         """
@@ -58,16 +59,10 @@ def run_sampling(self, n_datapoints):
             '-o', 'buildstock.csv'
         ]
         subprocess.run(args, check=True, env=os.environ, cwd=self.output_dir)
-        destination_dir = os.path.join(self.output_dir, 'housing_characteristics')
-        if os.path.exists(destination_dir):
-            shutil.rmtree(destination_dir)
-        shutil.copytree(
-            os.path.join(self.project_dir, 'housing_characteristics'),
-            destination_dir
-        )
+        destination_dir = os.path.dirname(self.csv_path)
         assert(os.path.isdir(destination_dir))
         shutil.move(
             os.path.join(self.buildstock_dir, 'resources', 'buildstock.csv'),
             destination_dir
         )
-        return os.path.join(destination_dir, 'buildstock.csv')
+        return self.csv_path

buildstockbatch/test/buildstock.csv

@@ -0,0 +1,11 @@
+Building,Location Region,Location,Location Weather Year,Location Weather Filename,Vintage,Heating Fuel,Usage Level,Geometry Building Type,Geometry Number Units,Geometry Foundation Type,Geometry House Size,Geometry Stories,Geometry Garage,Geometry Shared Walls,Geometry Is Multifamily Low Rise,Occupants,Orientation,Eaves,Door Area,Window Areas,Overhangs,Neighbors,Insulation Unfinished Attic,Insulation Finished Roof,Roof Material,Insulation Slab,Insulation Crawlspace,Insulation Unfinished Basement,Insulation Finished Basement,Insulation Pier Beam,Insulation Interzonal Floor,Insulation Wall,Windows,Doors,Water Heater,Hot Water Fixtures,Hot Water Distribution,Solar Hot Water,HVAC System Is Heat Pump,HVAC System Heat Pump,HVAC System Heating Electricity,HVAC System Heating Natural Gas,HVAC System Heating Propane,HVAC System Heating Fuel Oil,HVAC System Heating Other Fuel,HVAC System Heating None,HVAC System Cooling,HVAC System Cooling Type,Heating Setpoint,Cooling Setpoint,Ceiling Fan,Dehumidifier,Refrigerator,Cooking Range,Dishwasher,Clothes Washer,Clothes Dryer,Lighting,Plug Loads,Misc Extra Refrigerator,Misc Freezer,Misc Gas Fireplace,Misc Gas Grill,Misc Gas Lighting,Misc Hot Tub Spa,Misc Pool,Misc Well Pump,Ducts,Infiltration,Natural Ventilation,Mechanical Ventilation,Bathroom Spot Vent Hour,Range Spot Vent Hour,PV,Days Shifted
+1,CR09,TX_Houston-Bush.Intercontinental.AP.722430,TMY3,USA_TX_Houston-Bush.Intercontinental.AP.722430_TMY3.epw,2000s,Natural Gas,Low,Single-Family Detached,1,Slab,0-1499,1,2 Car,None,False,Auto,North,2 ft,20 ft^2,F15 B15 L15 R15,None,Left/Right at 15ft,"Ceiling R-30, Vented",Uninsulated,"Asphalt Shingles, Medium","2ft R10 Perimeter, R10 Gap",None,None,None,None,R-13,"Wood Stud, R-11","Clear, Single, Metal",Fiberglass,Gas Standard,50%,"Uninsulated, HomeRun, PEX",None,No,None,None,"Gas Furnace, 80% AFUE",None,None,None,None,"AC, SEER 13",Central,69F,75F,National Average,None,EF 15.9,"Gas, 80% Usage","290 Rated kWh, 80% Usage","EnergyStar, 80% Usage","Electric, 80% Usage",100% CFL,50%,"EF 6.9, National Average","EF 12, National Average",National Average,National Average,National Average,National Average,National Average,National Average,"20% Leakage, R-8",10 ACH50,"Year-Round, 3 days/wk",None,Hour21,Hour8,None,Day3
+2,CR04,IL_Chicago-OHare.Intl.AP.725300,TMY3,USA_IL_Chicago-OHare.Intl.AP.725300_TMY3.epw,<1950,Natural Gas,Medium,Single-Family Detached,1,Heated Basement,1500-2499,2+,None,None,False,Auto,East,2 ft,20 ft^2,F15 B15 L15 R15,None,Left/Right at 15ft,"Ceiling R-38, Vented",Uninsulated,"Asphalt Shingles, Medium",None,None,None,Uninsulated,None,Uninsulated,"Wood Stud, Uninsulated","Clear, Single, Non-metal",Fiberglass,Gas Standard,100%,"Uninsulated, TrunkBranch, Copper",None,No,None,None,"Gas Furnace, 80% AFUE",None,None,None,None,"FIXME Room AC, EER 10.7, 20% Conditioned",Room,68F,73F,National Average,None,EF 15.9,"Gas, 100% Usage","290 Rated kWh, 100% Usage","EnergyStar, 100% Usage","Electric, 100% Usage",100% CFL,100%,"EF 6.9, National Average","EF 12, National Average",National Average,National Average,National Average,National Average,National Average,National Average,In Finished Space,15 ACH50,"Year-Round, 3 days/wk",None,Hour2,Hour14,None,Day2
+3,CR08,MO_St.Louis-Lambert.Intl.AP.724340,TMY3,USA_MO_St.Louis-Lambert.Intl.AP.724340_TMY3.epw,<1950,Natural Gas,Medium,Single-Family Detached,1,Heated Basement,0-1499,1,None,None,False,Auto,East,2 ft,20 ft^2,F15 B15 L15 R15,None,Left/Right at 15ft,"Ceiling R-19, Vented",Uninsulated,"Asphalt Shingles, Medium",None,None,None,Uninsulated,None,Uninsulated,"Wood Stud, Uninsulated","Clear, Single, Non-metal",Fiberglass,Gas Standard,100%,"Uninsulated, TrunkBranch, Copper",None,No,None,None,"Gas Furnace, 80% AFUE",None,None,None,None,"AC, SEER 10",Central,68F,74F,National Average,None,EF 10.5,"Electric, 100% Usage","318 Rated kWh, 100% Usage","EnergyStar, 100% Usage","Electric, 100% Usage",60% CFL,100%,"EF 6.9, National Average","EF 12, National Average",National Average,National Average,National Average,National Average,National Average,National Average,In Finished Space,20 ACH50,"Year-Round, 3 days/wk",None,Hour5,Hour17,None,Day7
+4,CR02,MN_Minneapolis-St.Paul.Intl.AP.726580,TMY3,USA_MN_Minneapolis-St.Paul.Intl.AP.726580_TMY3.epw,<1950,Natural Gas,Low,Single-Family Detached,1,Heated Basement,1500-2499,1,None,None,False,Auto,West,2 ft,20 ft^2,F15 B15 L15 R15,None,Left/Right at 15ft,"Ceiling R-30, Vented",R-30,"Asphalt Shingles, Medium",None,None,None,Uninsulated,None,Uninsulated,"Wood Stud, Uninsulated","Clear, Single, Non-metal",Fiberglass,Gas Standard,50%,"Uninsulated, TrunkBranch, Copper",None,No,None,None,"Gas Furnace, 80% AFUE",None,None,None,None,"AC, SEER 10",Central,67F,73F,National Average,None,EF 15.9,"Electric, 80% Usage","290 Rated kWh, 80% Usage","Standard, 80% Usage","Electric, 80% Usage",100% CFL,50%,"EF 6.9, National Average","EF 12, National Average",National Average,National Average,National Average,National Average,National Average,National Average,In Finished Space,10 ACH50,"Year-Round, 3 days/wk",None,Hour0,Hour19,None,Day5
+5,CR04,MI_Detroit.Metro.AP.725370,TMY3,USA_MI_Detroit.Metro.AP.725370_TMY3.epw,<1950,Natural Gas,High,Single-Family Detached,1,Heated Basement,0-1499,2+,None,None,False,Auto,South,2 ft,20 ft^2,F15 B15 L15 R15,None,Left/Right at 15ft,"Ceiling R-19, Vented",Uninsulated,"Asphalt Shingles, Medium",None,None,None,Uninsulated,None,Uninsulated,"Wood Stud, Uninsulated","Clear, Double, Non-metal, Air",Fiberglass,Gas Standard,200%,"Uninsulated, TrunkBranch, Copper",None,No,None,None,"Gas Furnace, 92.5% AFUE",None,None,None,None,"AC, SEER 10",Central,68F,72F,National Average,None,EF 15.9,"Gas, 120% Usage","290 Rated kWh, 120% Usage","EnergyStar, 120% Usage","Electric, 120% Usage",100% Incandescent,200%,"EF 6.9, National Average","EF 12, National Average",National Average,National Average,National Average,National Average,National Average,National Average,In Finished Space,15 ACH50,"Year-Round, 3 days/wk",None,Hour22,Hour17,None,Day4
+6,CR09,FL_Orlando.Intl.AP.722050,TMY3,USA_FL_Orlando.Intl.AP.722050_TMY3.epw,2000s,Electricity,Medium,Single-Family Detached,1,Slab,1500-2499,1,2 Car,None,False,Auto,North,2 ft,20 ft^2,F15 B15 L15 R15,None,Left/Right at 15ft,"Ceiling R-38, Vented",Uninsulated,"Asphalt Shingles, Medium",Uninsulated,None,None,None,None,R-13,"Wood Stud, R-11","Clear, Double, Non-metal, Air",Fiberglass,Electric Standard,100%,"Uninsulated, HomeRun, PEX",None,No,None,Electric Furnace,None,None,None,None,None,"AC, SEER 10",Central,70F,75F,National Average,None,EF 15.9,"Electric, 100% Usage","318 Rated kWh, 100% Usage","Standard, 100% Usage","Electric, 100% Usage",100% CFL,100%,"EF 6.9, National Average","EF 12, National Average",National Average,National Average,National Average,National Average,National Average,National Average,"30% Leakage, R-8",8 ACH50,"Year-Round, 3 days/wk",None,Hour1,Hour11,None,Day9
+7,CR11,CA_Long.Beach-Daugherty.Field.722970,TMY3,USA_CA_Long.Beach-Daugherty.Field.722970_TMY3.epw,1950s,Natural Gas,High,Single-Family Detached,1,Slab,0-1499,1,None,None,False,Auto,South,2 ft,20 ft^2,F15 B15 L15 R15,None,Left/Right at 15ft,"Ceiling R-30, Vented",R-30,"Asphalt Shingles, Medium",Uninsulated,None,None,None,None,Uninsulated,"Wood Stud, Uninsulated","Clear, Single, Metal",Fiberglass,Gas Standard,200%,"Uninsulated, TrunkBranch, Copper",None,No,None,None,"Gas Furnace, 80% AFUE",None,None,None,None,None,None,70F,73F,National Average,None,EF 15.9,"Gas, 120% Usage","318 Rated kWh, 120% Usage","Standard, 120% Usage","Gas, 120% Usage",100% Incandescent,200%,"EF 6.9, National Average","EF 12, National Average",National Average,National Average,National Average,National Average,National Average,National Average,"20% Leakage, R-4",20 ACH50,"Year-Round, 3 days/wk",None,Hour6,Hour16,None,Day6
+8,CR07,NJ_Newark.Intl.AP.725020,TMY3,USA_NJ_Newark.Intl.AP.725020_TMY3.epw,<1950,Natural Gas,Medium,Single-Family Detached,1,Crawl,1500-2499,2+,None,None,False,Auto,West,2 ft,20 ft^2,F15 B15 L15 R15,None,Left/Right at 15ft,"Ceiling R-19, Vented",R-30,"Asphalt Shingles, Medium",None,"Uninsulated, Vented",None,None,None,Uninsulated,"Wood Stud, Uninsulated","Clear, Single, Non-metal",Fiberglass,Gas Standard,100%,"Uninsulated, TrunkBranch, Copper",None,No,None,None,"Gas Furnace, 80% AFUE",None,None,None,None,"FIXME Room AC, EER 10.7, 20% Conditioned",Room,68F,73F,National Average,None,EF 15.9,"Gas, 100% Usage","290 Rated kWh, 100% Usage","Standard, 100% Usage","Gas, 100% Usage",100% Incandescent,100%,"EF 6.9, National Average","EF 12, National Average",National Average,National Average,National Average,National Average,National Average,National Average,"20% Leakage, Uninsulated",15 ACH50,"Year-Round, 3 days/wk",None,Hour23,Hour9,None,Day1
+9,CR03,CT_Bridgeport-Sikorsky.Mem.AP.725040,TMY3,USA_CT_Bridgeport-Sikorsky.Mem.AP.725040_TMY3.epw,<1950,Fuel Oil,Low,Single-Family Detached,1,Unheated Basement,1500-2499,2+,2 Car,None,False,Auto,East,2 ft,20 ft^2,F15 B15 L15 R15,None,Left/Right at 15ft,"Ceiling R-19, Vented",R-30,"Asphalt Shingles, Medium",None,None,Uninsulated,None,None,Uninsulated,"Wood Stud, Uninsulated","Clear, Double, Non-metal, Air",Fiberglass,Oil Standard,50%,"Uninsulated, TrunkBranch, Copper",None,No,None,None,None,None,"Oil Furnace, 80% AFUE",None,None,"FIXME Room AC, EER 10.7, 20% Conditioned",Room,68F,73F,National Average,None,EF 10.5,"Electric, 80% Usage","318 Rated kWh, 80% Usage","EnergyStar, 80% Usage","Electric, 80% Usage",100% CFL,50%,"EF 6.9, National Average","EF 12, National Average",National Average,National Average,National Average,National Average,National Average,National Average,"20% Leakage, Uninsulated",15 ACH50,"Year-Round, 3 days/wk",None,Hour3,Hour18,None,Day8
+10,CR09,TX_Dallas-Fort.Worth.Intl.AP.722590,TMY3,USA_TX_Dallas-Fort.Worth.Intl.AP.722590_TMY3.epw,2000s,Natural Gas,Medium,Single-Family Detached,1,Slab,2500-3499,2+,2 Car,None,False,Auto,South,2 ft,20 ft^2,F15 B15 L15 R15,None,Left/Right at 15ft,"Ceiling R-30, Vented",R-30,"Asphalt Shingles, Medium",Uninsulated,None,None,None,None,R-13,"Wood Stud, R-19","Low-E, Double, Non-metal, Air, L-Gain",Fiberglass,Gas Standard,100%,"Uninsulated, HomeRun, PEX",None,No,None,None,"Gas Furnace, 92.5% AFUE",None,None,None,None,"AC, SEER 10",Central,69F,75F,National Average,None,EF 15.9,"Gas, 100% Usage","290 Rated kWh, 100% Usage","EnergyStar, 100% Usage","Electric, 100% Usage",100% Incandescent,100%,"EF 6.9, National Average","EF 12, National Average",National Average,National Average,National Average,National Average,National Average,National Average,"10% Leakage, R-8",7 ACH50,"Year-Round, 3 days/wk",None,Hour4,Hour12,None,Day0