Merge pull request #3306 from architecture-building-systems/3304-sens…

…or-data

Read and write sensor data using feather instead of json

cea/inputlocator.py

@@ -970,12 +970,12 @@ def get_solar_radiation_folder(self):
         return self._ensure_folder(self.scenario, 'outputs', 'data', 'solar-radiation')
 
     def get_radiation_building(self, building):
-        """scenario/outputs/data/solar-radiation/${building}_insolation.json"""
+        """scenario/outputs/data/solar-radiation/${building}_radiation.csv"""
         return os.path.join(self.get_solar_radiation_folder(), '%s_radiation.csv' % building)
 
     def get_radiation_building_sensors(self, building):
-        """scenario/outputs/data/solar-radiation/${building}_insolation_Whm2.json"""
-        return os.path.join(self.get_solar_radiation_folder(), '%s_insolation_Whm2.json' % building)
+        """scenario/outputs/data/solar-radiation/${building}_insolation_Whm2.feather"""
+        return os.path.join(self.get_solar_radiation_folder(), '%s_insolation_Whm2.feather' % building)
 
     def get_radiation_metadata(self, building):
         """scenario/outputs/data/solar-radiation/{building}_geometrgy.csv"""

cea/resources/radiation/daysim.py

@@ -18,6 +18,8 @@
 __email__ = "cea@arch.ethz.ch"
 __status__ = "Production"
 
+from pyarrow import feather
+
 from cea.constants import HOURS_IN_YEAR
 from cea.resources.radiation.geometry_generator import BuildingGeometry
 
@@ -325,8 +327,7 @@ def isolation_daysim(chunk_n, cea_daysim, building_names, locator, radiance_para
 
 
 def write_sensor_results(sensor_data_path, sensor_values):
-    with open(sensor_data_path, 'w') as outfile:
-        json.dump(sensor_values.T.to_dict(orient="list"), outfile)
+    feather.write_feather(sensor_values, sensor_data_path, compression="zstd")
 
 
 def write_aggregated_results(building_name, sensor_values, locator, date):

cea/technologies/solar/photovoltaic_thermal.py

@@ -68,7 +68,7 @@ def calc_PVT(locator, config, latitude, longitude, weather_data, date_local, bui
     """
     t0 = time.perf_counter()
 
-    radiation_json_path = locator.get_radiation_building_sensors(building_name)
+    radiation_path = locator.get_radiation_building_sensors(building_name)
     metadata_csv_path = locator.get_radiation_metadata(building_name)
 
     # solar properties
@@ -82,7 +82,7 @@ def calc_PVT(locator, config, latitude, longitude, weather_data, date_local, bui
 
     # select sensor point with sufficient solar radiation
     max_annual_radiation, annual_radiation_threshold, sensors_rad_clean, sensors_metadata_clean = \
-        solar_equations.filter_low_potential(radiation_json_path, metadata_csv_path, config)
+        solar_equations.filter_low_potential(radiation_path, metadata_csv_path, config)
 
     print('filtering low potential sensor points done for building %s' % building_name)
 

cea/technologies/solar/solar_collector.py

@@ -62,7 +62,7 @@ def calc_SC(locator, config, latitude, longitude, weather_data, date_local, buil
 
     type_panel = config.solar.type_SCpanel
 
-    radiation_csv = locator.get_radiation_building_sensors(building=building_name)
+    radiation_path = locator.get_radiation_building_sensors(building=building_name)
     metadata_csv = locator.get_radiation_metadata(building=building_name)
 
     # solar properties
@@ -75,7 +75,7 @@ def calc_SC(locator, config, latitude, longitude, weather_data, date_local, buil
 
     # select sensor point with sufficient solar radiation
     max_annual_radiation, annual_radiation_threshold, sensors_rad_clean, sensors_metadata_clean = \
-        solar_equations.filter_low_potential(radiation_csv, metadata_csv, config)
+        solar_equations.filter_low_potential(radiation_path, metadata_csv, config)
 
     print('filtering low potential sensor points done for building %s' % building_name)
 

cea/utilities/solar_equations.py

@@ -11,6 +11,8 @@
 import datetime
 import collections
 from math import *
+
+from pyarrow import feather
 from timezonefinder import TimezoneFinder
 import pytz
 from cea.constants import HOURS_IN_YEAR
@@ -205,18 +207,18 @@ def get_equation_of_time(day_date):
 
 # filter sensor points with low solar potential
 
-def filter_low_potential(radiation_json_path, metadata_csv_path, config):
+def filter_low_potential(radiation_sensor_path, metadata_csv_path, config):
     """
     To filter the sensor points/hours with low radiation potential.
 
     #. keep sensors above min radiation
     #. eliminate points when hourly production < 50 W/m2
     #. augment the solar radiation due to differences between panel reflectance and original reflectances used in daysim
 
-    :param radiation_csv: solar insulation data on all surfaces of each building
-    :type radiation_csv: .csv
+    :param radiation_sensor_path: solar insulation data on all surfaces of each building
+    :type radiation_csv: str
     :param metadata_csv: solar insulation sensor data of each building
-    :type metadata_csv: .csv
+    :type metadata_csv: str
     :return max_annual_radiation: yearly horizontal radiation [Wh/m2/year]
     :rtype max_annual_radiation: float
     :return annual_radiation_threshold: minimum yearly radiation threshold for sensor selection [Wh/m2/year]
@@ -241,7 +243,7 @@ def f(x):
             return x
 
     # read radiation file
-    sensors_rad = pd.read_json(radiation_json_path)
+    sensors_rad = feather.read_feather(radiation_sensor_path)
     sensors_metadata = pd.read_csv(metadata_csv_path)
 
     # join total radiation to sensor_metadata

environment.yml

@@ -43,6 +43,7 @@ dependencies:
 - pyyaml
 - timezonefinder
 - python-cufflinks
+- pyarrow
 - pip:
   - xlrd
   - xlwt