Merge pull request #150 from nismod/rebase_changes

Rebase changes

energy_demand/assumptions/strategy_vars_def.py

@@ -47,8 +47,10 @@ def generate_default_parameter_narratives(
             # Standard narrative for multidimensional narrative
             for sub_var_name, sub_var_entries in var_entries.items():
 
-                scenario_value = sub_var_entries['scenario_value']
-
+                try:
+                    scenario_value = sub_var_entries['scenario_value']
+                except TypeError:
+                    raise TypeError("{}: Var_entries dict: {}".format(var_name, var_entries))
                 # -----------------------------------
                 # Crate single-step default narratives (up to end_year)
                 # -----------------------------------

energy_demand/basic/basic_functions.py

@@ -6,24 +6,6 @@
 import numpy as np
 from pyproj import Proj, transform
 
-def convert_config_to_correct_type(config):
-    """Convert config types into correct types
-    """
-    out_dict = defaultdict(dict)
-
-    for path in config['PATHS']:
-        out_dict['PATHS'][path] = config.get('PATHS', path)
-
-    for config_section in config['CONFIG']:
-        out_dict['CONFIG'][config_section] = config.getint('CONFIG', config_section)
-
-    for criteria in config['CRITERIA']:
-        if criteria == 'reg_selection_csv_name':
-            out_dict['CRITERIA'][criteria] = config.get('CRITERIA', criteria)
-        else:
-            out_dict['CRITERIA'][criteria] = config.getboolean('CRITERIA', criteria)
-
-    return dict(out_dict)
 
 def dict_depth(dictionary):
     """Get depth of nested dict

energy_demand/main.py

@@ -93,9 +93,7 @@ def energy_demand_model(
     print("Configuration path: " + str(path_config))
 
     # Get configuration
-    config = configparser.ConfigParser()
-    config.read(path_config)
-    config = basic_functions.convert_config_to_correct_type(config)
+    config = data_loader.read_config_file(path_config)
 
     #config_file_path = os.path.join(path_main, 'wrapperconfig.ini') 
     #config = data_loader.read_config_file(config_file_path)
@@ -155,14 +153,14 @@ def energy_demand_model(
     # --------------------
     # Load all other paths
     # --------------------
-    data['paths'] = data_loader.load_paths(config['PATHS']['path_energy_demand_config'])
-    data['local_paths'] = data_loader.get_local_paths(local_data_path)
+    data['paths'] = config['CONFIG_DATA']
+    data['local_paths'] = config['DATA_PATHS']
 
     # Manually overwrriting startegy variable path
     data['local_paths']['path_strategy_vars'] = path_strategy_vars
 
-    data['path_new_scenario'] = os.path.abspath(os.path.join(os.path.dirname(local_data_path), "results", name_scenario_run))
-    data['result_paths'] = data_loader.get_result_paths(data['path_new_scenario'])
+    data['path_new_scenario'] = config['PATHS']['path_new_scenario']
+    data['result_paths'] = config['RESULT_DATA']
 
     basic_functions.create_folder(data['path_new_scenario'])
 
@@ -456,7 +454,7 @@ def energy_demand_model(
         # --------------------------------------
         path_folder_weather_yr = os.path.join(data['path_new_scenario'], str("simulation_results"))
 
-        data['result_paths'] = data_loader.get_result_paths(path_folder_weather_yr)
+        data['result_paths'] = data_loader.get_weather_result_paths(path_folder_weather_yr)
 
         folders_to_create = [
             path_folder_weather_yr,

energy_demand/model.py

@@ -1,4 +1,27 @@
 """The function `EnergyDemandModel` executes all the submodels of the energy demand model
+
+Notes
+-----
+some observations from @willu47
+
+- suggest moving bulk of EnergyDemandModel class contents out of __init__ method
+  and into class methods, and then calling each method in turn
+- suggest explictly creating class properties to access results, rather than
+  the ``setattr`` magic
+- potential for parallelisation of the for-loop around regions for generation 
+  by dwelling stocks
+- suggest splitting ``aggregate_across_all_regions`` into separate methods as each
+  of these requires creating a very large array which is then held in memory,
+  along with all of the disaggregated results. Ideally, only those aggregatation
+  which are required could be run, then written to disk and then removed from
+  memory before moving onto the next
+- Lots of potential for a map-reduce approach to this, which could be streamlined
+  by using vectorised numpy methods consistently throughout the codebase rather
+  than for-loops over indexes of the arrays - this will result in memory and
+  performance improvements
+- Otherwise, the many of the methods are nicely packaged and will be easily
+  testable and refactored going forward, so some profiling would help direct
+  attention to where further work is most appropriate
 """
 import logging
 from collections import defaultdict

energy_demand/profiles/hdd_cdd.py

@@ -272,7 +272,7 @@ def effective_temps_min_max(temp_365, nr_day_to_av):
 
     Arguments
     ---------
-    temp_365 : np.array
+    temp_365 : np.ndarray
         Daily min or max temperatures
     nr_day_to_av : int
         Number of previous days to average current day
@@ -289,14 +289,14 @@ def effective_temps_min_max(temp_365, nr_day_to_av):
         effective_temp_365[day] = temp_365[day]
 
     # Iterate days in a year
-    for day in range(365)[nr_day_to_av:]: #Skip first dates in January
+    for day in range(365)[nr_day_to_av:]: # Skip first dates in January
 
         # Add todays temperature and previous effective temps
         tot_temp = temp_365[day]
 
         # Add effective temperature of previous day(s)
         for i in range(nr_day_to_av):
-            tot_temp = tot_temp + effective_temp_365[day - (i+1)] #not +=
+            tot_temp = tot_temp + effective_temp_365[day - (i+1)]  # not +=
 
         effective_temp_365[day] = tot_temp / (nr_day_to_av + 1)
 
@@ -335,10 +335,17 @@ def effective_temps(temp_yh, nr_day_to_av):
     """
     effective_temp_yh = np.zeros((365, 24), dtype="float")
 
+    if not hasattr(temp_yh, 'shape'):
+        msg = "Expected a numpy array, instead received a {}. Maybe set option " \
+              "`crit_temp_min_max = True` in config file."
+
+        raise TypeError(msg.format(type(temp_yh)))
+
     # Copy all border days
     for day in range(nr_day_to_av):
         effective_temp_yh[day] = temp_yh[day]
 
+
     # Iterate days in a year
     for day in range(365)[nr_day_to_av:]: #Skip first dates in January
 
@@ -347,7 +354,7 @@ def effective_temps(temp_yh, nr_day_to_av):
 
         # Add effective temperature of previous day(s)
         for i in range(nr_day_to_av):
-            tot_temp = tot_temp + effective_temp_yh[day - (i+1)] #not +=
+            tot_temp = tot_temp + effective_temp_yh[day - (i + 1)]  # not +=
 
         effective_temp_yh[day] = tot_temp / (nr_day_to_av + 1)