removed mistral 1 data preparation in initialisation script and updat…

…ed links to data and weather data

energy_demand/main.py

@@ -114,7 +114,7 @@ def energy_demand_model(
     if config['CRITERIA']['cluster_calc']:
         path_weather_data = "/soge-home/staff/cenv0553/_weather_realisation"
     else:
-        path_weather_data = "C:/Users/cenv0553/ED/data/scenarios"
+        path_weather_data = "C:/Users/cenv0553/ED/data/scenarios/climate"
         path_weather_data = "C:/Users/cenv0553/nismod2/data/energy_demand/scenarios"
 
     if name_config_path == 'h_max' or name_config_path == 'l_max' or name_config_path in ['test_run', 'h_max_0', 'h_max_10', 'h_max_0_only', 'h_max_10_only']:

energy_demand/plotting/fig_3_plot_over_time.py

@@ -60,16 +60,6 @@ def clasify_color(diff_mean, diff_std, threshold):
     else:
         color_pos = 0
 
-    '''if diff_mean < 0 and diff_std < 0:
-        color_pos = 3
-    elif diff_mean > 0 and diff_std < 0:
-        color_pos = 2
-    elif diff_mean < 0 and diff_std > 0:
-        color_pos = 4
-    elif diff_mean > 0 and diff_std > 0:
-        color_pos = 1
-        raise Exception("Something strange happend: {} {}".format(diff_mean, diff_std))'''
-
     return color_pos 
 
 def scenario_over_time(

energy_demand/plotting/result_mapping.py

@@ -1320,10 +1320,6 @@ def colors_plus_minus_map(
 
     elif min(bins) < 0:
         #logging.info("negative bins" + str(bins))
-        #print("negative bins" + str(bins))
-        #if len(bins) > 10:
-        #    raise Exception("Too many bins defined: Change interval criteria")
-        #if len(bins) == 10 or len(bins) == 9: # add extra color to reach 10 colors
         if 1== 1:
             if color_order:
                 color_list_pos = getattr(palettable.colorbrewer.sequential, 'Reds_9').hex_colors
@@ -1335,13 +1331,6 @@ def colors_plus_minus_map(
                 color_list_pos.append('#330808')
                 color_list_pos = getattr(palettable.colorbrewer.sequential, 'Greens_9').hex_colors # 'Greens_9'
                 color_list_pos.append('#0e2814')
-        #elif len(bins) == 9:
-        #    if color_order:
-        #        color_list_pos = getattr(palettable.colorbrewer.sequential, 'Reds_9').hex_colors
-        #        color_list_neg = getattr(palettable.colorbrewer.sequential, 'Greens_9').hex_colors
-        #    else:
-        #        color_list_neg = getattr(palettable.colorbrewer.sequential, 'Reds_9').hex_colors   # 'Reds_9'
-        #        color_list_pos = getattr(palettable.colorbrewer.sequential, 'Greens_9').hex_colors # 'Greens_9'
         elif len(bins) == 8:
             if color_order:
                 color_list_pos = getattr(palettable.colorbrewer.sequential, 'Reds_9').hex_colors

energy_demand/read_write/data_loader.py

@@ -483,7 +483,6 @@ def get_local_paths(config_file_path):
         ('path_population_data_for_disaggregation_LAD', 'str'),
         ('path_population_data_for_disaggregation_MSOA', 'str'),
         ('folder_raw_carbon_trust', 'str'),
-        ('folder_path_weater_stations', 'str'),
         ('path_floor_area_virtual_stock_by', 'str'),
         ('path_assumptions_db', 'str'),
         ('data_processed', 'str'),

energy_demand/read_write/read_data.py

@@ -616,11 +616,13 @@ def service_switch(df_service_switches):
         service_share_ey            [str]   Service share of 'tech' in 'switch_yr'
         sector                      [str]   Optional sector specific info where switch applies
     """
+    test_enduses = set([])
     service_switches = []
-    default_parameter = 999.0   #default parameter
+    default_parameter = 999.0 #default parameter
 
     for i in df_service_switches.index:
         enduse = df_service_switches.at[i, 'enduses_service_switch']
+        test_enduses.add(enduse)
         tech = df_service_switches.at[i, 'tech']
         service_share_ey = df_service_switches.at[i, 'switches_service']
         switch_yr = df_service_switches.at[i, 'end_yr']
@@ -639,6 +641,26 @@ def service_switch(df_service_switches):
                     service_share_ey=float(service_share_ey),
                     switch_yr=float(switch_yr),
                     sector=sector))
+
+    # --------------------------------------------
+    # Test if not 100% per enduse is defined
+    # --------------------------------------------
+    for enduse in test_enduses:
+
+        switch_yrs = {}
+        for switch in service_switches:
+            if switch.enduse == enduse:
+                year = switch.switch_yr
+                value = switch.service_share_ey
+                #print("... {}  {}  {}".format(year, value, enduse))
+                if year in switch_yrs.keys():
+                    switch_yrs[year] += value
+                else:
+                    switch_yrs[year] = value
+
+        for year, value in switch_yrs.items():
+            if value != 1.0:
+                raise Exception("WRONG SERVICE SWITHC INPUT AS NOT SUMS TO 1.0 (100%) {} {}  {}".format(value, year, enduse))
 
     return service_switches
 

energy_demand/result_processing/paper_II.py

@@ -93,7 +93,7 @@ def paper_II_plots(
             folder_path_weater_stations,
             path_shapefile=path_shapefile_input,
             fig_path=fig_path)
-        
+
         print("... plotted weather station distribution map")
 
     ####################################################################

energy_demand/scripts/cluster_execution.py

@@ -13,8 +13,7 @@ def my_function(simulation_number):
 
     same_weather_yr = True
     defined_weather_yr = 2015
-
-    run_smif = False 
+    run_smif = False
 
     # --------------------------
     # Get all weather yrs with data (maybe read existing years from data folder)

energy_demand/scripts/s_greate_NISMOD1_data.py

@@ -0,0 +1,55 @@
+
+"""Script to run after installation of HIRE
+
+Script function which are executed after model installation and
+"""
+import os
+import zipfile
+
+from energy_demand.read_write import data_loader
+from energy_demand.scripts.smif_data_related import script_data_preparation_MISTRAL_pop_gva
+
+
+def create_NISMOD1_data(path_to_zip_file, path_out):
+    """
+
+    Arguments
+    ----------
+    """
+    print("... start running initialisation scripts", flush=True)
+    config = data_loader.read_config_file(path_config_file)
+
+    # Extract NISMOD population data
+    path_to_zip_file = os.path.join(path_data, "population-economic-smif-csv-from-nismod-db.zip")
+    path_extraction = os.path.join(path_out, "MISTRAL_pop_gva")
+    zip_ref = zipfile.ZipFile(path_to_zip_file, 'r')
+    zip_ref.extractall(path_extraction)
+    zip_ref.close()
+
+    # Complete gva and pop data for every sector
+    path_pop = os.path.join(local_data_path, "scenarios", "MISTRAL_pop_gva", "data")
+    path_geography = os.path.join(local_data_path, "scenarios", "uk_pop_principal_2015_2050_MSOA_england.csv")
+    geography_name = "lad_uk_2016"
+
+    # All MISTRAL scenarios to prepare with correct config
+    scenarios_to_generate = [
+        'pop-baseline16_econ-c16_fuel-c16',
+        'pop-f_econ-c_fuel-c',
+        'pop-d_econ-c_fuel-c',]
+
+    script_data_preparation_MISTRAL_pop_gva.run(
+        path_to_folder=path_pop,
+        path_MSOA_baseline=path_geography,
+        MSOA_calculations=False,
+        geography_name=geography_name,
+        scenarios_to_generate=scenarios_to_generate)
+
+    print("... successfully finished setup")
+    return
+
+if __name__ == '__main__':
+    """
+    """
+    create_NISMOD1_data(
+        path_to_zip_file="C:/Users/cenv0553/nismod2/data/energy_demand/population-economic-smif-csv-from-nismod-db.zip",
+        path_out="C:/_NISMODI_DATA")

energy_demand/scripts/s_post_installation.py

@@ -30,12 +30,9 @@ def post_install_setup(args):
     print("... start running initialisation scripts", flush=True)
 
     path_config_file = args.config_file
-
-
     config = data_loader.read_config_file(path_config_file)
 
     local_data_path = config['PATHS']['path_local_data']
-
     base_yr = config['CONFIG']['base_yr']
 
     data = {}
@@ -93,16 +90,15 @@ def post_install_setup(args):
     zip_ref.close()
 
     # Complete gva and pop data for every sector
-    data_pop = os.path.join(local_data_path, "scenarios", "MISTRAL_pop_gva", "data")
+    path_pop = os.path.join(local_data_path, "scenarios", "MISTRAL_pop_gva", "data")
     path_geography = os.path.join(local_data_path, "scenarios", "uk_pop_principal_2015_2050_MSOA_england.csv")
     geography_name = "lad_uk_2016"
 
     # All MISTRAL scenarios to prepare with correct config
-    scenarios_to_generate = ['pop-baseline16_econ-c16_fuel-c16']
-    #scenarios_to_generate = []
+    scenarios_to_generate = [] #'pop-baseline16_econ-c16_fuel-c16', 'pop-f_econ-c_fuel-c', 'pop-d_econ-c_fuel-c',]
 
     script_data_preparation_MISTRAL_pop_gva.run(
-        path_to_folder=data_pop,
+        path_to_folder=path_pop,
         path_MSOA_baseline=path_geography,
         MSOA_calculations=False,
         geography_name=geography_name,

energy_demand/scripts/smif_data_related/generate_data_version.py

@@ -156,8 +156,7 @@ def package_data(
     #zip_handler_minimum.write(full_file_path, arcname='units.txt')
 
     # Add other files to full data
-    files_to_add_full = [
-        'population-economic-smif-csv-from-nismod-db.zip']
+    files_to_add_full = [] #'population-economic-smif-csv-from-nismod-db.zip']
 
     for file_to_add in files_to_add_full:
         full_file_path = os.path.join(data_folder_path, file_to_add)
@@ -177,4 +176,4 @@ def package_data(
     """
     # Map command line arguments to function arguments.
     #package_data(*sys.argv[1:])
-    package_data('v0.9.4', 'C:/Users/cenv0553/nismod2/data/energy_demand')
+    package_data('v0.9.6', 'C:/Users/cenv0553/nismod2/data/energy_demand')

local_run_config_file.ini

@@ -34,7 +34,6 @@ path_strategy_vars = C:/Users/cenv0553/ed/data/00_user_defined_variables
 path_population_data_for_disaggregation_LAD = C:/Users/cenv0553/ed/data/_raw_data/J-population_disagg_by/uk_pop_principal_2015_2050.csv  
 folder_raw_carbon_trust = C:/Users/cenv0553/ed/data/_raw_data/G_Carbon_Trust_advanced_metering_trial/
 path_population_data_for_disaggregation_MSOA = C:/Users/cenv0553/ed/data/_raw_data/J-population_disagg_by/uk_pop_principal_2015_2050_MSOA_lad.csv
-folder_path_weater_stations = C:/Users/cenv0553/ed/data/_raw_data/A-temperature_data/cleaned_weather_stations.csv
 path_floor_area_virtual_stock_by = C:/Users/cenv0553/ed/data/_raw_data/K-floor_area/floor_area_LAD_latest.csv
 path_assumptions_db = C:/Users/cenv0553/ed/data/_processed_data/assumptions_from_db
 data_processed = C:/Users/cenv0553/ed/data/_processed_data

local_run_config_file_cluster.ini

@@ -34,7 +34,6 @@ path_strategy_vars = /soge-home/staff/cenv0553/data/00_user_defined_variables
 path_population_data_for_disaggregation_LAD = /soge-home/staff/cenv0553/data/_raw_data/J-population_disagg_by/uk_pop_principal_2015_2050.csv  
 folder_raw_carbon_trust = /soge-home/staff/cenv0553/data/_raw_data/G_Carbon_Trust_advanced_metering_trial/
 path_population_data_for_disaggregation_MSOA = /soge-home/staff/cenv0553/data/_raw_data/J-population_disagg_by/uk_pop_principal_2015_2050_MSOA_lad.csv
-folder_path_weater_stations = /soge-home/staff/cenv0553/data/_raw_data/A-temperature_data/cleaned_weather_stations.csv
 path_floor_area_virtual_stock_by = /soge-home/staff/cenv0553/data/_raw_data/K-floor_area/floor_area_LAD_latest.csv
 path_assumptions_db = /soge-home/staff/cenv0553/data/_processed_data/assumptions_from_db
 data_processed = /soge-home/staff/cenv0553/data/_processed_data

tests/conftest.py

@@ -54,7 +54,6 @@ def config_file(tmpdir):
 #ONS principal projection
 folder_raw_carbon_trust = tests/energy_demand_minimal/_raw_data/G_Carbon_Trust_advanced_metering_trial/
 path_population_data_for_disaggregation_MSOA = tests/energy_demand_minimal/_raw_data/J-population_disagg_by/uk_pop_principal_2015_2050_MSOA_lad.csv
-folder_path_weater_stations = tests/energy_demand_minimal/_raw_data/A-temperature_data/cleaned_weather_stations.csv
 path_floor_area_virtual_stock_by = tests/energy_demand_minimal/_raw_data/K-floor_area/floor_area_LAD_latest.csv
 path_assumptions_db = tests/_processed_data/assumptions_from_db
 data_processed = tests/_processed_data

tests/read_write/test_data_loader.py

@@ -48,7 +48,6 @@ def config_file(tmpdir):
 #ONS principal projection
 folder_raw_carbon_trust = /vagrant/data/energy_demand/energy_demand_minimal/_raw_data/G_Carbon_Trust_advanced_metering_trial/
 path_population_data_for_disaggregation_MSOA = /vagrant/data/energy_demand/energy_demand_minimal/_raw_data/J-population_disagg_by/uk_pop_principal_2015_2050_MSOA_lad.csv
-folder_path_weater_stations = /vagrant/data/energy_demand/energy_demand_minimal/_raw_data/A-temperature_data/cleaned_weather_stations.csv
 path_floor_area_virtual_stock_by = /vagrant/data/energy_demand/energy_demand_minimal/_raw_data/K-floor_area/floor_area_LAD_latest.csv
 path_assumptions_db = /vagrant/data/energy_demand/_processed_data/assumptions_from_db
 data_processed = /vagrant/data/energy_demand/_processed_data