working on weather script

nismod · Mar 26, 2019 · e88ce69 · e88ce69
1 parent 72a9c80
commit e88ce69
Show file tree

Hide file tree

Showing 4 changed files with 51 additions and 48 deletions.
diff --git a/energy_demand/scripts/weather_at_home_data_processing/create_realisation.py b/energy_demand/scripts/weather_at_home_data_processing/create_realisation.py
@@ -22,6 +22,7 @@ def remap_year(year):
 def generate_weather_at_home_realisation(
         path_results,
         path_stiching_table,
+        scenarios=range(100),
         years=range(2015, 2051)
     ):
     """
@@ -40,14 +41,13 @@ def generate_weather_at_home_realisation(
     # Realisations
     realisations = list(df_path_stiching_table.columns)
 
-    attributes = ['wss', 'rsds']
+    attributes = ['rsds', 'wss']
 
-    for attribute in attributes:
-
-        columns = ['timestep', 'longitude', 'latitude', 'yearday', attribute]
-
-        for scenario_nr in range(100):
-            realisation = realisations[scenario_nr]
+    for scenario_nr in scenarios:
+        realisation = realisations[scenario_nr]
+
+        for attribute in attributes:
+            columns = ['timestep', 'longitude', 'latitude', 'yearday', attribute]
 
             print("... creating weather data for realisation " + str(realisation), flush=True)
             realisation_out = []
@@ -76,7 +76,7 @@ def generate_weather_at_home_realisation(
                             [sim_yr, station_long, station_lat, yearday, attribute_station[yearday]])
 
             # Write data to csv
-            print("...writing out", fluhs=True)
+            print("...writing out", flush=True)
             df = pd.DataFrame(realisation_out, columns=columns)
             path_out_csv = os.path.join(result_path_realizations, "weather_data_{}__{}.csv".format(realisation, attribute))
             df.to_csv(path_out_csv, index=False)
diff --git a/energy_demand/scripts/weather_at_home_data_processing/extract_weather_data.py b/energy_demand/scripts/weather_at_home_data_processing/extract_weather_data.py
@@ -207,12 +207,12 @@ def write_weather_stations(data_list):
     station_id_cnt = 0
     cnt = 0
 
-    for row in data_list.values:
+    for index in data_list.index:
         if cnt == 359:
 
             # Weather station metadata
-            station_lon = row[1]
-            station_lat = row[0]
+            station_lon = data_list.loc[index, 'lon']
+            station_lat = data_list.loc[index, 'lat']
 
             station_id = "station_id_{}".format(station_id_cnt)
 

diff --git a/energy_demand/scripts/weather_at_home_data_processing/map_weather_data.py b/energy_demand/scripts/weather_at_home_data_processing/map_weather_data.py
@@ -6,14 +6,13 @@
 from haversine import haversine
 
 from energy_demand.basic import basic_functions
-from energy_demand.scripts.weather_scripts import weather_scenario
 
 def spatially_map_data(
         path_results,
         path_weather_at_home_stations,
-        path_input_coordinates
+        path_input_coordinates,
+        scenarios
     ):
-
     scenario_names = ["NF{}".format(i) for i in range(1, 101)]
 
     # Path to scenarios
@@ -23,14 +22,15 @@ def spatially_map_data(
 
     # Read in input stations and coordinages to map
     stations_to_map_to = pd.read_csv(path_input_coordinates)
-    
+
     # Append new columns
     stations_to_map_to['value'] = 0
     stations_to_map_to['parameter'] = 0
     stations_to_map_to['scenario'] = 0
     stations_to_map_to['year'] = 0
 
     # Read in MARIUS grid weather stations
+    print("... read in weather stations per attribute", flush=True)
     attributes = ['wss', 'rsds']
     weather_stations_per_attribute = {}
     for attribute in attributes:
@@ -45,49 +45,50 @@ def spatially_map_data(
                 'longitude': stations_grid_cells.loc[index, 'longitude'],
                 'latitude': stations_grid_cells.loc[index, 'latitude']
             }
-        
+
         weather_stations_per_attribute[attribute] = stations_grid_cells_dict
-    
+
     # Iterate geography and assign closest weather station data
+    closest_weather_ids = {}
     for index in stations_to_map_to.index:
-
+
+        closest_weather_ids[index] = {}
+
         # Marius weather station
         station_lat = stations_to_map_to.loc[index, 'Latitude']
         station_lon = stations_to_map_to.loc[index, 'Longitude']
 
-        data_types = [
-                ('wind', 'wss') ,
-                ('insulation', 'rsds')]
+        data_types = [('wind', 'wss'), ('insulation', 'rsds')]
 
-        closest_weather_ids = {}
         for name_attribute, attribute in data_types:
 
             # Get closest Met Office weather station
             closest_marius_station = get_closest_weather_station(
                 latitude_reg=station_lat,
                 longitude_reg=station_lon,
-                weather_stations=stations_grid_cells_dict)
-
-            closest_weather_ids[name_attribute] = closest_marius_station
-
-        # Weather and solar data for all scenarios
-        for scenario_nr in range(100):
-            scenario_name = scenario_names[scenario_nr]
+                weather_stations=weather_stations_per_attribute[attribute])
 
-            for name_attribute, attribute in data_types.items():
-
-                path_data = os.path.join(path_to_scenario_data, "weather_data_{}__{}.csv".format(scenario_name, attribute))
-                data = pd.read_csv(path_data)
+            closest_weather_ids[index][name_attribute] = closest_marius_station
 
-                closest_weather_station_id = closest_weather_ids[name_attribute]
-                closest_data = data[closest_weather_station_id]
+    # Weather and solar data for all scenarios
+    for scenario_nr in scenarios: #range(100):
+        scenario_name = scenario_names[scenario_nr]
+        print("... {} ".format(scenario_name), flush=True)
 
-                for index in path_to_scenario_data.index:
-
-                    #region_id, Latitude, Longitude, region_name, scenario, parameter, value
-                    stations_to_map_to.loc[index, 'scenario'] = scenario_nr
-                    stations_to_map_to.loc[index, 'parameter'] = attribute
-                    stations_to_map_to.loc[index, 'value'] = closest_data
+        for name_attribute, attribute in data_types:
+            print("    ... {} ".format(name_attribute), flush=True)
+            path_data = os.path.join(path_to_scenario_data, "weather_data_{}__{}.csv".format(scenario_name, attribute))
+            data = pd.read_csv(path_data)
+
+            for index in stations_to_map_to.index:
+                print("        ... {} ".format(index), flush=True)
+                closest_weather_station_id = closest_weather_ids[index][name_attribute]
+                closest_data = data.loc[closest_weather_station_id]
+
+                #region_id, Latitude, Longitude, region_name, scenario, parameter, value
+                stations_to_map_to.loc[index, 'scenario'] = scenario_nr
+                stations_to_map_to.loc[index, 'parameter'] = attribute
+                stations_to_map_to.loc[index, 'value'] = closest_data
 
     # ----------------------------------------------------------
     # Write out data

diff --git a/energy_demand/scripts/weather_at_home_data_processing/workflow.py b/energy_demand/scripts/weather_at_home_data_processing/workflow.py
@@ -30,11 +30,11 @@
 path_extracted_files = "X:/nismod/data/energy_demand/J-MARIUS_data" # Path to folder with extracted files
 path_stiching_table = "X:/nismod/data/energy_demand/J-MARIUS_data/stitching_table/stitching_table_nf.dat" # Path to stiching table
 path_results = "X:/nismod/data/energy_supply/weather_files" # Path to store results
-path_input_coordinates = os.path.abspath("X:/nismod/data/energy_supply/regions.csv") # Path to file with coordinates to map onto
+path_input_coordinates = os.path.abspath("X:/nismod/data/energy_supply/regions_input.csv") # Path to file with coordinates to map onto
 
 extract_data = False
-stich_together = False
-append_closest_weather_data = True
+stich_together = True
+append_closest_weather_data = False
 
 if extract_data:
     # =================================
@@ -43,8 +43,8 @@
     # =================================
     extract_weather_data.weather_dat_prepare(
         path_extracted_files,
-        path_results,
-        years=[2020])
+        path_results)#,
+        #years=[2020])
     print("... finished extracting data")
 
 if stich_together:
@@ -53,7 +53,8 @@
     # =================================
     create_realisation.generate_weather_at_home_realisation(
         path_results=path_results,
-        path_stiching_table=path_stiching_table)
+        path_stiching_table=path_stiching_table,
+        scenarios=[50, 51]) #range(60, 71))
     print("... finished creating realisations")
 
 if append_closest_weather_data:
@@ -63,5 +64,6 @@
     map_weather_data.spatially_map_data(
         path_results=path_results,
         path_weather_at_home_stations=os.path.join(path_results, "_cleaned_csv"),
-        path_input_coordinates=path_input_coordinates)
+        path_input_coordinates=path_input_coordinates,
+        scenarios=range(100))
     print("... append closest weather information")