version before cloning it into nismod2

nismod · Nov 20, 2018 · 2b97ad4 · 2b97ad4
1 parent 8ed019b
commit 2b97ad4
Show file tree

Hide file tree

Showing 4 changed files with 35 additions and 15 deletions.
diff --git a/energy_demand/assumptions/strategy_vars_def.py b/energy_demand/assumptions/strategy_vars_def.py
@@ -54,9 +54,9 @@ def load_smif_parameters(
             # Get scenario value
             if mode == 'smif':  #smif mode
 
-                #try:
-                    #scenario_value = data_handle.get_parameter(var_name)
-                #except:
+                try:
+                    scenario_value = data_handle.get_parameter(var_name)
+                except:
                     logging.warning("IMPORTANT WARNING: Pparamter could not be loaded from smif: `%s`", var_name)
 
                     # ------------------------------------
@@ -89,7 +89,6 @@ def load_smif_parameters(
                         # ------------------------------------
                         #TODO
                         # This needs to be fixed by directly loading multiple paramters from SMIF
-                        #scenario_value = sub_var_entries['default_value']
                         scenario_value = sub_var_entries['scenario_value']
                 else: #local running
                     scenario_value = sub_var_entries['scenario_value']

diff --git a/energy_demand/read_write/data_loader.py b/energy_demand/read_write/data_loader.py
@@ -923,7 +923,7 @@ def load_temp_data(
             stations_t_min = np.load(path_t_min)
             stations_t_max = np.load(path_t_max)
             #stations = read_data.read_yaml(path_stations)
-            stations = df.read_csv(path_stations)
+            stations = pd.read_csv(path_stations)
 
             weather_stations_with_data[dummy_year] = stations
 

diff --git a/energy_demand/scripts/s_clean_original_weather_data.py b/energy_demand/scripts/s_clean_original_weather_data.py
@@ -3,8 +3,9 @@
 import os
 import csv
 import collections
-import numpy as np
 from collections import defaultdict
+import numpy as np
+import pandas as pd
 
 from energy_demand.basic import date_prop
 from energy_demand.basic import basic_functions
@@ -19,7 +20,7 @@ def count_sequence_of_zeros(sequence):
             if cnt > max_cnt_zeros:
                 max_cnt_zeros = cnt
             cnt = 0 #reset to zero again
-    
+
     return max_cnt_zeros
 
 def nan_helper(y):
@@ -88,7 +89,6 @@ def run(
         - In case of a leap year the 29 of February is ignored
     """
     print("... starting to clean original weather files")
-    all_values = []
 
     # Stations which are outisde of the uk and are ignored
     stations_outside_UK = [
@@ -114,11 +114,9 @@ def run(
 
     # Annual temperature file
     for file_name in all_annual_raw_files:
-        print(" ")
         print("... reading csv file: " + str(file_name), flush=True)
-        print(" ")
-        path_to_csv = os.path.join(path_files, file_name)
 
+        path_to_csv = os.path.join(path_files, file_name)
         temp_stations = {}
         temp_stations_min_max = defaultdict(dict)
 
@@ -154,8 +152,6 @@ def run(
                     else:
                         air_temp = float(row[35])
 
-                    if station_id == 1585:
-                        print("tt")
                     # Add weather station if not already added to dict
                     if station_id not in temp_stations:
                         if crit_min_max:
@@ -175,8 +171,28 @@ def run(
 
                     temp_stations[station_id][year_hour] = air_temp
 
+        # --------------------
         # Write out single file
-        #all_values
+        # --------------------
+        path_out_stations = os.path.join(path_out_files, 'stations.csv')
+        path_out_t_min = os.path.join(path_out_files, "t_min.npy")
+        path_out_t_max = os.path.join(path_out_files, "t_max.npy")
+
+        # Write out weather stations sorting of data
+        temp_station_names = list(temp_stations.keys())
+
+        df = pd.DataFrame(temp_station_names, columns=['station_id'])
+        df.to_csv(os.path.join(path_out_stations, "stations.csv"), index=False)
+
+        # Write to numpy
+        stations_t_min = list(i['t_min'] for i in temp_stations_min_min.values())
+        stations_t_max = list(i['t_max'] for i in temp_stations_min_max.values())
+        stations_t_min = np.array(stations_t_min)
+        stations_t_max = np.array(stations_t_max)
+
+        print(stations_t_min.shape)
+        np.save(path_out_t_min, stations_t_min)
+        np.save(path_out_t_max, stations_t_max)
 
 
 
@@ -259,10 +275,12 @@ def run(
 
     print("... finished cleaning weather data")
 
+'''
 run(
     path_files="//linux-filestore.ouce.ox.ac.uk/mistral/nismod/data/energy_demand/H-Met_office_weather_data/_meteo_data_2015",
     path_out_files="//linux-filestore.ouce.ox.ac.uk/mistral/nismod/data/energy_demand/H-Met_office_weather_data/_complete_meteo_data_all_yrs_cleaned_min_max",
     crit_missing_values=40,
     crit_nr_of_zeros=500,
     nr_daily_zeros=20,
     crit_min_max=True)
+'''
diff --git a/energy_demand/wrapper_model.py b/energy_demand/wrapper_model.py
@@ -207,6 +207,8 @@ def load_data_before_simulation(
         assumptions=data['assumptions'],
         default_streategy_vars=default_streategy_vars)
 
+    # <<<<<<<<<<<<<<<<<<<<<<<<<<<
+    #'''
     # -----------------------------------------
     # Read user defined stragey variable from csv files
     # TODO with smif update: Needs to be read in by SMIF and passed on directly to here
@@ -224,7 +226,8 @@ def load_data_before_simulation(
     # --------------------------------------------------------
     strategy_vars = data_loader.replace_variable(
         _user_defined_vars, strategy_vars)
-
+    #'''
+    # <<<<<<<<<<<<<<<<<<<<<<<<<<<
     strategy_vars_out = strategy_vars_def.autocomplete_strategy_vars(
         strategy_vars, narrative_crit=True)