Skip to content

Commit

Permalink
updating weather generatio nscript for supply model
Browse files Browse the repository at this point in the history
  • Loading branch information
Sven Eggimann committed Mar 25, 2019
1 parent 9971235 commit 72a9c80
Show file tree
Hide file tree
Showing 4 changed files with 200 additions and 171 deletions.
81 changes: 40 additions & 41 deletions energy_demand/scripts/weather_at_home_data_processing/create_realisation.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,44 +40,43 @@ def generate_weather_at_home_realisation(
# Realisations
realisations = list(df_path_stiching_table.columns)

columns = ['timestep', 'longitude', 'latitude', 'yearday', 'wss', 'rlds', 'rsds']

for scenario_nr in range(100):
realisation = realisations[scenario_nr]

print("... creating weather data for realisation " + str(realisation), flush=True)
realisation_out = []

for sim_yr in simulation_yrs:
print(" ... year: " + str(sim_yr), flush=True)

year = remap_year(sim_yr)

stiching_name = df_path_stiching_table[realisation][year]
path_weather_data = os.path.join(result_path_realizations, str(year), stiching_name)
path_wss = os.path.join(path_weather_data, "wss.npy")
path_rsds = os.path.join(path_weather_data, "rsds.npy")
path_stations = os.path.join(path_weather_data, "stations.csv")

wss = np.load(path_wss)
rsds = np.load(path_rsds)

stations = pd.read_csv(path_stations)
stations['timestep'] = sim_yr

nr_stations_array = len(list(stations.values))

for station_cnt in range(nr_stations_array):
wss_station = wss[station_cnt]
rsds_station = rsds[station_cnt]
station_long = stations.loc[station_cnt]['longitude']
station_lat = stations.loc[station_cnt]['latitude']
for yearday in range(365):
realisation_out.append(
[sim_yr, station_long, station_lat, yearday, wss_station[yearday], rsds_station[yearday]])

print("...writing out")
# Write data to csv
df = pd.DataFrame(realisation_out, columns=columns)
path_out_csv = os.path.join(result_path_realizations, "weather_data_{}.csv".format(realisation))
df.to_csv(path_out_csv, index=False)
attributes = ['wss', 'rsds']

for attribute in attributes:

columns = ['timestep', 'longitude', 'latitude', 'yearday', attribute]

for scenario_nr in range(100):
realisation = realisations[scenario_nr]

print("... creating weather data for realisation " + str(realisation), flush=True)
realisation_out = []

for sim_yr in years:
#print(" ... year: " + str(sim_yr), flush=True)
year = remap_year(sim_yr)
stiching_name = df_path_stiching_table[realisation][year]
path_weather_data = os.path.join(path_results, '_cleaned_csv', str(year), stiching_name)

path_attribute = os.path.join(path_weather_data, "{}.npy".format(attribute))
path_attribute_stations = os.path.join(path_results, '_cleaned_csv', "stations_{}.csv".format(attribute))

attribute_data = np.load(path_attribute)

stations = pd.read_csv(path_attribute_stations)
stations['timestep'] = sim_yr
nr_stations_array = len(list(stations.values))

for station_cnt in range(nr_stations_array):
attribute_station = attribute_data[station_cnt]
station_long = stations.loc[station_cnt]['longitude']
station_lat = stations.loc[station_cnt]['latitude']
for yearday in range(365):
realisation_out.append(
[sim_yr, station_long, station_lat, yearday, attribute_station[yearday]])

# Write data to csv
print("...writing out", fluhs=True)
df = pd.DataFrame(realisation_out, columns=columns)
path_out_csv = os.path.join(result_path_realizations, "weather_data_{}__{}.csv".format(realisation, attribute))
df.to_csv(path_out_csv, index=False)
124 changes: 81 additions & 43 deletions energy_demand/scripts/weather_at_home_data_processing/extract_weather_data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ def create_folder(path_folder, name_subfolder=None):
def weather_dat_prepare(
path_extracted_files,
path_results,
years=range(2020, 2049)
years=range(2020, 2050)
):
"""
"""
Expand All @@ -36,49 +36,66 @@ def weather_dat_prepare(

for year in years:
path_year = os.path.join(folder_results, str(year))
create_folder(path_year)

path_realizations = os.path.join(path_extracted_files, str(year))
realization_names = os.listdir(path_realizations)

for realization_name in realization_names:
print("... processing {} {}".format(str(year), str(realization_name)), flush=True)

path_realization = os.path.join(path_year, realization_name)
create_folder(path_realization)

# ------------------------
# Original data to extract
# ------------------------

# Daily mean wind speed at 10 m above ground
path_wind = os.path.join(path_realizations, realization_name, 'daily', 'WAH_{}_wss_daily_g2_{}.nc'.format(realization_name, year))

#Daily mean incoming shortwave radiation at the surface
path_rsds = os.path.join(path_realizations, realization_name, 'daily', 'WAH_{}_rsds_daily_g2_{}.nc'.format(realization_name, year))

# Load data
print(" ..load data", flush=True)
wss = get_temp_data_from_nc(path_wind, 'wss')
rsds = get_temp_data_from_nc(path_rsds, 'rsds')

# Convert 360 day to 365 days
print(" ..extend day", flush=True)
list_wss = extend_360_day_to_365(wss, 'wss')
list_rsds = extend_360_day_to_365(rsds, 'rsds')

# Write out single weather stations as numpy array
print(" ..write out", flush=True)
station_coordinates, stations_wss = write_weather_data(list_wss)
_, stations_rsds = write_weather_data(list_rsds)

# Write to csv
np.save(os.path.join(path_realization, "wss.npy"), stations_wss)
np.save(os.path.join(path_realization, "rsds.npy"), stations_rsds)

# Write weather coordinates
df = pd.DataFrame(station_coordinates, columns=['station_id', 'latitude', 'longitude'])
df.to_csv(os.path.join(folder_results, "stations.csv"), index=False)

if os.path.exists(path_realization):
pass #already extracted
else:
print("... processing {} {}".format(str(year), str(realization_name)), flush=True)
create_folder(path_realization)

# ------------------------
# Original data to extract
# ------------------------

# Daily mean wind speed at 10 m above ground
path_wind = os.path.join(path_realizations, realization_name, 'daily', 'WAH_{}_wss_daily_g2_{}.nc'.format(realization_name, year))

#Daily mean incoming shortwave radiation at the surface
path_rsds = os.path.join(path_realizations, realization_name, 'daily', 'WAH_{}_rsds_daily_g2_{}.nc'.format(realization_name, year))

# Load data
print(" ..load data", flush=True)
wss = get_temp_data_from_nc(path_wind, 'wss')
rsds = get_temp_data_from_nc(path_rsds, 'rsds')

# Write out weather stations
station_coordinates_wss = write_weather_stations(wss)
station_coordinates_rsrds = write_weather_stations(rsds)

# Write weather coordinates
path_station_coordinates_wss = os.path.join(folder_results, "stations_wss.csv")
if os.path.exists(path_station_coordinates_wss):
pass
else:
df = pd.DataFrame(station_coordinates_wss, columns=['station_id', 'latitude', 'longitude'])
df.to_csv(os.path.join(folder_results, "stations_wss.csv"), index=False)

path_station_coordinates_rsrds = os.path.join(folder_results, "stations_rsds.csv")
if os.path.exists(path_station_coordinates_rsrds):
pass
else:
df = pd.DataFrame(station_coordinates_rsrds, columns=['station_id', 'latitude', 'longitude'])
df.to_csv(path_station_coordinates_rsrds, index=False)

# Convert 360 day to 365 days
print(" ..extend day", flush=True)
list_wss = extend_360_day_to_365(wss, 'wss')
list_rsds = extend_360_day_to_365(rsds, 'rsds')

# Write out single weather stations as numpy array
print(" ..write out", flush=True)
data_wss = write_weather_data(list_wss)
data_rsds = write_weather_data(list_rsds)

# Write to csv
np.save(os.path.join(path_realization, "wss.npy"), data_wss)
np.save(os.path.join(path_realization, "rsds.npy"), data_rsds)

print("... finished cleaning weather data")


Expand Down Expand Up @@ -152,8 +169,6 @@ def write_weather_data(data_list):
data_list : list
[[lat, long, yearday365, value]]
"""
station_coordinates = []

assert not len(data_list) % 365 #Check if dividable by 365
nr_stations = int(len(data_list) / 365)

Expand All @@ -171,6 +186,30 @@ def write_weather_data(data_list):
# 365 day data for weather station
stations_data[station_id_cnt] = station_data

# Reset
station_data = np.zeros((365))
station_id_cnt += 1
cnt = -1
cnt += 1

return stations_data

def write_weather_stations(data_list):
"""Write weather data to array
data_list : list
[[lat, long, yearday365, value]]
"""
station_coordinates = []

assert not len(data_list.values) % 360 #Check if dividable by 365
nr_stations = int(len(data_list.values) / 360)

station_id_cnt = 0
cnt = 0

for row in data_list.values:
if cnt == 359:

# Weather station metadata
station_lon = row[1]
station_lat = row[0]
Expand All @@ -181,9 +220,8 @@ def write_weather_data(data_list):
station_coordinates.append([station_id, station_lat, station_lon])

# Reset
station_data = np.zeros((365))
station_id_cnt += 1
cnt = -1
cnt += 1

return station_coordinates, stations_data
return station_coordinates

0 comments on commit 72a9c80

Please sign in to comment.