From a07d249c2a3839af73e81a8228b50568fa77aafa Mon Sep 17 00:00:00 2001 From: uvchik Date: Wed, 6 Nov 2019 16:01:53 +0100 Subject: [PATCH 01/16] Bring naming and parameter order in line --- deflex/basic_scenario.py | 69 ++++++++++++++++------------------ deflex/de21_default.ini_backup | 2 +- deflex/deflex.ini | 2 +- deflex/demand.py | 4 +- deflex/powerplants.py | 13 +++---- deflex/scenario_tools.py | 2 +- deflex/transmission.py | 15 ++++++++ tests/test_power_plants.py | 15 +++++--- 8 files changed, 68 insertions(+), 54 deletions(-) diff --git a/deflex/basic_scenario.py b/deflex/basic_scenario.py index bb42232b..3ea4cab6 100644 --- a/deflex/basic_scenario.py +++ b/deflex/basic_scenario.py @@ -34,34 +34,34 @@ from deflex import config as cfg -def create_scenario(year, geo, round_values, weather_year=None): +def create_scenario(regions, year, name, round_values=True, weather_year=None): table_collection = {} logging.info('BASIC SCENARIO - STORAGES') - table_collection['storages'] = scenario_storages(geo) + table_collection['storages'] = scenario_storages(regions, year, name) logging.info('BASIC SCENARIO - POWER PLANTS') - table_collection = powerplants_scenario( - table_collection, year, round_values) + table_collection = scenario_powerplants( + table_collection, regions, year, name, round_values) logging.info('BASIC SCENARIO - TRANSMISSION') table_collection['transmission'] = scenario_transmission(table_collection) logging.info('BASIC SCENARIO - CHP PLANTS') - table_collection = chp_scenario(table_collection, year, geo, + table_collection = scenario_chp(table_collection, regions, year, name, weather_year=weather_year) logging.info('BASIC SCENARIO - DECENTRALISED HEAT') - table_collection['decentralised_heating'] = decentralised_heating() + table_collection['decentralised_heat'] = scenario_decentralised_heat() logging.info('BASIC SCENARIO - SOURCES') - table_collection = create_commodity_sources(year, table_collection) + table_collection = scenario_commodity_sources(year, table_collection) table_collection['volatile_series'] = scenario_feedin( - year, weather_year=weather_year) + regions, year, name, weather_year=weather_year) logging.info('BASIC SCENARIO - DEMAND') table_collection['demand_series'] = scenario_demand( - year, geo, weather_year=weather_year) + regions, year, name, weather_year=weather_year) return table_collection @@ -96,10 +96,9 @@ def scenario_transmission(table_collection): return elec_trans -def scenario_storages(regions): - name = '{0}_region'.format(cfg.get('init', 'map')) - stor = storages.pumped_hydroelectric_storage( - regions, name).transpose() +def scenario_storages(regions, year, name): + stor = storages.pumped_hydroelectric_storage_by_region( + regions, year, name).transpose() return pd.concat([stor], axis=1, keys=['phes']).swaplevel(0, 1, 1) @@ -129,8 +128,8 @@ def add_pp_limit(table_collection, year): return table_collection -def create_commodity_sources(year, table_collection): - commodity_src = scenario_commodity_sources(year) +def scenario_commodity_sources(year, table_collection): + commodity_src = create_commodity_sources(year) commodity_src = commodity_src.swaplevel().unstack() msg = ("The unit for {0} of the source is '{1}'. " @@ -162,7 +161,7 @@ def create_commodity_sources(year, table_collection): return table_collection -def scenario_commodity_sources(year, use_znes_2014=True): +def create_commodity_sources(year, use_znes_2014=True): cs = commodity_sources.get_commodity_sources() rename_cols = {key.lower(): value for key, value in cfg.get_dict('source_names').items()} @@ -178,29 +177,29 @@ def scenario_commodity_sources(year, use_znes_2014=True): return cs_year -def scenario_demand(year, geo, weather_year=None): - demand_series = scenario_elec_demand(year, pd.DataFrame(), geo, +def scenario_demand(regions, year, name, weather_year=None): + demand_series = scenario_elec_demand(pd.DataFrame(), regions, year, name, weather_year=weather_year) - demand_series = scenario_heat_demand(year, demand_series, geo, + demand_series = scenario_heat_demand(demand_series, regions, year, weather_year=weather_year) return demand_series -def scenario_heat_demand(year, table, geo, weather_year=None): +def scenario_heat_demand(table, regions, year, weather_year=None): idx = table.index # Use the index of the existing time series table = pd.concat([table, demand.get_heat_profiles_deflex( - year, geo, idx, weather_year=weather_year)], axis=1) + regions, year, idx, weather_year=weather_year)], axis=1) return table.sort_index(1) -def scenario_elec_demand(year, table, geo, weather_year=None): +def scenario_elec_demand(table, regions, year, name, weather_year=None): if weather_year is None: demand_year = year else: demand_year = weather_year df = demand_elec.get_entsoe_profile_by_region( - geo, demand_year, geo.name, annual_demand='bmwi') + regions, demand_year, name, annual_demand='bmwi') df = pd.concat([df], axis=1, keys=['electrical_load']).swaplevel(0, 1, 1) df = df.reset_index(drop=True) if not calendar.isleap(year) and len(df) > 8760: @@ -208,35 +207,34 @@ def scenario_elec_demand(year, table, geo, weather_year=None): return pd.concat([table, df], axis=1).sort_index(1) -def scenario_feedin(year, weather_year=None): - name = '{0}_region'.format(cfg.get('init', 'map')) +def scenario_feedin(regions, year, name, weather_year=None): wy = weather_year try: feedin = coastdat.scenario_feedin(year, name, weather_year=wy) except FileNotFoundError: - d_regions = geometries.deflex_regions() coastdat.get_feedin_per_region( - year, d_regions, name, weather_year=wy) + year, regions, name, weather_year=wy) feedin = coastdat.scenario_feedin(year, name, weather_year=wy) return feedin -def decentralised_heating(): +def scenario_decentralised_heat(): filename = os.path.join(cfg.get('paths', 'data_deflex'), cfg.get('heating', 'table')) return pd.read_csv(filename, header=[0, 1], index_col=[0]) -def chp_scenario(table_collection, year, geo, weather_year=None): +def scenario_chp(table_collection, regions, year, name, weather_year=None): # values from heat balance - cb = energy_balance.get_transformation_balance_by_region(year, geo) + cb = energy_balance.get_transformation_balance_by_region( + regions, year, name) cb.rename(columns={'re': cfg.get('chp', 'renewable_source')}, inplace=True) heat_b = reegis_powerplants.calculate_chp_share_and_efficiency(cb) heat_demand = demand.get_heat_profiles_deflex( - year, geo, weather_year=weather_year) + year, regions, weather_year=weather_year) return chp_table(heat_b, heat_demand, table_collection) @@ -325,13 +323,12 @@ def chp_table(heat_b, heat_demand, table_collection, regions=None): return table_collection -def powerplants_scenario(table_collection, year, round_values=None): +def scenario_powerplants(table_collection, regions, year, name, round_values): """Get power plants for the scenario year """ - pp = powerplants.get_deflex_pp_by_year(year, overwrite_capacity=True) - region_column = '{0}_region'.format(cfg.get('init', 'map')) - return create_powerplants(pp, table_collection, year, region_column, - round_values) + pp = powerplants.get_deflex_pp_by_year(regions, year, name, + overwrite_capacity=True) + return create_powerplants(pp, table_collection, year, name, round_values) def create_powerplants(pp, table_collection, year, diff --git a/deflex/de21_default.ini_backup b/deflex/de21_default.ini_backup index 85d6d38b..ff5ebe67 100644 --- a/deflex/de21_default.ini_backup +++ b/deflex/de21_default.ini_backup @@ -77,7 +77,7 @@ readme_file = timeseries_readme.md json_file = timeseries_datapackage.json [heating] -table = decentralised_heating.csv +table = scenario_decentralised_heat.csv [demand] ego_file_deflex = oep_ego_demand_{map}.h5 diff --git a/deflex/deflex.ini b/deflex/deflex.ini index 345bae71..577e69c4 100644 --- a/deflex/deflex.ini +++ b/deflex/deflex.ini @@ -68,7 +68,7 @@ readme_file = timeseries_readme.md json_file = timeseries_datapackage.json [heating] -table = decentralised_heating.csv +table = scenario_decentralised_heat.csv [demand_heat] heat_profile_region = heat_profile_{map}_{year}.csv diff --git a/deflex/demand.py b/deflex/demand.py index e6d6e090..3b8b31b6 100644 --- a/deflex/demand.py +++ b/deflex/demand.py @@ -19,7 +19,7 @@ from reegis import demand_heat -def get_heat_profiles_deflex(year, deflex_geo, time_index=None, +def get_heat_profiles_deflex(deflex_geo, year, time_index=None, weather_year=None, keep_unit=False): """ @@ -49,7 +49,7 @@ def get_heat_profiles_deflex(year, deflex_geo, time_index=None, 'heat_profiles_{year}_{map}'.format(year=year, map=deflex_geo.name)) demand_region = demand_heat.get_heat_profiles_by_region( - year, deflex_geo, to_csv=fn, weather_year=weather_year).groupby( + deflex_geo, year, to_csv=fn, weather_year=weather_year).groupby( level=[0, 1], axis=1).sum() # Decentralised demand is combined to a nation-wide demand if not part diff --git a/deflex/powerplants.py b/deflex/powerplants.py index ff3e7864..3dc6458c 100644 --- a/deflex/powerplants.py +++ b/deflex/powerplants.py @@ -16,13 +16,12 @@ from reegis import geometries as reegis_geometries from reegis import config as cfg from reegis import powerplants -from deflex import geometries # Todo: Revise and test. -def pp_reegis2deflex(filename_in=None, filename_out=None): +def pp_reegis2deflex(regions, name, filename_in=None, filename_out=None): """ Add federal states and deflex regions to powerplant table from reegis. As the process takes a while the result is stored for further usage. @@ -39,10 +38,8 @@ def pp_reegis2deflex(filename_in=None, filename_out=None): map=cfg.get('init', 'map')) # Add deflex regions to powerplants - deflex_regions = geometries.deflex_regions() - name = '{0}_region'.format(cfg.get('init', 'map')) pp = powerplants.add_regions_to_powerplants( - deflex_regions, name, dump=False, filename=filename_in) + regions, name, dump=False, filename=filename_in) # Add federal states to powerplants federal_states = reegis_geometries.get_federal_states_polygon() @@ -115,12 +112,14 @@ def process_pp_table(pp): return pp -def get_deflex_pp_by_year(year, overwrite_capacity=False): +def get_deflex_pp_by_year(regions, year, name, overwrite_capacity=False): """ Parameters ---------- + regions : GeoDataFrame year : int + name : str overwrite_capacity : bool By default (False) a new column "capacity_" is created. If set to True the old capacity column will be overwritten. @@ -136,7 +135,7 @@ def get_deflex_pp_by_year(year, overwrite_capacity=False): if not os.path.isfile(filename): msg = "File '{0}' does not exist. Will create it from reegis file." logging.debug(msg.format(filename)) - filename = pp_reegis2deflex() + filename = pp_reegis2deflex(regions, name) pp = pd.DataFrame(pd.read_hdf(filename, 'pp', mode='r')) # Remove unwanted data sets diff --git a/deflex/scenario_tools.py b/deflex/scenario_tools.py index 26359ce1..1e35a1f3 100644 --- a/deflex/scenario_tools.py +++ b/deflex/scenario_tools.py @@ -341,7 +341,7 @@ def add_decentralised_heating_systems(table_collection, nodes, extra_regions): logging.debug("Add decentralised_heating_systems to nodes dictionary.") cs = table_collection['commodity_source']['DE'] dts = table_collection['demand_series'] - dh = table_collection['decentralised_heating'] + dh = table_collection['scenario_decentralised_heat'] demand_regions = list({'DE_demand'}.union(set(extra_regions))) for d_region in demand_regions: diff --git a/deflex/transmission.py b/deflex/transmission.py index a4948d33..8df25623 100644 --- a/deflex/transmission.py +++ b/deflex/transmission.py @@ -71,6 +71,21 @@ def get_electrical_transmission_default(): def get_electrical_transmission_renpass(): + """ + Calculate the capacity of the transmission lines for the de21 map from + the ranpass data using the renpass methods. + + Cite: + Mehtod: renpass F.Wiese (page 49) + Database: ... + + Returns + ------- + + Examples + -------- + + """ f_security = cfg.get('transmission', 'security_factor') current_max = cfg.get('transmission', 'current_max') diff --git a/tests/test_power_plants.py b/tests/test_power_plants.py index 501d9358..25dbe93e 100644 --- a/tests/test_power_plants.py +++ b/tests/test_power_plants.py @@ -2,8 +2,7 @@ import requests from nose.tools import eq_, assert_raises_regexp from unittest.mock import MagicMock -from deflex import config as cfg -from deflex import powerplants +from deflex import config as cfg, powerplants, geometries def test_01_download_reegis_power_plants(): @@ -19,10 +18,11 @@ def test_01_download_reegis_power_plants(): def test_02_create_deflex_powerplants(): - cfg.tmp_set('init', 'map', 'de21') + de = geometries.deflex_regions('de21') fn_in = os.path.join(cfg.get('paths', 'powerplants'), 'reegis_pp_test.h5') fn_out = os.path.join(cfg.get('paths', 'powerplants'), 'deflex_pp_test.h5') - powerplants.pp_reegis2deflex(filename_in=fn_in, filename_out=fn_out) + powerplants.pp_reegis2deflex(de, 'de21', filename_in=fn_in, + filename_out=fn_out) def test_03_download_deflex_full_pp(): @@ -38,14 +38,17 @@ def test_03_download_deflex_full_pp(): def test_04_deflex_power_plants_by_year(): - pp = powerplants.get_deflex_pp_by_year(2014, overwrite_capacity=True) + de = geometries.deflex_regions('de21') + pp = powerplants.get_deflex_pp_by_year(de, 2014, 'de21', + overwrite_capacity=True) eq_(int(pp['capacity'].sum()), 181489) def test_05_not_existing_file(): cfg.tmp_set('paths', 'powerplants', '/home/pet/') + de = geometries.deflex_regions('de22') powerplants.pp_reegis2deflex = MagicMock(return_value='/home/pet/pp.h5') with assert_raises_regexp(Exception, "File /home/pet/pp.h5 does not exist"): - powerplants.get_deflex_pp_by_year(2012) + powerplants.get_deflex_pp_by_year(de, 2012, 'de22') From cdc261cdcbbf01092fc48b9c4468cb50484a9ca2 Mon Sep 17 00:00:00 2001 From: uvchik Date: Thu, 7 Nov 2019 10:04:58 +0100 Subject: [PATCH 02/16] Revise transmission module and add docstrings and tests --- deflex/transmission.py | 141 ++++++++++++++++++++++++++++++++--------- 1 file changed, 111 insertions(+), 30 deletions(-) diff --git a/deflex/transmission.py b/deflex/transmission.py index 8df25623..b1a9dfe7 100644 --- a/deflex/transmission.py +++ b/deflex/transmission.py @@ -24,6 +24,7 @@ def get_grid_capacity(grid, plus, minus): + """Read the grid capacity from a given region pair from the renpass db.""" tmp_grid = grid.query("plus_region_id == {:0d} & ".format(plus) + "minus_region_id == {:1d} & ".format(minus) + "scenario_name == 'status_quo_2012_distance'") @@ -37,54 +38,135 @@ def get_grid_capacity(grid, plus, minus): return capacity, distance -def get_electrical_transmission_deflex(duplicate=False): - renpas_maps = ['de21', 'de22'] - if cfg.get('init', 'map') in renpas_maps: - df = get_electrical_transmission_renpass() - else: - df = get_electrical_transmission_default() +def add_reverse_direction(df): + """ + Duplicate all entries of a DataFrame with a reverse index. The index must + contain a dash between two sub-strings. + """ + values = df.copy() - if duplicate: - values = df.copy() + def id_inverter(name): + """Swap the sub-parts of a string left and right of a dash.""" + return '-'.join([name.split('-')[1], name.split('-')[0]]) - def id_inverter(name): - return '-'.join([name.split('-')[1], name.split('-')[0]]) + df.index = df.index.map(id_inverter) - df.index = df.index.map(id_inverter) + return pd.DataFrame(pd.concat([values, df])) - df = pd.DataFrame(pd.concat([values, df])) - return df +def get_electrical_transmission_default(rmap=None, power_lines=None, + both_directions=False): + """ + Creates a default set of transmission capacities, distance and efficiency. + The map of the lines must exist in the geometries directory. The default + values are infinity for the capacity, nan for the distance and 1 for the + efficiency. + + Parameters + ---------- + rmap : str + The name of the transmission line map, that is part of deflex. + power_lines : iterable[str] + A list of names of transmission lines. All name must contain a dash + between the id of the regions (FromRegion-ToRegion). + both_directions : bool + If True any line will be replicated in the reverse direction. + + Returns + ------- + pd.DataFrame : Transmission capacity, distance and efficiency between + regions. -def get_electrical_transmission_default(): - try: - pwr_lines = pd.DataFrame(geometries.deflex_power_lines()) - except FileNotFoundError: - pwr_lines = pd.DataFrame() + Examples + -------- + >>> df = get_electrical_transmission_default('de21') + >>> df.loc['DE10-DE12', 'capacity'] + inf + >>> df.loc['DE10-DE12', 'distance'] + nan + >>> df.loc['DE10-DE12', 'efficiency'] + 1.0 + >>> len(df) + 39 + >>> len(get_electrical_transmission_default('de22')) + 40 + >>> len(get_electrical_transmission_default('de17')) + 31 + >>> len(get_electrical_transmission_default('de02')) + 1 + >>> my_lines = ['reg1-reg2', 'reg2-reg3'] + >>> df = get_electrical_transmission_default(power_lines=my_lines) + >>> df.loc['reg1-reg2', 'capacity'] + inf + >>> df = get_electrical_transmission_default(power_lines=my_lines, + ... both_directions=True) + >>> df.loc['reg2-reg1', 'capacity'] + inf + + """ + if power_lines is None: + power_lines = pd.DataFrame(geometries.deflex_power_lines(rmap)).index df = pd.DataFrame() - for l in pwr_lines.index: + for l in power_lines: df.loc[l, 'capacity'] = float('inf') df.loc[l, 'distance'] = float('nan') df.loc[l, 'efficiency'] = 1 + + if both_directions is True: + df = add_reverse_direction(df) return df -def get_electrical_transmission_renpass(): +def get_electrical_transmission_renpass(both_directions=False): """ - Calculate the capacity of the transmission lines for the de21 map from - the ranpass data using the renpass methods. + Prepare the transmission capacity and distance between de21 regions from + the renpass database. The original table of the reegis database is + transferred to a csv file, which is part of the reegis package. As renpass + is deprecated it will not change in the future. The index uses the format + 'region1-region2'. The distance is taken from centroid to centroid. Every + region pair exists only once. + + The capacity calculation is taken from the description of the renpass + package [1]_. The data is taken from the renpass database [2]_. - Cite: - Mehtod: renpass F.Wiese (page 49) - Database: ... + This function is only valid for the original renpass region set. + + Parameters + ---------- + both_directions : bool + If True any line will be replicated in the reverse direction. Returns ------- + pd.DataFrame : Transmission capacity and distance between regions. + + References + ---------- + .. [1] Wiese, Frauke (2015). „Renewable Energy Pathways Simulation System + – Open Source as an approach to meet challenges in energy modeling“. + Diss. University of Flensburg. URL : + https://www.reiner-lemoine-stiftung.de/pdf/dissertationen + /Dissertation_Frauke_Wiese.pdf. + .. [2] Wiese, F.: Renpass - Renewable Energy Pathways Simulation System, + https://github.com/fraukewiese/renpass Examples -------- - + >>> df = get_electrical_transmission_renpass() + >>> int(df.loc['DE11-DE17', 'capacity']) + 2506 + >>> int(df.loc['DE18-DE17', 'distance']) + 119 + >>> df.loc['DE08-DE06'] + capacity 7519.040402 + distance 257.000000 + Name: DE08-DE06, dtype: float64 + >>> df = get_electrical_transmission_renpass(both_directions=True) + >>> int(df.loc['DE11-DE17', 'capacity']) + 2506 + >>> int(df.loc['DE17-DE11', 'capacity']) + 2506 """ f_security = cfg.get('transmission', 'security_factor') current_max = cfg.get('transmission', 'current_max') @@ -126,12 +208,11 @@ def get_electrical_transmission_renpass(): # plot_grid(pwr_lines) df = pwr_lines[['capacity', 'distance']] - return df + if both_directions is True: + df = add_reverse_direction(df) -def get_grid(): - return pd.read_csv(os.path.join('data', 'grid', 'de21_transmission.csv'), - index_col='Unnamed: 0') + return df if __name__ == "__main__": From 9449296c7e48670900b6abcae618d604b2c4cb53 Mon Sep 17 00:00:00 2001 From: uvchik Date: Thu, 7 Nov 2019 11:13:35 +0100 Subject: [PATCH 03/16] Fix key --- deflex/scenario_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deflex/scenario_tools.py b/deflex/scenario_tools.py index 1e35a1f3..c8d3f7ef 100644 --- a/deflex/scenario_tools.py +++ b/deflex/scenario_tools.py @@ -341,7 +341,7 @@ def add_decentralised_heating_systems(table_collection, nodes, extra_regions): logging.debug("Add decentralised_heating_systems to nodes dictionary.") cs = table_collection['commodity_source']['DE'] dts = table_collection['demand_series'] - dh = table_collection['scenario_decentralised_heat'] + dh = table_collection['decentralised_heat'] demand_regions = list({'DE_demand'}.union(set(extra_regions))) for d_region in demand_regions: From 61d5fa605a07439c2a4e10bb3c00e492652b77b5 Mon Sep 17 00:00:00 2001 From: uvchik Date: Thu, 7 Nov 2019 11:13:43 +0100 Subject: [PATCH 04/16] Fix layout --- deflex/transmission.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/deflex/transmission.py b/deflex/transmission.py index b1a9dfe7..cb3d4fe9 100644 --- a/deflex/transmission.py +++ b/deflex/transmission.py @@ -143,11 +143,11 @@ def get_electrical_transmission_renpass(both_directions=False): References ---------- - .. [1] Wiese, Frauke (2015). „Renewable Energy Pathways Simulation System - – Open Source as an approach to meet challenges in energy modeling“. - Diss. University of Flensburg. URL : - https://www.reiner-lemoine-stiftung.de/pdf/dissertationen - /Dissertation_Frauke_Wiese.pdf. + .. [1] Wiese, Frauke (2015). „Renewable Energy Pathways Simulation + System – Open Source as an approach to meet challenges in energy + modeling“. Diss. University of Flensburg. URL : + https://www.reiner-lemoine-stiftung.de/pdf/dissertationen + /Dissertation_Frauke_Wiese.pdf. .. [2] Wiese, F.: Renpass - Renewable Energy Pathways Simulation System, https://github.com/fraukewiese/renpass From d2282a62e7f73febdd67ee8f5ace15d184ee0e6c Mon Sep 17 00:00:00 2001 From: uvchik Date: Thu, 7 Nov 2019 11:15:41 +0100 Subject: [PATCH 05/16] Fix URL --- deflex/transmission.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deflex/transmission.py b/deflex/transmission.py index cb3d4fe9..03378e8b 100644 --- a/deflex/transmission.py +++ b/deflex/transmission.py @@ -146,8 +146,7 @@ def get_electrical_transmission_renpass(both_directions=False): .. [1] Wiese, Frauke (2015). „Renewable Energy Pathways Simulation System – Open Source as an approach to meet challenges in energy modeling“. Diss. University of Flensburg. URL : - https://www.reiner-lemoine-stiftung.de/pdf/dissertationen - /Dissertation_Frauke_Wiese.pdf. + https://www.reiner-lemoine-stiftung.de/pdf/dissertationen/Dissertation_Frauke_Wiese.pdf. .. [2] Wiese, F.: Renpass - Renewable Energy Pathways Simulation System, https://github.com/fraukewiese/renpass From 5d8e3b2ff694042a6a945279fbbc5ae994e84530 Mon Sep 17 00:00:00 2001 From: uvchik Date: Thu, 7 Nov 2019 11:18:22 +0100 Subject: [PATCH 06/16] Fix key in test data --- tests/data/deflex_2013_de02_test.xls | Bin 2416128 -> 2416128 bytes ...sed_heating.csv => decentralised_heat.csv} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/data/deflex_2014_de21_test_csv/{decentralised_heating.csv => decentralised_heat.csv} (100%) diff --git a/tests/data/deflex_2013_de02_test.xls b/tests/data/deflex_2013_de02_test.xls index 3e3120562b361179d2c90113edf238cbff0a56b6..b889be03e6b5465b09f47b2c4d47cbdde5dfba73 100644 GIT binary patch delta 435 zcmWmA&r1SP5C`!0_Svef`D@)Dnf)-|EhB^oB8Vp?2u3_ zVz;26uEieSp*r;kL_vg+5FI)M{Q(_>z7BjCm|R40};Alb3A+RSzka3RY$-TdWe5JbiX+3MpBc-Qs4!+R0i~5+T?2 ziDJYOKiy`4>=p8vQZAp}C}yo(cg`TI&ZA;7dhC4o{ghVhx_anr2RWk^r>>@@rt!>m z=T|e(Ylfyx&50V{KTId*w0$FPccm@waD+~+x(ZC@-mUa1{-MMSy9_bFWwfA8GQ F{sE^md#?Zh delta 427 zcmX}nze~eV5C`!0lA5-r#@IAX{Qc|UprVK%x`~q<4vs2<6pT`^Eu=*R5xVq5u?l9W zqc{jU2>}D@;J={wgNmX!`X7iOUKbyHc-;5kjw{}Hyhtuy6|VSi%1B6JQBPOxXgfQJ zF?yyqq8c;l>}rXbtVORy4Q$!Tq_WFPdB;sV`4ZFF>6=B32+J38Zfdob8SMU}mZ_}Z zrn32!oSSjn5=*diudG^aeP6X^BU{g{r3&da=YN*P7W`Y)mid8S)ilo0+cma)M;9_j zf7bxXTa1@0nQ{A=YRUTfI+ggeoMa0K96>^m5kd%c2ns?NA%YM^s7Fu{Vh9Zg8bTw2 zj?jbwmF diff --git a/tests/data/deflex_2014_de21_test_csv/decentralised_heating.csv b/tests/data/deflex_2014_de21_test_csv/decentralised_heat.csv similarity index 100% rename from tests/data/deflex_2014_de21_test_csv/decentralised_heating.csv rename to tests/data/deflex_2014_de21_test_csv/decentralised_heat.csv From e86dde2774e8e25e605be0df16cc039ca5fc2e07 Mon Sep 17 00:00:00 2001 From: uvchik Date: Thu, 7 Nov 2019 11:25:46 +0100 Subject: [PATCH 07/16] Remove unreachable code --- deflex/transmission.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/deflex/transmission.py b/deflex/transmission.py index 03378e8b..e4f3ad33 100644 --- a/deflex/transmission.py +++ b/deflex/transmission.py @@ -197,13 +197,6 @@ def get_electrical_transmission_renpass(both_directions=False): elif cap2 == 0: pwr_lines.loc[l, 'capacity'] = cap1 pwr_lines.loc[l, 'distance'] = dist1 - elif cap1 == cap2: - pwr_lines.loc[l, 'capacity'] = cap1 - pwr_lines.loc[l, 'distance'] = dist1 - else: - msg = ("The same line ({0}, {1}) has different capacities: {2} " - "!= {3}") - raise ValueError(msg.format(a, b, cap1, cap2)) # plot_grid(pwr_lines) df = pwr_lines[['capacity', 'distance']] From b4cf8a81cc0727887d63bb79e0075d006251c5ad Mon Sep 17 00:00:00 2001 From: uvchik Date: Thu, 7 Nov 2019 11:33:56 +0100 Subject: [PATCH 08/16] Fix return layout and complete reference --- deflex/transmission.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/deflex/transmission.py b/deflex/transmission.py index e4f3ad33..7eee15fb 100644 --- a/deflex/transmission.py +++ b/deflex/transmission.py @@ -74,8 +74,8 @@ def get_electrical_transmission_default(rmap=None, power_lines=None, Returns ------- - pd.DataFrame : Transmission capacity, distance and efficiency between - regions. + pd.DataFrame + Transmission capacity, distance and efficiency between regions Examples -------- @@ -124,8 +124,9 @@ def get_electrical_transmission_renpass(both_directions=False): the renpass database. The original table of the reegis database is transferred to a csv file, which is part of the reegis package. As renpass is deprecated it will not change in the future. The index uses the format - 'region1-region2'. The distance is taken from centroid to centroid. Every - region pair exists only once. + 'region1-region2'. The distance is taken from centroid to centroid. By + default every region pair exists only once. It is possible to get an entry + in both directions if the parameter `both_directions` is set True. The capacity calculation is taken from the description of the renpass package [1]_. The data is taken from the renpass database [2]_. @@ -139,7 +140,8 @@ def get_electrical_transmission_renpass(both_directions=False): Returns ------- - pd.DataFrame : Transmission capacity and distance between regions. + pd.DataFrame + Transmission capacity and distance between regions References ---------- @@ -147,6 +149,7 @@ def get_electrical_transmission_renpass(both_directions=False): System – Open Source as an approach to meet challenges in energy modeling“. Diss. University of Flensburg. URL : https://www.reiner-lemoine-stiftung.de/pdf/dissertationen/Dissertation_Frauke_Wiese.pdf. + (page 49) .. [2] Wiese, F.: Renpass - Renewable Energy Pathways Simulation System, https://github.com/fraukewiese/renpass @@ -174,7 +177,6 @@ def get_electrical_transmission_renpass(both_directions=False): cfg.get('paths', 'data_deflex'), cfg.get('transmission', 'transmission_renpass'))) - # renpass F.Wiese (page 49) grid['capacity_calc'] = (grid.circuits * current_max * grid.voltage * f_security * math.sqrt(3) / 1000) From 084b1e11a1b644431a2df982b12085316fadd811 Mon Sep 17 00:00:00 2001 From: uvchik Date: Thu, 7 Nov 2019 21:45:23 +0100 Subject: [PATCH 09/16] Use most actual reegis version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 32d2f3be..2411df41 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ def read(fname): install_requires=[ 'oemof >= 0.3.0', 'pandas >= 0.17.0', - 'reegis == v0.1.0-rc.8', + 'reegis == v0.1.0-rc.9', 'demandlib', 'workalendar', 'networkx', From 04db3d9298223ea2f0e5a0bec965dd87a3940149 Mon Sep 17 00:00:00 2001 From: uvchik Date: Thu, 7 Nov 2019 21:45:46 +0100 Subject: [PATCH 10/16] Adapt config file --- deflex/de21_default.ini_backup | 1 - deflex/deflex.ini | 1 - 2 files changed, 2 deletions(-) diff --git a/deflex/de21_default.ini_backup b/deflex/de21_default.ini_backup index ff5ebe67..8622e86a 100644 --- a/deflex/de21_default.ini_backup +++ b/deflex/de21_default.ini_backup @@ -11,7 +11,6 @@ scenario = local_root, scenarios deflex_feedin = local_root, data, feedin, {map}_region, {year} [geometry] -germany_polygon = germany_polygon.csv deflex_polygon = region_{type}_{map}_{suffix}.csv region_polygon_simple = region_polygons_{map}_simple.csv region_label = region_labels_{map}.csv diff --git a/deflex/deflex.ini b/deflex/deflex.ini index 577e69c4..873587f0 100644 --- a/deflex/deflex.ini +++ b/deflex/deflex.ini @@ -10,7 +10,6 @@ scenario = local_root, scenarios deflex_feedin = local_root, data, feedin, {map}_region, {year} [geometry] -germany_polygon = germany_polygon.csv deflex_polygon = region_{type}_{map}{suffix} region_polygon_simple = region_polygons_{map}_simple.csv region_label = region_labels_{map}.csv From a1829d5acb671090ef716e0ea0172d8d3bdb4ad8 Mon Sep 17 00:00:00 2001 From: uvchik Date: Thu, 7 Nov 2019 21:46:17 +0100 Subject: [PATCH 11/16] Revise scenario transmission --- deflex/basic_scenario.py | 114 ++++++++++++++++++++------------------- deflex/geometries.py | 41 ++++++++++++++ 2 files changed, 100 insertions(+), 55 deletions(-) diff --git a/deflex/basic_scenario.py b/deflex/basic_scenario.py index 3ea4cab6..1322ddb4 100644 --- a/deflex/basic_scenario.py +++ b/deflex/basic_scenario.py @@ -34,7 +34,8 @@ from deflex import config as cfg -def create_scenario(regions, year, name, round_values=True, weather_year=None): +def create_scenario(regions, year, name, round_values=True, weather_year=None, + cupper_plate=False): table_collection = {} logging.info('BASIC SCENARIO - STORAGES') @@ -45,7 +46,8 @@ def create_scenario(regions, year, name, round_values=True, weather_year=None): table_collection, regions, year, name, round_values) logging.info('BASIC SCENARIO - TRANSMISSION') - table_collection['transmission'] = scenario_transmission(table_collection) + table_collection['transmission'] = scenario_transmission( + table_collection, regions, name, cupper_plate=cupper_plate) logging.info('BASIC SCENARIO - CHP PLANTS') table_collection = scenario_chp(table_collection, regions, year, name, @@ -65,15 +67,66 @@ def create_scenario(regions, year, name, round_values=True, weather_year=None): return table_collection -def scenario_transmission(table_collection): +def scenario_storages(regions, year, name): + stor = storages.pumped_hydroelectric_storage_by_region( + regions, year, name).transpose() + return pd.concat([stor], axis=1, keys=['phes']).swaplevel(0, 1, 1) + + +def scenario_powerplants(table_collection, regions, year, name, round_values): + """Get power plants for the scenario year + """ + pp = powerplants.get_deflex_pp_by_year(regions, year, name, + overwrite_capacity=True) + return create_powerplants(pp, table_collection, year, name, round_values) + + +def create_powerplants(pp, table_collection, year, + region_column='deflex_region', round_values=None): + """This function works for all power plant tables with an equivalent + structure e.g. power plants by state or other regions.""" + logging.info("Adding power plants to your scenario.") + + replace_names = cfg.get_dict('source_names') + replace_names.update(cfg.get_dict('source_groups')) + + pp['energy_source_level_2'].replace(replace_names, inplace=True) + + pp['model_classes'] = pp['energy_source_level_2'].replace( + cfg.get_dict('model_classes')) + + pp = pp.groupby( + ['model_classes', region_column, 'energy_source_level_2']).sum()[ + ['capacity', 'capacity_in']] + + for model_class in pp.index.get_level_values(level=0).unique(): + pp_class = pp.loc[model_class] + if model_class != 'volatile_source': + pp_class['efficiency'] = (pp_class['capacity'] / + pp_class['capacity_in'] * 100) + del pp_class['capacity_in'] + if round_values is not None: + pp_class = pp_class.round(round_values) + if 'efficiency' in pp_class: + pp_class['efficiency'] = pp_class['efficiency'].div(100) + pp_class = pp_class.transpose() + pp_class.index.name = 'parameter' + table_collection[model_class] = pp_class + table_collection = add_pp_limit(table_collection, year) + return table_collection + + +def scenario_transmission(table_collection, regions, name, cupper_plate=False): vs = table_collection['volatile_source'] # This should be done automatic e.g. if representative point outside the # landmass polygon. - offshore_regions = ( - cfg.get_dict_list('offshore_regions_set')[cfg.get('init', 'map')]) + offshore_regions = geometries.divide_off_and_onshore(regions) - elec_trans = transmission.get_electrical_transmission_deflex() + if name in ['de21', 'de22']: + elec_trans = transmission.get_electrical_transmission_renpass() + else: + elec_trans = transmission.get_electrical_transmission_default() # Set transmission capacity of offshore power lines to installed capacity # Multiply the installed capacity with 1.1 to get a buffer of 10%. @@ -96,12 +149,6 @@ def scenario_transmission(table_collection): return elec_trans -def scenario_storages(regions, year, name): - stor = storages.pumped_hydroelectric_storage_by_region( - regions, year, name).transpose() - return pd.concat([stor], axis=1, keys=['phes']).swaplevel(0, 1, 1) - - def add_pp_limit(table_collection, year): if len(cfg.get_dict('limited_transformer').keys()) > 0: # Multiply with 1000 to get MWh (bmwi: GWh) @@ -323,49 +370,6 @@ def chp_table(heat_b, heat_demand, table_collection, regions=None): return table_collection -def scenario_powerplants(table_collection, regions, year, name, round_values): - """Get power plants for the scenario year - """ - pp = powerplants.get_deflex_pp_by_year(regions, year, name, - overwrite_capacity=True) - return create_powerplants(pp, table_collection, year, name, round_values) - - -def create_powerplants(pp, table_collection, year, - region_column='deflex_region', round_values=None): - """This function works for all power plant tables with an equivalent - structure e.g. power plants by state or other regions.""" - logging.info("Adding power plants to your scenario.") - - replace_names = cfg.get_dict('source_names') - replace_names.update(cfg.get_dict('source_groups')) - - pp['energy_source_level_2'].replace(replace_names, inplace=True) - - pp['model_classes'] = pp['energy_source_level_2'].replace( - cfg.get_dict('model_classes')) - - pp = pp.groupby( - ['model_classes', region_column, 'energy_source_level_2']).sum()[ - ['capacity', 'capacity_in']] - - for model_class in pp.index.get_level_values(level=0).unique(): - pp_class = pp.loc[model_class] - if model_class != 'volatile_source': - pp_class['efficiency'] = (pp_class['capacity'] / - pp_class['capacity_in'] * 100) - del pp_class['capacity_in'] - if round_values is not None: - pp_class = pp_class.round(round_values) - if 'efficiency' in pp_class: - pp_class['efficiency'] = pp_class['efficiency'].div(100) - pp_class = pp_class.transpose() - pp_class.index.name = 'parameter' - table_collection[model_class] = pp_class - table_collection = add_pp_limit(table_collection, year) - return table_collection - - def clean_time_series(table_collection): dts = table_collection['demand_series'] vts = table_collection['volatile_series'] diff --git a/deflex/geometries.py b/deflex/geometries.py index 3e11f91a..c85869bb 100644 --- a/deflex/geometries.py +++ b/deflex/geometries.py @@ -13,6 +13,7 @@ # Python libraries import os +from collections import namedtuple # Internal libraries from deflex import config as cfg @@ -91,3 +92,43 @@ def deflex_power_lines(rmap=None, rtype='lines'): lines.set_index('name', inplace=True) lines.name = rmap return lines + + +def divide_off_and_onshore(regions): + """ + Sort regions into onshore and offshore regions. A namedtuple with two list + of regions ids will be returned. Fetch the `onshore` and `offshore` + attribute of the named tuple to get the list. + + Parameters + ---------- + regions : GeoDataFrame + A region set with the region id in the index. + + Returns + ------- + named tuple + + Examples + -------- + >>> reg = deflex_regions('de02') + >>> divide_off_and_onshore(reg).onshore + ['DE01'] + >>> reg = deflex_regions('de21') + >>> divide_off_and_onshore(reg).offshore + ['DE19', 'DE20', 'DE21'] + """ + region_type = namedtuple('RegionType', 'offshore onshore') + regions.geometry = regions.centroid + # print(regions) + germany_onshore = geo.load( + cfg.get('paths', 'geometry'), + cfg.get('geometry', 'germany_polygon')) + print(cfg.get('paths', 'geometry'), + cfg.get('geometry', 'germany_polygon')) + gdf = geo.spatial_join_with_buffer(regions, germany_onshore, + 'onshore', limit=0) + # print(gdf) + onshore = list(gdf.loc[gdf.onshore == 0].index) + offshore = list(gdf.loc[gdf.onshore == 'unknown'].index) + return region_type(offshore=offshore, onshore=onshore) From 50147697f56b2a151b4e3f17256e1423bb082cc5 Mon Sep 17 00:00:00 2001 From: uvchik Date: Thu, 7 Nov 2019 21:49:14 +0100 Subject: [PATCH 12/16] Remove print statements --- deflex/geometries.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deflex/geometries.py b/deflex/geometries.py index c85869bb..05bb2eed 100644 --- a/deflex/geometries.py +++ b/deflex/geometries.py @@ -120,15 +120,15 @@ def divide_off_and_onshore(regions): """ region_type = namedtuple('RegionType', 'offshore onshore') regions.geometry = regions.centroid - # print(regions) + germany_onshore = geo.load( cfg.get('paths', 'geometry'), cfg.get('geometry', 'germany_polygon')) - print(cfg.get('paths', 'geometry'), - cfg.get('geometry', 'germany_polygon')) + gdf = geo.spatial_join_with_buffer(regions, germany_onshore, 'onshore', limit=0) - # print(gdf) + onshore = list(gdf.loc[gdf.onshore == 0].index) offshore = list(gdf.loc[gdf.onshore == 'unknown'].index) + return region_type(offshore=offshore, onshore=onshore) From e3aa3c52b2ed029a9135f0d29ee3096aa09fbe7c Mon Sep 17 00:00:00 2001 From: uvchik Date: Tue, 12 Nov 2019 20:15:09 +0100 Subject: [PATCH 13/16] Add tests for scenario creation --- deflex/basic_scenario.py | 259 +++++++++++++++++++--- deflex/deflex.ini | 4 +- tests/test_scenario_demand.py | 29 +++ tests/test_scenario_feedin.py | 33 +++ tests/test_scenario_powerplant_and_chp.py | 54 +++++ 5 files changed, 341 insertions(+), 38 deletions(-) create mode 100644 tests/test_scenario_demand.py create mode 100644 tests/test_scenario_feedin.py create mode 100644 tests/test_scenario_powerplant_and_chp.py diff --git a/deflex/basic_scenario.py b/deflex/basic_scenario.py index 1322ddb4..c68e5e19 100644 --- a/deflex/basic_scenario.py +++ b/deflex/basic_scenario.py @@ -35,7 +35,7 @@ def create_scenario(regions, year, name, round_values=True, weather_year=None, - cupper_plate=False): + copperplate=False): table_collection = {} logging.info('BASIC SCENARIO - STORAGES') @@ -47,7 +47,7 @@ def create_scenario(regions, year, name, round_values=True, weather_year=None, logging.info('BASIC SCENARIO - TRANSMISSION') table_collection['transmission'] = scenario_transmission( - table_collection, regions, name, cupper_plate=cupper_plate) + table_collection, regions, name, copperplate=copperplate) logging.info('BASIC SCENARIO - CHP PLANTS') table_collection = scenario_chp(table_collection, regions, year, name, @@ -57,7 +57,7 @@ def create_scenario(regions, year, name, round_values=True, weather_year=None, table_collection['decentralised_heat'] = scenario_decentralised_heat() logging.info('BASIC SCENARIO - SOURCES') - table_collection = scenario_commodity_sources(year, table_collection) + table_collection = scenario_commodity_sources(table_collection, year) table_collection['volatile_series'] = scenario_feedin( regions, year, name, weather_year=weather_year) @@ -68,6 +68,30 @@ def create_scenario(regions, year, name, round_values=True, weather_year=None, def scenario_storages(regions, year, name): + """ + Fetch storage, pump and turbine capacity and their efficiency of + hydroelectric storages for each deflex region. + + Parameters + ---------- + regions + year + name + + Returns + ------- + + Examples + -------- + >>> regions = geometries.deflex_regions(rmap='de17') + >>> deflex_storages = scenario_storages(regions, 2012, 'de17') + >>> list(deflex_storages.columns.get_level_values(0)) + ['DE01', 'DE03', 'DE05', 'DE06', 'DE08', 'DE09', 'DE14', 'DE15', 'DE16'] + >>> int(deflex_storages.loc['turbine', 'DE03']) + 220 + >>> int(deflex_storages.loc['energy', 'DE16']) + 12115 + """ stor = storages.pumped_hydroelectric_storage_by_region( regions, year, name).transpose() return pd.concat([stor], axis=1, keys=['phes']).swaplevel(0, 1, 1) @@ -75,6 +99,17 @@ def scenario_storages(regions, year, name): def scenario_powerplants(table_collection, regions, year, name, round_values): """Get power plants for the scenario year + + Examples + -------- + >>> regions = geometries.deflex_regions(rmap='de21') + >>> pp = scenario_powerplants( + ... dict(), regions, 2014, 'de21', 1) # doctest: +SKIP + >>> float(pp['volatile_source']['DE03', 'wind']) # doctest: +SKIP + 3052.8 + >>> float(pp['transformer'].loc[ + ... 'capacity', ('DE03', 'lignite')]) # doctest: +SKIP + 1135.6 """ pp = powerplants.get_deflex_pp_by_year(regions, year, name, overwrite_capacity=True) @@ -116,40 +151,18 @@ def create_powerplants(pp, table_collection, year, return table_collection -def scenario_transmission(table_collection, regions, name, cupper_plate=False): - vs = table_collection['volatile_source'] - - # This should be done automatic e.g. if representative point outside the - # landmass polygon. - offshore_regions = geometries.divide_off_and_onshore(regions) - - if name in ['de21', 'de22']: - elec_trans = transmission.get_electrical_transmission_renpass() - else: - elec_trans = transmission.get_electrical_transmission_default() - - # Set transmission capacity of offshore power lines to installed capacity - # Multiply the installed capacity with 1.1 to get a buffer of 10%. - for offreg in offshore_regions: - elec_trans.loc[elec_trans.index.str.contains(offreg), 'capacity'] = ( - vs[offreg].sum().sum() * 1.1) +def add_pp_limit(table_collection, year): + """ - elec_trans = ( - pd.concat([elec_trans], axis=1, keys=['electrical']).sort_index(1)) - general_efficiency = cfg.get('transmission', 'general_efficiency') - if general_efficiency is not None: - elec_trans['electrical', 'efficiency'] = general_efficiency - else: - msg = ("The calculation of the efficiency by distance is not yet " - "implemented") - raise NotImplementedError(msg) - if cfg.get('init', 'map') == 'de22': - elec_trans.loc['DE22-DE01', ('electrical', 'efficiency')] = 0.9999 - elec_trans.loc['DE22-DE01', ('electrical', 'capacity')] = 9999999 - return elec_trans + Parameters + ---------- + table_collection + year + Returns + ------- -def add_pp_limit(table_collection, year): + """ if len(cfg.get_dict('limited_transformer').keys()) > 0: # Multiply with 1000 to get MWh (bmwi: GWh) repp = bmwi.bmwi_re_energy_capacity() * 1000 @@ -175,7 +188,91 @@ def add_pp_limit(table_collection, year): return table_collection -def scenario_commodity_sources(year, table_collection): +def scenario_transmission(table_collection, regions, name, copperplate=False): + """Get power plants for the scenario year + + Examples + -------- + >>> regions = geometries.deflex_regions(rmap='de21') # doctest: +SKIP + >>> pp = scenario_powerplants(dict(), regions, 2014, 'de21', 1 + ... ) # doctest: +SKIP + >>> lines = scenario_transmission(pp, regions, 'de21') # doctest: +SKIP + >>> int(lines.loc['DE07-DE05', ('electrical', 'capacity')] + ... ) # doctest: +SKIP + 1978 + >>> int(lines.loc['DE07-DE05', ('electrical', 'distance')] + ... ) # doctest: +SKIP + 199 + >>> float(lines.loc['DE07-DE05', ('electrical', 'efficiency')] + ... ) # doctest: +SKIP + 0.9 + >>> lines = scenario_transmission(pp, regions, 'de21', copperplate=True + ... ) # doctest: +SKIP + >>> float(lines.loc['DE07-DE05', ('electrical', 'capacity')] + ... ) # doctest: +SKIP + inf + >>> float(lines.loc['DE07-DE05', ('electrical', 'distance')] + ... ) # doctest: +SKIP + nan + >>> float(lines.loc['DE07-DE05', ('electrical', 'efficiency')] + ... ) # doctest: +SKIP + 1.0 + """ + vs = table_collection['volatile_source'] + + # This should be done automatic e.g. if representative point outside the + # landmass polygon. + offshore_regions = geometries.divide_off_and_onshore(regions).offshore + + if name in ['de21', 'de22'] and not copperplate: + elec_trans = transmission.get_electrical_transmission_renpass() + general_efficiency = cfg.get('transmission', 'general_efficiency') + if general_efficiency is not None: + elec_trans['efficiency'] = general_efficiency + else: + msg = ("The calculation of the efficiency by distance is not yet " + "implemented") + raise NotImplementedError(msg) + else: + elec_trans = transmission.get_electrical_transmission_default() + + # Set transmission capacity of offshore power lines to installed capacity + # Multiply the installed capacity with 1.1 to get a buffer of 10%. + for offreg in offshore_regions: + elec_trans.loc[elec_trans.index.str.contains(offreg), 'capacity'] = ( + vs[offreg].sum().sum() * 1.1) + + elec_trans = ( + pd.concat([elec_trans], axis=1, keys=['electrical']).sort_index(1)) + if cfg.get('init', 'map') == 'de22': + elec_trans.loc['DE22-DE01', ('electrical', 'efficiency')] = 0.9999 + elec_trans.loc['DE22-DE01', ('electrical', 'capacity')] = 9999999 + return elec_trans + + +def scenario_commodity_sources(table_collection, year): + """ + + Parameters + ---------- + year + table_collection + + Returns + ------- + + Examples + -------- + >>> regions = geometries.deflex_regions(rmap='de21') # doctest: +SKIP + >>> pp = scenario_powerplants(dict(), regions, 2014, 'de21', 1 + ... ) # doctest: +SKIP + >>> src = scenario_commodity_sources(pp, 2014) # doctest: +SKIP + >>> src = src['commodity_source'] # doctest: +SKIP + >>> round(src.loc['costs', ('DE', 'hard coal')], 2) # doctest: +SKIP + 8.93 + >>> round(src.loc['emission', ('DE', 'natural gas')], 2) # doctest: +SKIP + 201.24 + """ commodity_src = create_commodity_sources(year) commodity_src = commodity_src.swaplevel().unstack() @@ -209,6 +306,17 @@ def scenario_commodity_sources(year, table_collection): def create_commodity_sources(year, use_znes_2014=True): + """ + + Parameters + ---------- + year + use_znes_2014 + + Returns + ------- + + """ cs = commodity_sources.get_commodity_sources() rename_cols = {key.lower(): value for key, value in cfg.get_dict('source_names').items()} @@ -225,6 +333,28 @@ def create_commodity_sources(year, use_znes_2014=True): def scenario_demand(regions, year, name, weather_year=None): + """ + + Parameters + ---------- + regions + year + name + weather_year + + Returns + ------- + + Examples + -------- + >>> regions = geometries.deflex_regions(rmap='de21') # doctest: +SKIP + >>> my_demand = scenario_demand(regions, 2014, 'de21') # doctest: +SKIP + >>> int(my_demand['DE01', 'district heating'].sum()) # doctest: +SKIP + 18639262 + >>> int(my_demand['DE05', 'electrical_load'].sum()) # doctest: +SKIP + 10069 + + """ demand_series = scenario_elec_demand(pd.DataFrame(), regions, year, name, weather_year=weather_year) demand_series = scenario_heat_demand(demand_series, regions, year, @@ -255,6 +385,35 @@ def scenario_elec_demand(table, regions, year, name, weather_year=None): def scenario_feedin(regions, year, name, weather_year=None): + """ + + Parameters + ---------- + regions + year + name + weather_year + + Returns + ------- + + Examples + -------- + >>> regions = geometries.deflex_regions(rmap='de21') # doctest: +SKIP + >>> f = scenario_feedin(regions, 2014, 'de21') # doctest: +SKIP + >>> f['DE01'].sum() # doctest: +SKIP + geothermal 4380.000000 + hydro 1346.632529 + solar 913.652083 + wind 2152.983589 + dtype: float64 + >>> f['DE16'].sum() + geothermal 4380.000000 + hydro 1346.632529 + solar 903.527200 + wind 1753.673492 + dtype: float64 + """ wy = weather_year try: feedin = coastdat.scenario_feedin(year, name, weather_year=wy) @@ -272,7 +431,35 @@ def scenario_decentralised_heat(): def scenario_chp(table_collection, regions, year, name, weather_year=None): + """ + Parameters + ---------- + table_collection + regions + year + name + weather_year + + Returns + ------- + + Examples + -------- + >>> regions = geometries.deflex_regions(rmap='de21') # doctest: +SKIP + >>> pp = scenario_powerplants(dict(), regions, 2014, 'de21', 1 + ... ) # doctest: +SKIP + >>> int(pp['transformer'].loc['capacity', ('DE01', 'hard coal')] + ... ) # doctest: +SKIP + 1291 + >>> transf = scenario_chp(pp, regions, 2014, 'de21') # doctest: +SKIP + >>> transf = transf['transformer'] # doctest: +SKIP + >>> int(transf.loc['capacity', ('DE01', 'hard coal')]) # doctest: +SKIP + 485 + >>> int(transf.loc['capacity_elec_chp', ('DE01', 'hard coal')] + ... ) # doctest: +SKIP + 806 + """ # values from heat balance cb = energy_balance.get_transformation_balance_by_region( @@ -281,7 +468,7 @@ def scenario_chp(table_collection, regions, year, name, weather_year=None): heat_b = reegis_powerplants.calculate_chp_share_and_efficiency(cb) heat_demand = demand.get_heat_profiles_deflex( - year, regions, weather_year=weather_year) + regions, year, weather_year=weather_year) return chp_table(heat_b, heat_demand, table_collection) diff --git a/deflex/deflex.ini b/deflex/deflex.ini index 873587f0..8220f5fb 100644 --- a/deflex/deflex.ini +++ b/deflex/deflex.ini @@ -54,8 +54,8 @@ commodity_sources_file = commodity_sources.csv renewable_source = bioenergy [feedin] -feedin_deflex_pattern = {year}_feedin_{map}_region_normalised_{type}.csv -feedin_deflex_pattern_var = {year}_feedin_{map}_region_normalised_{type}_var_{weather_year}.csv +feedin_deflex_pattern = {year}_feedin_{map}_normalised_{type}.csv +feedin_deflex_pattern_var = {year}_feedin_{map}_normalised_{type}_var_{weather_year}.csv geothermal_full_load_hours = 4380 [time_series] diff --git a/tests/test_scenario_demand.py b/tests/test_scenario_demand.py new file mode 100644 index 00000000..b6fb3d83 --- /dev/null +++ b/tests/test_scenario_demand.py @@ -0,0 +1,29 @@ +import os +import requests +from nose.tools import eq_, assert_raises_regexp, with_setup +from unittest.mock import MagicMock +from deflex import config as cfg, basic_scenario, geometries +from reegis.tools import download_file + + +def setup_func(): + """Download pp-file from osf.""" + + url = 'https://osf.io/m435r/download' + path = cfg.get('paths', 'demand') + file = 'heat_profile_state_2014_weather_2014.csv' + filename = os.path.join(path, file) + download_file(filename, url) + + url = 'https://osf.io/m435r/download' + file = 'heat_profile_state_2014_weather_2014.csv' + filename = os.path.join(path, file) + download_file(filename, url) + + +@with_setup(setup_func) +def scenario_demand_test(): + regions = geometries.deflex_regions(rmap='de21') + d = basic_scenario.scenario_demand(regions, 2014, 'de21') + eq_(int(d['DE01', 'district heating'].sum()), 18639262) + eq_(int(d['DE05', 'electrical_load'].sum()), 10069) diff --git a/tests/test_scenario_feedin.py b/tests/test_scenario_feedin.py new file mode 100644 index 00000000..85e5b5a2 --- /dev/null +++ b/tests/test_scenario_feedin.py @@ -0,0 +1,33 @@ +import os +import requests +from nose.tools import eq_, assert_raises_regexp, with_setup +from unittest.mock import MagicMock +from deflex import config as cfg, basic_scenario, geometries +from reegis.tools import download_file + + +def setup_func(): + """Download pp-file from osf.""" + + downloads = [ + ('n7ahr', 'geothermal'), + ('5n7t3', 'hydro'), + ('2qwv7', 'solar'), + ('9dvpf', 'wind')] + + for d in downloads: + url = 'https://osf.io/{0}/download'.format(d[0]) + path = os.path.join( + cfg.get('paths', 'feedin'), 'de21', '2014') + file = '2014_feedin_de21_normalised_{0}.csv'.format(d[1]) + filename = os.path.join(path, file) + os.makedirs(path, exist_ok=True) + download_file(filename, url) + + +@with_setup(setup_func) +def scenario_feedin_test(): + regions = geometries.deflex_regions(rmap='de21') + f = basic_scenario.scenario_feedin(regions, 2014, 'de21') + # eq_(int(d['DE01', 'district heating'].sum()), 18639262) + # eq_(int(d['DE05', 'electrical_load'].sum()), 10069) diff --git a/tests/test_scenario_powerplant_and_chp.py b/tests/test_scenario_powerplant_and_chp.py new file mode 100644 index 00000000..d8b982cc --- /dev/null +++ b/tests/test_scenario_powerplant_and_chp.py @@ -0,0 +1,54 @@ +import os +import requests +from nose.tools import eq_ +from deflex import config as cfg, basic_scenario, geometries + + +class TestScenarioPowerplantsAndCHP: + @classmethod + def setUpClass(cls): + """Download pp-file from osf.""" + url = 'https://osf.io/qtc56/download' + path = cfg.get('paths', 'powerplants') + file = 'de21_pp.h5' + filename = os.path.join(path, file) + + if not os.path.isfile(filename): + req = requests.get(url) + with open(filename, 'wb') as fout: + fout.write(req.content) + cls.regions = geometries.deflex_regions(rmap='de21') + cls.pp = basic_scenario.scenario_powerplants( + dict(), cls.regions, 2014, 'de21', 1) + + def scenario_pp_test(self): + eq_(float(self.pp['volatile_source']['DE03', 'wind']), 3052.8) + eq_(float(self.pp['transformer'].loc['capacity', ('DE03', 'lignite')]), + 1135.6) + + def test_scenario_transmission(self): + lines = basic_scenario.scenario_transmission( + self.pp, self.regions, 'de21') + eq_(int(lines.loc['DE07-DE05', ('electrical', 'capacity')]), 1978) + eq_(int(lines.loc['DE07-DE05', ('electrical', 'distance')]), 199) + eq_(float(lines.loc['DE07-DE05', ('electrical', 'efficiency')]), 0.9) + lines = basic_scenario.scenario_transmission( + self.pp, self.regions, 'de21', copperplate=True) + eq_(float(lines.loc['DE07-DE05', ('electrical', 'capacity')]), + float('inf')) + eq_(str(lines.loc['DE07-DE05', ('electrical', 'distance')]), 'nan') + eq_(float(lines.loc['DE07-DE05', ('electrical', 'efficiency')]), 1.0) + + def test_scenario_commodity_sources(self): + src = basic_scenario.scenario_commodity_sources( + self.pp, 2014)['commodity_source'] + eq_(round(src.loc['costs', ('DE', 'hard coal')], 2), 8.93) + eq_(round(src.loc['emission', ('DE', 'natural gas')], 2), 201.24) + + def test_chp(self): + eq_(int(self.pp['transformer'].loc['capacity', ('DE01', 'hard coal')]), + 1291) + transf = basic_scenario.scenario_chp( + self.pp, self.regions, 2014, 'de21')['transformer'] + eq_(int(transf.loc['capacity', ('DE01', 'hard coal')]), 485) + eq_(int(transf.loc['capacity_elec_chp', ('DE01', 'hard coal')]), 806) From 64889fc9137ccdf5a3206cec79c350879a3663ca Mon Sep 17 00:00:00 2001 From: uvchik Date: Tue, 12 Nov 2019 20:30:56 +0100 Subject: [PATCH 14/16] Skip doctest --- deflex/basic_scenario.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deflex/basic_scenario.py b/deflex/basic_scenario.py index c68e5e19..9551c355 100644 --- a/deflex/basic_scenario.py +++ b/deflex/basic_scenario.py @@ -407,7 +407,7 @@ def scenario_feedin(regions, year, name, weather_year=None): solar 913.652083 wind 2152.983589 dtype: float64 - >>> f['DE16'].sum() + >>> f['DE16'].sum() # doctest: +SKIP geothermal 4380.000000 hydro 1346.632529 solar 903.527200 From 1f3ed69e5423e8b3960c754e998e963fd9383d78 Mon Sep 17 00:00:00 2001 From: uvchik Date: Tue, 12 Nov 2019 21:23:32 +0100 Subject: [PATCH 15/16] Fix tests --- deflex/powerplants.py | 2 +- setup.py | 2 +- tests/data/windzone_de21.csv | 40 +++++++++++++++++++++++ tests/test_power_plants.py | 30 +++-------------- tests/test_scenario_a_init.py | 16 +++++++++ tests/test_scenario_demand.py | 8 ++--- tests/test_scenario_feedin.py | 15 ++++++--- tests/test_scenario_powerplant_and_chp.py | 19 ++++------- 8 files changed, 82 insertions(+), 50 deletions(-) create mode 100644 tests/data/windzone_de21.csv create mode 100644 tests/test_scenario_a_init.py diff --git a/deflex/powerplants.py b/deflex/powerplants.py index 3dc6458c..ae261d35 100644 --- a/deflex/powerplants.py +++ b/deflex/powerplants.py @@ -130,7 +130,7 @@ def get_deflex_pp_by_year(regions, year, name, overwrite_capacity=False): """ filename = os.path.join(cfg.get('paths', 'powerplants'), cfg.get('powerplants', 'deflex_pp')).format( - map=cfg.get('init', 'map')) + map=name) logging.info("Get deflex power plants for {0}.".format(year)) if not os.path.isfile(filename): msg = "File '{0}' does not exist. Will create it from reegis file." diff --git a/setup.py b/setup.py index 2411df41..5f58a372 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ def read(fname): packages=find_packages(), package_dir={'deflex': 'deflex'}, extras_require={ - 'dev': ['nose', 'sphinx', 'sphinx_rtd_theme']}, + 'dev': ['nose', 'sphinx', 'sphinx_rtd_theme', 'requests']}, install_requires=[ 'oemof >= 0.3.0', 'pandas >= 0.17.0', diff --git a/tests/data/windzone_de21.csv b/tests/data/windzone_de21.csv new file mode 100644 index 00000000..8df73d42 --- /dev/null +++ b/tests/data/windzone_de21.csv @@ -0,0 +1,40 @@ +DE01,2.0,0.9245130810720856 +DE01,3.0,0.07339057457608637 +DE01,4.0,0.0020963443518280315 +DE02,2.0,1.0 +DE03,1.0,0.03605995438744842 +DE03,2.0,0.9639400456125516 +DE04,2.0,1.0 +DE05,1.0,0.08027441649069571 +DE05,2.0,0.9197255835093042 +DE06,1.0,0.3888873593465007 +DE06,2.0,0.6111126406534994 +DE07,1.0,0.8744249793815474 +DE07,2.0,0.12557502061845272 +DE08,1.0,0.5322536596437746 +DE08,2.0,0.4677463403562254 +DE09,1.0,0.868205592262098 +DE09,2.0,0.13179440773790196 +DE10,1.0,0.10550332272898885 +DE10,2.0,0.8944966772710111 +DE11,1.0,1.0 +DE12,1.0,0.9684934490757606 +DE12,2.0,0.03150655092423938 +DE13,2.0,0.16256747641822888 +DE13,3.0,0.08047045383770378 +DE13,4.0,0.7569620697440674 +DE14,2.0,0.4292547437301253 +DE14,3.0,0.2897940033242233 +DE14,4.0,0.28095125294565143 +DE15,1.0,0.1380148995065702 +DE15,2.0,0.8619851004934298 +DE16,1.0,1.0 +DE17,1.0,0.9081897899946221 +DE17,2.0,0.09181021000537798 +DE18,1.0,0.7408788968739876 +DE18,2.0,0.25912110312601244 +DE19,3.0,0.09465791940018745 +DE19,4.0,0.9053420805998126 +DE20,4.0, +DE21,2.0,0.06976744186046512 +DE21,4.0,0.9302325581395349 diff --git a/tests/test_power_plants.py b/tests/test_power_plants.py index 25dbe93e..38d5e889 100644 --- a/tests/test_power_plants.py +++ b/tests/test_power_plants.py @@ -1,8 +1,8 @@ import os -import requests from nose.tools import eq_, assert_raises_regexp from unittest.mock import MagicMock from deflex import config as cfg, powerplants, geometries +from reegis.tools import download_file def test_01_download_reegis_power_plants(): @@ -11,10 +11,7 @@ def test_01_download_reegis_power_plants(): file = 'reegis_pp_test.h5' filename = os.path.join(path, file) - if not os.path.isfile(filename): - req = requests.get(url) - with open(filename, 'wb') as fout: - fout.write(req.content) + download_file(filename, url) def test_02_create_deflex_powerplants(): @@ -25,26 +22,8 @@ def test_02_create_deflex_powerplants(): filename_out=fn_out) -def test_03_download_deflex_full_pp(): - url = 'https://osf.io/qdx2c/download' - filename = os.path.join( - cfg.get('paths', 'powerplants'), - cfg.get('powerplants', 'deflex_pp')).format( - map=cfg.get('init', 'map')) - if not os.path.isfile(filename): - req = requests.get(url) - with open(filename, 'wb') as fout: - fout.write(req.content) - - -def test_04_deflex_power_plants_by_year(): - de = geometries.deflex_regions('de21') - pp = powerplants.get_deflex_pp_by_year(de, 2014, 'de21', - overwrite_capacity=True) - eq_(int(pp['capacity'].sum()), 181489) - - -def test_05_not_existing_file(): +def test_03_not_existing_file(): + old_value = cfg.get('paths', 'powerplants') cfg.tmp_set('paths', 'powerplants', '/home/pet/') de = geometries.deflex_regions('de22') powerplants.pp_reegis2deflex = MagicMock(return_value='/home/pet/pp.h5') @@ -52,3 +31,4 @@ def test_05_not_existing_file(): with assert_raises_regexp(Exception, "File /home/pet/pp.h5 does not exist"): powerplants.get_deflex_pp_by_year(de, 2012, 'de22') + cfg.tmp_set('paths', 'powerplants', old_value) diff --git a/tests/test_scenario_a_init.py b/tests/test_scenario_a_init.py new file mode 100644 index 00000000..9fa1ac65 --- /dev/null +++ b/tests/test_scenario_a_init.py @@ -0,0 +1,16 @@ +import os +import requests +from deflex import config as cfg + + +def test_downloaf_pp_from_osf(): + """Download pp-file from osf.""" + url = 'https://osf.io/qtc56/download' + path = cfg.get('paths', 'powerplants') + file = 'de21_pp.h5' + filename = os.path.join(path, file) + + if not os.path.isfile(filename): + req = requests.get(url) + with open(filename, 'wb') as fout: + fout.write(req.content) diff --git a/tests/test_scenario_demand.py b/tests/test_scenario_demand.py index b6fb3d83..7be47934 100644 --- a/tests/test_scenario_demand.py +++ b/tests/test_scenario_demand.py @@ -1,7 +1,5 @@ import os -import requests -from nose.tools import eq_, assert_raises_regexp, with_setup -from unittest.mock import MagicMock +from nose.tools import eq_, with_setup from deflex import config as cfg, basic_scenario, geometries from reegis.tools import download_file @@ -15,8 +13,8 @@ def setup_func(): filename = os.path.join(path, file) download_file(filename, url) - url = 'https://osf.io/m435r/download' - file = 'heat_profile_state_2014_weather_2014.csv' + url = 'https://osf.io/6vmdh/download' + file = 'oep_ego_demand_combined.h5' filename = os.path.join(path, file) download_file(filename, url) diff --git a/tests/test_scenario_feedin.py b/tests/test_scenario_feedin.py index 85e5b5a2..73347f42 100644 --- a/tests/test_scenario_feedin.py +++ b/tests/test_scenario_feedin.py @@ -1,7 +1,6 @@ import os -import requests -from nose.tools import eq_, assert_raises_regexp, with_setup -from unittest.mock import MagicMock +from shutil import copyfile +from nose.tools import with_setup, eq_ from deflex import config as cfg, basic_scenario, geometries from reegis.tools import download_file @@ -24,10 +23,16 @@ def setup_func(): os.makedirs(path, exist_ok=True) download_file(filename, url) + src = os.path.join(os.path.dirname(__file__), 'data', 'windzone_de21.csv') + trg = os.path.join(cfg.get('paths', 'powerplants'), 'windzone_de21.csv') + copyfile(src, trg) + @with_setup(setup_func) def scenario_feedin_test(): + cfg.tmp_set('init', 'map', 'de21') regions = geometries.deflex_regions(rmap='de21') f = basic_scenario.scenario_feedin(regions, 2014, 'de21') - # eq_(int(d['DE01', 'district heating'].sum()), 18639262) - # eq_(int(d['DE05', 'electrical_load'].sum()), 10069) + eq_(int(f['DE01'].sum()['wind']), 2159) + eq_(int(f['DE01'].sum()['solar']), 913) + eq_(int(f['DE16'].sum()['wind']), 1753) diff --git a/tests/test_scenario_powerplant_and_chp.py b/tests/test_scenario_powerplant_and_chp.py index d8b982cc..6b52ce12 100644 --- a/tests/test_scenario_powerplant_and_chp.py +++ b/tests/test_scenario_powerplant_and_chp.py @@ -1,26 +1,19 @@ -import os -import requests from nose.tools import eq_ -from deflex import config as cfg, basic_scenario, geometries +from deflex import config as cfg, basic_scenario, geometries, powerplants class TestScenarioPowerplantsAndCHP: @classmethod def setUpClass(cls): - """Download pp-file from osf.""" - url = 'https://osf.io/qtc56/download' - path = cfg.get('paths', 'powerplants') - file = 'de21_pp.h5' - filename = os.path.join(path, file) - - if not os.path.isfile(filename): - req = requests.get(url) - with open(filename, 'wb') as fout: - fout.write(req.content) cls.regions = geometries.deflex_regions(rmap='de21') cls.pp = basic_scenario.scenario_powerplants( dict(), cls.regions, 2014, 'de21', 1) + def test_04_deflex_power_plants_by_year(self): + pp = powerplants.get_deflex_pp_by_year(self.regions, 2014, 'de21', + overwrite_capacity=True) + eq_(int(pp['capacity'].sum()), 181489) + def scenario_pp_test(self): eq_(float(self.pp['volatile_source']['DE03', 'wind']), 3052.8) eq_(float(self.pp['transformer'].loc['capacity', ('DE03', 'lignite')]), From 957c605891139aa7e35138506371e437eefedc4b Mon Sep 17 00:00:00 2001 From: uvchik Date: Tue, 12 Nov 2019 21:56:45 +0100 Subject: [PATCH 16/16] Fix pep8 issues --- tests/test_scenario_powerplant_and_chp.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_scenario_powerplant_and_chp.py b/tests/test_scenario_powerplant_and_chp.py index 6b52ce12..997d9cb3 100644 --- a/tests/test_scenario_powerplant_and_chp.py +++ b/tests/test_scenario_powerplant_and_chp.py @@ -1,5 +1,5 @@ from nose.tools import eq_ -from deflex import config as cfg, basic_scenario, geometries, powerplants +from deflex import basic_scenario, geometries, powerplants class TestScenarioPowerplantsAndCHP: @@ -9,7 +9,7 @@ def setUpClass(cls): cls.pp = basic_scenario.scenario_powerplants( dict(), cls.regions, 2014, 'de21', 1) - def test_04_deflex_power_plants_by_year(self): + def test_01_deflex_power_plants_by_year(self): pp = powerplants.get_deflex_pp_by_year(self.regions, 2014, 'de21', overwrite_capacity=True) eq_(int(pp['capacity'].sum()), 181489) @@ -18,7 +18,7 @@ def scenario_pp_test(self): eq_(float(self.pp['volatile_source']['DE03', 'wind']), 3052.8) eq_(float(self.pp['transformer'].loc['capacity', ('DE03', 'lignite')]), 1135.6) - + def test_scenario_transmission(self): lines = basic_scenario.scenario_transmission( self.pp, self.regions, 'de21') @@ -31,13 +31,13 @@ def test_scenario_transmission(self): float('inf')) eq_(str(lines.loc['DE07-DE05', ('electrical', 'distance')]), 'nan') eq_(float(lines.loc['DE07-DE05', ('electrical', 'efficiency')]), 1.0) - + def test_scenario_commodity_sources(self): src = basic_scenario.scenario_commodity_sources( self.pp, 2014)['commodity_source'] eq_(round(src.loc['costs', ('DE', 'hard coal')], 2), 8.93) eq_(round(src.loc['emission', ('DE', 'natural gas')], 2), 201.24) - + def test_chp(self): eq_(int(self.pp['transformer'].loc['capacity', ('DE01', 'hard coal')]), 1291)