From e15b0aad23dd5f3d348987ec3070d5d846749622 Mon Sep 17 00:00:00 2001 From: AmeliaNadal Date: Wed, 14 Dec 2022 20:22:46 +0100 Subject: [PATCH 01/16] Update CHANGELOG --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 041cc6bad..7af16c2e3 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -448,6 +448,8 @@ Changed `#1035 `_ * Add missing VOM costs for heat sector components `#942 `_ +* Add sanity checks for gas sector in eGon2035 + `#864 `_ Bug Fixes --------- From e57797e9508c69de99a4ba9547175f3754b2455b Mon Sep 17 00:00:00 2001 From: AmeliaNadal Date: Wed, 14 Dec 2022 20:24:07 +0100 Subject: [PATCH 02/16] Create new functions reused in sanity_checks Separate download of data in dedicated function for H2 storages and the insertion of gas pipelines in two functions in order to be able to reuse the new functions in the sanity checks. --- src/egon/data/datasets/gas_grid.py | 43 +++++++++++++++---- .../data/datasets/hydrogen_etrago/__init__.py | 4 +- .../data/datasets/hydrogen_etrago/storage.py | 8 +++- 3 files changed, 44 insertions(+), 11 deletions(-) mode change 100644 => 100755 src/egon/data/datasets/hydrogen_etrago/storage.py diff --git a/src/egon/data/datasets/gas_grid.py b/src/egon/data/datasets/gas_grid.py index 0f87fb5df..42eb68a19 100755 --- a/src/egon/data/datasets/gas_grid.py +++ b/src/egon/data/datasets/gas_grid.py @@ -329,10 +329,10 @@ def insert_gas_buses_abroad(scn_name="eGon2035"): return gdf_abroad_buses -def insert_gas_pipeline_list( +def define_gas_pipeline_list( gas_nodes_list, abroad_gas_nodes_list, scn_name="eGon2035" ): - """Insert list of gas pipelines from SciGRID_gas IGGIELGN data + """Define gas pipelines in Germany from SciGRID_gas IGGIELGN data Insert detailled description @@ -347,7 +347,8 @@ def insert_gas_pipeline_list( Returns ------- - None + gas_pipelines_list : pandas.DataFrame + Dataframe containing the gas pipelines in Germany """ abroad_gas_nodes_list = abroad_gas_nodes_list.set_index("country") @@ -356,8 +357,6 @@ def insert_gas_pipeline_list( "main_gas_carrier" ] - engine = db.engine() - # Select next id value new_id = db.next_etrago_id("link") @@ -660,8 +659,6 @@ def insert_gas_pipeline_list( "NUTS1_0", "NUTS1_1", "country_code", - "country_0", - "country_1", "diameter", "pipe_class", "classification", @@ -672,6 +669,33 @@ def insert_gas_pipeline_list( ] ) + return gas_pipelines_list + + +def insert_gas_pipeline_list(gas_pipelines_list, scn_name="eGon2035"): + """Insert list of gas pipelines in the database + + Insert detailled description + + Parameters + ---------- + gas_pipelines_list : pandas.DataFrame + Dataframe containing the gas pipelines in Germany + scn_name : str + Name of the scenario + + """ + main_gas_carrier = get_sector_parameters("gas", scenario=scn_name)[ + "main_gas_carrier" + ] + engine = db.engine() + gas_pipelines_list = gas_pipelines_list.drop( + columns=[ + "country_0", + "country_1", + ] + ) + # Clean db db.execute_sql( f"""DELETE FROM grid.egon_etrago_link @@ -752,7 +776,10 @@ def insert_gas_data(): insert_CH4_nodes_list(gas_nodes_list) abroad_gas_nodes_list = insert_gas_buses_abroad() - insert_gas_pipeline_list(gas_nodes_list, abroad_gas_nodes_list) + gas_pipeline_list = define_gas_pipeline_list( + gas_nodes_list, abroad_gas_nodes_list + ) + insert_gas_pipeline_list(gas_pipeline_list) remove_isolated_gas_buses() diff --git a/src/egon/data/datasets/hydrogen_etrago/__init__.py b/src/egon/data/datasets/hydrogen_etrago/__init__.py index eba35fc71..3a5db5744 100755 --- a/src/egon/data/datasets/hydrogen_etrago/__init__.py +++ b/src/egon/data/datasets/hydrogen_etrago/__init__.py @@ -15,10 +15,10 @@ insert_power_to_h2_to_power_eGon100RE, ) from egon.data.datasets.hydrogen_etrago.storage import ( - calculate_and_map_saltcavern_storage_potential, insert_H2_overground_storage, insert_H2_saltcavern_storage, insert_H2_storage_eGon100RE, + write_saltcavern_potential, ) @@ -29,7 +29,7 @@ def __init__(self, dependencies): version="0.0.1", dependencies=dependencies, tasks=( - calculate_and_map_saltcavern_storage_potential, + write_saltcavern_potential, insert_hydrogen_buses, insert_hydrogen_buses_eGon100RE, ), diff --git a/src/egon/data/datasets/hydrogen_etrago/storage.py b/src/egon/data/datasets/hydrogen_etrago/storage.py old mode 100644 new mode 100755 index 617f7b8e6..ee7e6e726 --- a/src/egon/data/datasets/hydrogen_etrago/storage.py +++ b/src/egon/data/datasets/hydrogen_etrago/storage.py @@ -159,7 +159,6 @@ def calculate_and_map_saltcavern_storage_potential(): # select onshore vg250 data sources = config.datasets()["bgr"]["sources"] - targets = config.datasets()["bgr"]["targets"] vg250_data = db.select_geodataframe( f"""SELECT * FROM {sources['vg250_federal_states']['schema']}. @@ -337,7 +336,14 @@ def calculate_and_map_saltcavern_storage_potential(): epsg=25832 ).area / potential_areas.groupby("gen")["shape_star"].transform("sum") + return potential_areas + +def write_saltcavern_potential(): + """Write saltcavern potentials in the database""" + potential_areas = calculate_and_map_saltcavern_storage_potential() + # write information to saltcavern data + targets = config.datasets()["bgr"]["targets"] potential_areas.to_crs(epsg=4326).to_postgis( targets["storage_potential"]["table"], db.engine(), From db0ae446331ebc2fa2f53e3421290bb60e868ebb Mon Sep 17 00:00:00 2001 From: AmeliaNadal Date: Wed, 14 Dec 2022 20:30:34 +0100 Subject: [PATCH 03/16] Add sanity checks for gas in Germany in eGon2035 --- src/egon/data/datasets/sanity_checks.py | 437 ++++++++++++++++++++++++ 1 file changed, 437 insertions(+) mode change 100644 => 100755 src/egon/data/datasets/sanity_checks.py diff --git a/src/egon/data/datasets/sanity_checks.py b/src/egon/data/datasets/sanity_checks.py old mode 100644 new mode 100755 index 29fe89c5b..902c748e8 --- a/src/egon/data/datasets/sanity_checks.py +++ b/src/egon/data/datasets/sanity_checks.py @@ -5,6 +5,7 @@ supply tables. Authors: @ALonso, @dana, @nailend, @nesnoj, @khelfen """ +import ast from math import isclose from pathlib import Path @@ -48,6 +49,15 @@ load_building_data, scenario_data, ) +from egon.data.datasets.gas_grid import ( + define_gas_nodes_list, + define_gas_pipeline_list, + insert_gas_buses_abroad, +) +from egon.data.datasets.hydrogen_etrago.storage import ( + calculate_and_map_saltcavern_storage_potential, +) +from egon.data.datasets.pypsaeursec import read_network from egon.data.datasets.scenario_parameters import get_sector_parameters from egon.data.datasets.storages.home_batteries import get_cbat_pbat_ratio import egon.data @@ -73,6 +83,7 @@ def __init__(self, dependencies): sanitycheck_emobility_mit, sanitycheck_pv_rooftop_buildings, sanitycheck_home_batteries, + etrago_eGon2035_gas_DE, }, ) @@ -1404,3 +1415,429 @@ def sanitycheck_home_batteries(): ) assert (home_batteries_df.round(6) == df.round(6)).all().all() + +def sanity_check_gas_buses(scn): + """Execute sanity checks for the gas buses in Germany + Returns print statements as sanity checks for the CH4 and + H2_grid grid buses in Germany. The deviation is calculated between + the number gas grid buses in the database and the original + Scigrid_gas number of gas buses. + Parameters + ---------- + scn_name : str + Name of the scenario + """ + logger.info(f"BUSES") + + target_file = ( + Path(".") / "datasets" / "gas_data" / "data" / "IGGIELGN_Nodes.csv" + ) + + Grid_buses_list = pd.read_csv( + target_file, + delimiter=";", + decimal=".", + usecols=["country_code"], + ) + + Grid_buses_list = Grid_buses_list[ + Grid_buses_list["country_code"].str.match("DE") + ] + input_grid_buses = len(Grid_buses_list.index) + + for carrier in ["CH4", "H2_grid"]: + + output_grid_buses_df = db.select_dataframe( + f""" + SELECT bus_id + FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND country = 'DE' + AND carrier = '{carrier}'; + """, + warning=False, + ) + output_grid_buses = len(output_grid_buses_df.index) + + e_grid_buses = ( + round( + (output_grid_buses - input_grid_buses) / input_grid_buses, + 2, + ) + * 100 + ) + logger.info(f"Deviation {carrier} buses: {e_grid_buses} %") + + +def sanity_check_CH4_stores(scn): + """Execute sanity checks for the CH4 stores in Germany + Returns print statements as sanity checks for the CH4 stores + capacity in Germany. The deviation is calculated between: + * the sum of the capacities of the stores with carrier 'CH4' + in the database (for one scenario) and + * the sum of: + * the capacity the gas grid allocated to CH4 (total capacity + in eGon2035 and capacity reduced the share of the grid + allocated to H2 in eGon100RE) and + * the sum of the capacities of the stores in the source + document (Storages from the SciGRID_gas data) + Parameters + ---------- + scn_name : str + Name of the scenario + """ + output_CH4_stores = db.select_dataframe( + f"""SELECT SUM(e_nom::numeric) as e_nom_germany + FROM grid.egon_etrago_store + WHERE scn_name = '{scn}' + AND carrier = 'CH4' + AND bus IN + (SELECT bus_id + FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND country = 'DE' + AND carrier = 'CH4'); + """, + warning=False, + )["e_nom_germany"].values[0] + + target_file = ( + Path(".") / "datasets" / "gas_data" / "data" / "IGGIELGN_Storages.csv" + ) + + CH4_storages_list = pd.read_csv( + target_file, + delimiter=";", + decimal=".", + usecols=["country_code", "param"], + ) + + CH4_storages_list = CH4_storages_list[ + CH4_storages_list["country_code"].str.match("DE") + ] + + max_workingGas_M_m3 = [] + end_year = [] + for index, row in CH4_storages_list.iterrows(): + param = ast.literal_eval(row["param"]) + end_year.append(param["end_year"]) + max_workingGas_M_m3.append(param["max_workingGas_M_m3"]) + CH4_storages_list["max_workingGas_M_m3"] = max_workingGas_M_m3 + CH4_storages_list["end_year"] = [ + float("inf") if x == None else x for x in end_year + ] + + # Remove unused storage units + CH4_storages_list = CH4_storages_list[ + CH4_storages_list["end_year"] + >= get_sector_parameters("global", scn)["population_year"] + ] + + if scn == "eGon2035": + grid_cap = 130000 + elif scn == "eGon100RE": + grid_cap = 13000 * ( + 1 + - get_sector_parameters("gas", "eGon100RE")[ + "retrofitted_CH4pipeline-to-H2pipeline_share" + ] + ) + conv_factor = 10830 # gross calorific value = 39 MJ/m3 (eurogas.org) + input_CH4_stores = ( + conv_factor * sum(CH4_storages_list["max_workingGas_M_m3"].to_list()) + + grid_cap + ) + + e_CH4_stores = ( + round( + (output_CH4_stores - input_CH4_stores) / input_CH4_stores, + 2, + ) + * 100 + ) + logger.info(f"Deviation CH4 stores: {e_CH4_stores} %") + + +def sanity_check_H2_saltcavern_stores(scn): + """Execute sanity checks for the H2 saltcavern stores in Germany + Returns print as sanity checks for the H2 saltcavern potential + storage capacity in Germany. The deviation is calculated between: + * the sum of the of the H2 saltcavern potential storage capacity + (e_nom_max) in the database and + * the sum of the H2 saltcavern potential storage capacity + assumed to be the ratio of the areas of 500 m radius around + substations in each german federal state and the estimated + total hydrogen storage potential of the corresponding federal + state (data from InSpEE-DS report). + This test works also in test mode. + Parameters + ---------- + scn_name : str + Name of the scenario + """ + output_H2_stores = db.select_dataframe( + f"""SELECT SUM(e_nom_max::numeric) as e_nom_max_germany + FROM grid.egon_etrago_store + WHERE scn_name = '{scn}' + AND carrier = 'H2_underground' + AND bus IN + (SELECT bus_id + FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND country = 'DE' + AND carrier = 'H2_saltcavern'); + """, + warning=False, + )["e_nom_max_germany"].values[0] + + storage_potentials = calculate_and_map_saltcavern_storage_potential() + storage_potentials["storage_potential"] = ( + storage_potentials["area_fraction"] * storage_potentials["potential"] + ) + input_H2_stores = sum(storage_potentials["storage_potential"].to_list()) + + e_H2_stores = ( + round( + (output_H2_stores - input_H2_stores) / input_H2_stores, + 2, + ) + * 100 + ) + logger.info(f"Deviation H2 saltcavern stores: {e_H2_stores} %") + + +def sanity_check_CH4_grid(scn): + """Execute sanity checks for the gas grid capacity in Germany + Returns print statements as sanity checks for the CH4 links + (pipelines) in Germany. The deviation is calculated between + the sum of the power (p_nom) of all the CH4 pipelines in Germany + for one scenario in the database and the sum of the powers of the + imported pipelines. + In eGon100RE, the sum is reduced by the share of the grid that is + allocated to hydrogen (share calculated by PyPSA-eur-sec). + This test works also in test mode. + Parameters + ---------- + scn_name : str + Name of the scenario + Returns + ------- + scn_name : float + Sum of the power (p_nom) of all the pipelines in Germany + """ + grid_carrier = "CH4" + output_gas_grid = db.select_dataframe( + f"""SELECT SUM(p_nom::numeric) as p_nom_germany + FROM grid.egon_etrago_link + WHERE scn_name = '{scn}' + AND carrier = '{grid_carrier}' + AND bus0 IN + (SELECT bus_id + FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND country = 'DE' + AND carrier = '{grid_carrier}') + AND bus1 IN + (SELECT bus_id + FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND country = 'DE' + AND carrier = '{grid_carrier}') + ; + """, + warning=False, + )["p_nom_germany"].values[0] + + gas_nodes_list = define_gas_nodes_list() + abroad_gas_nodes_list = insert_gas_buses_abroad() + gas_grid = define_gas_pipeline_list(gas_nodes_list, abroad_gas_nodes_list) + gas_grid_germany = gas_grid[ + (gas_grid["country_0"] == "DE") & (gas_grid["country_1"] == "DE") + ] + p_nom_total = sum(gas_grid_germany["p_nom"].to_list()) + + if scn == "eGon2035": + input_gas_grid = p_nom_total + if scn == "eGon100RE": + input_gas_grid = p_nom_total * ( + 1 + - get_sector_parameters("gas", "eGon100RE")[ + "retrofitted_CH4pipeline-to-H2pipeline_share" + ] + ) + + e_gas_grid = ( + round( + (output_gas_grid - input_gas_grid) / input_gas_grid, + 2, + ) + * 100 + ) + logger.info(f"Deviation of the capacity of the CH4 grid: {e_gas_grid} %") + + return p_nom_total + + +def etrago_eGon2035_gas_DE(): + """Execute basic sanity checks for the gas sector in eGon2035 + Returns print statements as sanity checks for the gas sector in + the eGon2035 scenario for the following components in Germany: + * Buses: with the function :py:func:`sanity_check_gas_buses` + * Loads: for the carriers 'CH4_for_industry' and 'H2_for_industry' + the deviation is calculated between the sum of the loads in the + database and the sum the loads in the sources document + (opendata.ffe database) + * Generators: the deviation is calculated between the sums of the + nominal powers of the gas generators in the database and of + the ones in the sources document (Biogaspartner Einspeiseatlas + Deutschland from the dena and Productions from the SciGRID_gas + data) + * Stores: deviations for stores with following carriers are + calculated: + * 'CH4': with the function :py:func:`sanity_check_CH4_stores` + * 'H2_underground': with the function :py:func:`sanity_check_H2_saltcavern_stores` + * Links: with the function :py:func:`sanity_check_CH4_grid` + """ + scn = "eGon2035" + + if TESTMODE_OFF: + logger.info(f"Gas sanity checks for scenario {scn}") + + # Buses + sanity_check_gas_buses(scn) + + # Loads + logger.info(f"LOADS") + + path = Path(".") / "datasets" / "gas_data" / "demand" + corr_file = path / "region_corr.json" + df_corr = pd.read_json(corr_file) + df_corr = df_corr.loc[:, ["id_region", "name_short"]] + df_corr.set_index("id_region", inplace=True) + + for carrier in ["CH4_for_industry", "H2_for_industry"]: + + output_gas_demand = db.select_dataframe( + f"""SELECT (SUM( + (SELECT SUM(p) + FROM UNNEST(b.p_set) p))/1000000)::numeric as load_twh + FROM grid.egon_etrago_load a + JOIN grid.egon_etrago_load_timeseries b + ON (a.load_id = b.load_id) + JOIN grid.egon_etrago_bus c + ON (a.bus=c.bus_id) + AND b.scn_name = '{scn}' + AND a.scn_name = '{scn}' + AND c.scn_name = '{scn}' + AND c.country = 'DE' + AND a.carrier = '{carrier}'; + """, + warning=False, + )["load_twh"].values[0] + + input_gas_demand = pd.read_json( + path / (carrier + "_eGon2035.json") + ) + input_gas_demand = input_gas_demand.loc[:, ["id_region", "value"]] + input_gas_demand.set_index("id_region", inplace=True) + input_gas_demand = pd.concat( + [input_gas_demand, df_corr], axis=1, join="inner" + ) + input_gas_demand["NUTS0"] = (input_gas_demand["name_short"].str)[ + 0:2 + ] + input_gas_demand = input_gas_demand[ + input_gas_demand["NUTS0"].str.match("DE") + ] + input_gas_demand = sum(input_gas_demand.value.to_list()) / 1000000 + + e_demand = ( + round( + (output_gas_demand - input_gas_demand) / input_gas_demand, + 2, + ) + * 100 + ) + logger.info(f"Deviation {carrier}: {e_demand} %") + + # Generators + logger.info(f"GENERATORS") + carrier_generator = "CH4" + + output_gas_generation = db.select_dataframe( + f"""SELECT SUM(p_nom::numeric) as p_nom_germany + FROM grid.egon_etrago_generator + WHERE scn_name = '{scn}' + AND carrier = '{carrier_generator}' + AND bus IN + (SELECT bus_id + FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND country = 'DE' + AND carrier = '{carrier_generator}'); + """, + warning=False, + )["p_nom_germany"].values[0] + + target_file = ( + Path(".") + / "datasets" + / "gas_data" + / "data" + / "IGGIELGN_Productions.csv" + ) + + NG_generators_list = pd.read_csv( + target_file, + delimiter=";", + decimal=".", + usecols=["country_code", "param"], + ) + + NG_generators_list = NG_generators_list[ + NG_generators_list["country_code"].str.match("DE") + ] + + p_NG = 0 + for index, row in NG_generators_list.iterrows(): + param = ast.literal_eval(row["param"]) + p_NG = p_NG + param["max_supply_M_m3_per_d"] + conversion_factor = 437.5 # MCM/day to MWh/h + p_NG = p_NG * conversion_factor + + basename = "Biogaspartner_Einspeiseatlas_Deutschland_2021.xlsx" + target_file = Path(".") / "datasets" / "gas_data" / basename + + conversion_factor_b = 0.01083 # m^3/h to MWh/h + p_biogas = ( + pd.read_excel( + target_file, + usecols=["Einspeisung Biomethan [(N*m^3)/h)]"], + )["Einspeisung Biomethan [(N*m^3)/h)]"].sum() + * conversion_factor_b + ) + + input_gas_generation = p_NG + p_biogas + e_generation = ( + round( + (output_gas_generation - input_gas_generation) + / input_gas_generation, + 2, + ) + * 100 + ) + logger.info( + f"Deviation {carrier_generator} generation: {e_generation} %" + ) + + # Stores + logger.info(f"STORES") + sanity_check_CH4_stores(scn) + sanity_check_H2_saltcavern_stores(scn) + + # Links + logger.info(f"LINKS") + sanity_check_CH4_grid(scn) + + else: + print("Testmode is on, skipping sanity check.") From 241b915516405148a483dd3dd5371d83332e40bb Mon Sep 17 00:00:00 2001 From: AmeliaNadal Date: Wed, 14 Dec 2022 20:31:32 +0100 Subject: [PATCH 04/16] Format documentation --- src/egon/data/datasets/sanity_checks.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/egon/data/datasets/sanity_checks.py b/src/egon/data/datasets/sanity_checks.py index 902c748e8..80aef9ee4 100755 --- a/src/egon/data/datasets/sanity_checks.py +++ b/src/egon/data/datasets/sanity_checks.py @@ -68,10 +68,15 @@ class SanityChecks(Dataset): + #: + name: str = "SanityChecks" + #: + version: str = "0.0.6" + def __init__(self, dependencies): super().__init__( - name="SanityChecks", - version="0.0.6", + name=self.name, + version=self.version, dependencies=dependencies, tasks={ etrago_eGon2035_electricity, From 9b33d5a6b13c8c4286c5b074a98c4a79a027d9ec Mon Sep 17 00:00:00 2001 From: AmeliaNadal Date: Mon, 19 Dec 2022 09:45:11 +0100 Subject: [PATCH 05/16] Separate definition and insertion in the DB of abroad gas buses --- src/egon/data/datasets/gas_grid.py | 53 ++++++++++++++++++------- src/egon/data/datasets/sanity_checks.py | 4 +- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/src/egon/data/datasets/gas_grid.py b/src/egon/data/datasets/gas_grid.py index 42eb68a19..4fc501566 100755 --- a/src/egon/data/datasets/gas_grid.py +++ b/src/egon/data/datasets/gas_grid.py @@ -223,12 +223,10 @@ def insert_CH4_nodes_list(gas_nodes_list): dtype={"geom": Geometry()}, ) +def define_gas_buses_abroad(scn_name="eGon2035"): + """Define central CH4 buses in foreign countries for eGon2035 -def insert_gas_buses_abroad(scn_name="eGon2035"): - """Insert central CH4 buses in foreign countries for eGon2035 - - Detailled description to be completed: - Insert central gas buses in foreign countries to db, same buses + Define central gas buses in foreign countries to db, same buses than the foreign AC buses Parameters @@ -241,6 +239,7 @@ def insert_gas_buses_abroad(scn_name="eGon2035"): gdf_abroad_buses : dataframe Dataframe containing the CH4 buses in the neighbouring countries and one in the center of Germany in test mode + """ # Select sources and targets from dataset configuration sources = config.datasets()["electrical_neighbours"]["sources"] @@ -249,15 +248,6 @@ def insert_gas_buses_abroad(scn_name="eGon2035"): "main_gas_carrier" ] - # Connect to local database - engine = db.engine() - db.execute_sql( - f""" - DELETE FROM grid.egon_etrago_bus WHERE "carrier" = '{main_gas_carrier}' AND - scn_name = '{scn_name}' AND country != 'DE'; - """ - ) - # Select the foreign buses gdf_abroad_buses = central_buses_egon100(sources) gdf_abroad_buses = gdf_abroad_buses.drop_duplicates(subset=["country"]) @@ -316,6 +306,41 @@ def insert_gas_buses_abroad(scn_name="eGon2035"): columns={"geometry": "geom"} ).set_geometry("geom", crs=4326) + return gdf_abroad_buses + + +def insert_gas_buses_abroad(scn_name="eGon2035"): + """Insert central CH4 buses in foreign countries for eGon2035 + + Insert central gas buses in foreign countries to db, same buses + than the foreign AC buses + + Parameters + ---------- + scn_name : str + Name of the scenario + + Returns + ------- + gdf_abroad_buses : dataframe + Dataframe containing the CH4 buses in the neighbouring countries + and one in the center of Germany in test mode + """ + main_gas_carrier = get_sector_parameters("gas", scenario=scn_name)[ + "main_gas_carrier" + ] + + # Connect to local database + engine = db.engine() + db.execute_sql( + f""" + DELETE FROM grid.egon_etrago_bus WHERE "carrier" = '{main_gas_carrier}' AND + scn_name = '{scn_name}' AND country != 'DE'; + """ + ) + + gdf_abroad_buses = define_gas_buses_abroad(scn_name) + # Insert to db print(gdf_abroad_buses) gdf_abroad_buses.to_postgis( diff --git a/src/egon/data/datasets/sanity_checks.py b/src/egon/data/datasets/sanity_checks.py index 80aef9ee4..1402fb521 100755 --- a/src/egon/data/datasets/sanity_checks.py +++ b/src/egon/data/datasets/sanity_checks.py @@ -52,7 +52,7 @@ from egon.data.datasets.gas_grid import ( define_gas_nodes_list, define_gas_pipeline_list, - insert_gas_buses_abroad, + define_gas_buses_abroad, ) from egon.data.datasets.hydrogen_etrago.storage import ( calculate_and_map_saltcavern_storage_potential, @@ -1654,7 +1654,7 @@ def sanity_check_CH4_grid(scn): )["p_nom_germany"].values[0] gas_nodes_list = define_gas_nodes_list() - abroad_gas_nodes_list = insert_gas_buses_abroad() + abroad_gas_nodes_list = define_gas_buses_abroad() gas_grid = define_gas_pipeline_list(gas_nodes_list, abroad_gas_nodes_list) gas_grid_germany = gas_grid[ (gas_grid["country_0"] == "DE") & (gas_grid["country_1"] == "DE") From 4ea1031658287b2e72a81aa6af816f7844f5a506 Mon Sep 17 00:00:00 2001 From: AmeliaNadal Date: Thu, 29 Dec 2022 14:52:41 +0100 Subject: [PATCH 06/16] New imports and apply isort --- src/egon/data/datasets/sanity_checks.py | 26 ++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/egon/data/datasets/sanity_checks.py b/src/egon/data/datasets/sanity_checks.py index 1402fb521..f461ddf00 100755 --- a/src/egon/data/datasets/sanity_checks.py +++ b/src/egon/data/datasets/sanity_checks.py @@ -5,9 +5,9 @@ supply tables. Authors: @ALonso, @dana, @nailend, @nesnoj, @khelfen """ -import ast from math import isclose from pathlib import Path +import ast from sqlalchemy import Numeric from sqlalchemy.sql import and_, cast, func, or_ @@ -42,21 +42,29 @@ EgonPfHvStore, EgonPfHvStoreTimeseries, ) -from egon.data.datasets.power_plants.pv_rooftop_buildings import ( - PV_CAP_PER_SQ_M, - ROOF_FACTOR, - SCENARIOS, - load_building_data, - scenario_data, -) from egon.data.datasets.gas_grid import ( + define_gas_buses_abroad, define_gas_nodes_list, define_gas_pipeline_list, - define_gas_buses_abroad, +) +from egon.data.datasets.gas_neighbours.eGon2035 import ( + calc_capacities, + calc_ch4_storage_capacities, + calc_global_ch4_demand, + calc_global_power_to_h2_demand, + calculate_ch4_grid_capacities, + import_ch4_demandTS, ) from egon.data.datasets.hydrogen_etrago.storage import ( calculate_and_map_saltcavern_storage_potential, ) +from egon.data.datasets.power_plants.pv_rooftop_buildings import ( + PV_CAP_PER_SQ_M, + ROOF_FACTOR, + SCENARIOS, + load_building_data, + scenario_data, +) from egon.data.datasets.pypsaeursec import read_network from egon.data.datasets.scenario_parameters import get_sector_parameters from egon.data.datasets.storages.home_batteries import get_cbat_pbat_ratio From e78faeaf1f4cff5ff383fd2b7964ba6841d60133 Mon Sep 17 00:00:00 2001 From: AmeliaNadal Date: Thu, 29 Dec 2022 14:55:59 +0100 Subject: [PATCH 07/16] Complete sanity checks of gas buses --- src/egon/data/datasets/sanity_checks.py | 55 ++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/src/egon/data/datasets/sanity_checks.py b/src/egon/data/datasets/sanity_checks.py index f461ddf00..f3089fc1c 100755 --- a/src/egon/data/datasets/sanity_checks.py +++ b/src/egon/data/datasets/sanity_checks.py @@ -1429,19 +1429,65 @@ def sanitycheck_home_batteries(): assert (home_batteries_df.round(6) == df.round(6)).all().all() + def sanity_check_gas_buses(scn): """Execute sanity checks for the gas buses in Germany - Returns print statements as sanity checks for the CH4 and - H2_grid grid buses in Germany. The deviation is calculated between - the number gas grid buses in the database and the original - Scigrid_gas number of gas buses. + + Returns print statements as sanity checks for the CH4, H2_grid and + H2_saltcavern buses. + * For all of them, it is checked if they are not isolated. + * For the grid buses, the deviation is calculated between the + number of gas grid buses in the database and the original + Scigrid_gas number of gas buses in Germany. + Parameters ---------- scn_name : str Name of the scenario + """ logger.info(f"BUSES") + # Are gas buses isolated? + corresponding_carriers = { + "eGon2035": { + "CH4": "CH4", + "H2_grid": "H2_feedin", + "H2_saltcavern": "power_to_H2", + }, + # "eGon100RE": { + # "CH4": "CH4", + # "H2_grid": "H2_retrofit", + # "H2_saltcavern": "H2_extension", + # } + } + for key in corresponding_carriers[scn]: + isolated_gas_buses = db.select_dataframe( + f""" + SELECT bus_id, carrier, country + FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND carrier = '{key}' + AND country = 'DE' + AND bus_id NOT IN + (SELECT bus0 + FROM grid.egon_etrago_link + WHERE scn_name = '{scn}' + AND carrier = '{corresponding_carriers[scn][key]}') + AND bus_id NOT IN + (SELECT bus1 + FROM grid.egon_etrago_link + WHERE scn_name = '{scn}' + AND carrier = '{corresponding_carriers[scn][key]}') + ; + """, + warning=False, + ) + if not isolated_gas_buses.empty: + logger.info(f"Isolated {key} buses:") + logger.info(isolated_gas_buses) + + # Deviation of the gas grid buses number target_file = ( Path(".") / "datasets" / "gas_data" / "data" / "IGGIELGN_Nodes.csv" ) @@ -1459,7 +1505,6 @@ def sanity_check_gas_buses(scn): input_grid_buses = len(Grid_buses_list.index) for carrier in ["CH4", "H2_grid"]: - output_grid_buses_df = db.select_dataframe( f""" SELECT bus_id From 543ea5a8bd7397dd405fdfbc21e34b331e947c3c Mon Sep 17 00:00:00 2001 From: AmeliaNadal Date: Thu, 29 Dec 2022 15:00:16 +0100 Subject: [PATCH 08/16] Add functions sanity_check_gas_one_port and sanity_check_gas_links --- src/egon/data/datasets/sanity_checks.py | 230 +++++++++++++++++++++++- 1 file changed, 229 insertions(+), 1 deletion(-) diff --git a/src/egon/data/datasets/sanity_checks.py b/src/egon/data/datasets/sanity_checks.py index f3089fc1c..f8107df63 100755 --- a/src/egon/data/datasets/sanity_checks.py +++ b/src/egon/data/datasets/sanity_checks.py @@ -1664,8 +1664,171 @@ def sanity_check_H2_saltcavern_stores(scn): logger.info(f"Deviation H2 saltcavern stores: {e_H2_stores} %") +def sanity_check_gas_one_port(scn): + """Check connections of gas one-port components + + Verify that gas one-port component (loads, generators, stores) are + all connected to a bus (of the right carrier) present in the data + base. Return print statements if this is not the case. + These sanity checks are not specific to Germany, they also include + the neighbouring countries. + + Parameters + ---------- + scn_name : str + Name of the scenario + + """ + if scn == "eGon2035": + # Loads + ## CH4_for_industry Germany + isolated_one_port_c = db.select_dataframe( + f""" + SELECT load_id, bus, carrier, scn_name + FROM grid.egon_etrago_load + WHERE scn_name = '{scn}' + AND carrier = 'CH4_for_industry' + AND bus NOT IN + (SELECT bus_id + FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND country = 'DE' + AND carrier = 'CH4') + ; + """, + warning=False, + ) + if not isolated_one_port_c.empty: + logger.info(f"Isolated loads:") + logger.info(isolated_one_port_c) + + ## CH4_for_industry abroad + isolated_one_port_c = db.select_dataframe( + f""" + SELECT load_id, bus, carrier, scn_name + FROM grid.egon_etrago_load + WHERE scn_name = '{scn}' + AND carrier = 'CH4' + AND bus NOT IN + (SELECT bus_id + FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND country != 'DE' + AND carrier = 'CH4') + ; + """, + warning=False, + ) + if not isolated_one_port_c.empty: + logger.info(f"Isolated loads:") + logger.info(isolated_one_port_c) + + ## H2_for_industry + isolated_one_port_c = db.select_dataframe( + f""" + SELECT load_id, bus, carrier, scn_name + FROM grid.egon_etrago_load + WHERE scn_name = '{scn}' + AND carrier = 'H2_for_industry' + AND (bus NOT IN + (SELECT bus_id + FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND country = 'DE' + AND carrier = 'H2_grid') + AND bus NOT IN + (SELECT bus_id + FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND country != 'DE' + AND carrier = 'AC')) + ; + """, + warning=False, + ) + if not isolated_one_port_c.empty: + logger.info(f"Isolated loads:") + logger.info(isolated_one_port_c) + + # Genrators + isolated_one_port_c = db.select_dataframe( + f""" + SELECT generator_id, bus, carrier, scn_name + FROM grid.egon_etrago_generator + WHERE scn_name = '{scn}' + AND carrier = 'CH4' + AND bus NOT IN + (SELECT bus_id + FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND carrier = 'CH4'); + ; + """, + warning=False, + ) + if not isolated_one_port_c.empty: + logger.info(f"Isolated generators:") + logger.info(isolated_one_port_c) + + # Stores + ## CH4 and H2_underground + corresponding_carriers = { + "CH4": "CH4", + "H2_saltcavern": "H2_underground", + } + for key in corresponding_carriers: + isolated_one_port_c = db.select_dataframe( + f""" + SELECT store_id, bus, carrier, scn_name + FROM grid.egon_etrago_store + WHERE scn_name = '{scn}' + AND carrier = '{corresponding_carriers[key]}' + AND bus NOT IN + (SELECT bus_id + FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND carrier = '{key}') + ; + """, + warning=False, + ) + if not isolated_one_port_c.empty: + logger.info(f"Isolated stores:") + logger.info(isolated_one_port_c) + + ## H2_overground + isolated_one_port_c = db.select_dataframe( + f""" + SELECT store_id, bus, carrier, scn_name + FROM grid.egon_etrago_store + WHERE scn_name = '{scn}' + AND carrier = 'H2_overground' + AND bus NOT IN + (SELECT bus_id + FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND country = 'DE' + AND carrier = 'H2_saltcavern') + AND bus NOT IN + (SELECT bus_id + FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND country = 'DE' + AND carrier = 'H2_grid') + ; + """, + warning=False, + ) + if not isolated_one_port_c.empty: + logger.info(f"Isolated stores:") + logger.info(isolated_one_port_c) + + # elif scn == "eGon2035": + + def sanity_check_CH4_grid(scn): """Execute sanity checks for the gas grid capacity in Germany + Returns print statements as sanity checks for the CH4 links (pipelines) in Germany. The deviation is calculated between the sum of the power (p_nom) of all the CH4 pipelines in Germany @@ -1736,8 +1899,60 @@ def sanity_check_CH4_grid(scn): return p_nom_total +def sanity_check_gas_links(scn): + """Check connections of gas links + + Verify that gas links are all connected to buses present in the data + base. Return print statements if this is not the case. + This sanity check is not specific to Germany, it also includes + the neighbouring countries. + + Parameters + ---------- + scn_name : str + Name of the scenario + + """ + carriers = [ + "CH4", + "H2_feedin", + "H2_to_CH4", + "CH4_to_H2", + "H2_to_power", + "power_to_H2", + "OCGT", + "central_gas_boiler", + "central_gas_CHP", + "central_gas_CHP_heat", + "industrial_gas_CHP", + ] + for c in carriers: + link_with_missing_bus = db.select_dataframe( + f""" + SELECT link_id, bus0, bus1, carrier, scn_name + FROM grid.egon_etrago_link + WHERE scn_name = '{scn}' + AND carrier = '{c}' + AND (bus0 NOT IN + (SELECT bus_id + FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}') + OR bus1 NOT IN + (SELECT bus_id + FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}')) + ; + """, + warning=False, + ) + if not link_with_missing_bus.empty: + logger.info(f"Links with missing bus:") + logger.info(link_with_missing_bus) + + def etrago_eGon2035_gas_DE(): """Execute basic sanity checks for the gas sector in eGon2035 + Returns print statements as sanity checks for the gas sector in the eGon2035 scenario for the following components in Germany: * Buses: with the function :py:func:`sanity_check_gas_buses` @@ -1754,7 +1969,15 @@ def etrago_eGon2035_gas_DE(): calculated: * 'CH4': with the function :py:func:`sanity_check_CH4_stores` * 'H2_underground': with the function :py:func:`sanity_check_H2_saltcavern_stores` - * Links: with the function :py:func:`sanity_check_CH4_grid` + * One-port components (loads, generators, stores): verification + that they are all connected to a bus present in the data base + with the function :py:func:`sanity_check_gas_one_port` + * Links: verification: + * that the gas links are all connected to buses present in + the data base with the function :py:func:`sanity_check_gas_links` + * of the capacity of the gas grid with the function + :py:func:`sanity_check_CH4_grid` + """ scn = "eGon2035" @@ -1893,9 +2116,14 @@ def etrago_eGon2035_gas_DE(): sanity_check_CH4_stores(scn) sanity_check_H2_saltcavern_stores(scn) + # One-port components + sanity_check_gas_one_port(scn) + # Links logger.info(f"LINKS") sanity_check_CH4_grid(scn) + sanity_check_gas_links(scn) else: print("Testmode is on, skipping sanity check.") + From 5f7f38885b95ca420a34dc4dce21c9ba978b2587 Mon Sep 17 00:00:00 2001 From: AmeliaNadal Date: Thu, 29 Dec 2022 15:02:06 +0100 Subject: [PATCH 09/16] Add sanity checks for gas abroad --- src/egon/data/datasets/sanity_checks.py | 231 ++++++++++++++++++++++++ 1 file changed, 231 insertions(+) diff --git a/src/egon/data/datasets/sanity_checks.py b/src/egon/data/datasets/sanity_checks.py index f8107df63..0d6081d4d 100755 --- a/src/egon/data/datasets/sanity_checks.py +++ b/src/egon/data/datasets/sanity_checks.py @@ -2127,3 +2127,234 @@ def etrago_eGon2035_gas_DE(): else: print("Testmode is on, skipping sanity check.") + +def etrago_eGon2035_gas_abroad(): + """Execute basic sanity checks for the gas sector in eGon2035 abroad + + Returns print statements as sanity checks for the gas sector in + the eGon2035 scenario for the following components in Germany: + * Buses + * Loads: for the carriers 'CH4' and 'H2_for_industry' + the deviation is calculated between the sum of the loads in the + database and the sum in the sources document (TYND) + * Generators: the deviation is calculated between the sums of the + nominal powers of the methane generators abroad in the database + and of the ones in the sources document (TYNP) + * Stores: the deviation for methane stores abroad is calculated + between the sum of the capacities in the data base and the one + of the source document (SciGRID_gas data) + * Links: verification of the capacity of the crossbordering gas + grid pipelines. + + """ + scn = "eGon2035" + + if TESTMODE_OFF: + logger.info(f"Gas sanity checks abroad for scenario {scn}") + + # Buses + logger.info(f"BUSES") + + # Are gas buses isolated? + corresponding_carriers = { + "eGon2035": { + "CH4": "CH4", + }, + # "eGon100RE": { + # "CH4": "CH4", + # "H2_grid": "H2_retrofit", + # } + } + for key in corresponding_carriers[scn]: + isolated_gas_buses_abroad = db.select_dataframe( + f""" + SELECT bus_id, carrier, country + FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND carrier = '{key}' + AND country != 'DE' + AND bus_id NOT IN + (SELECT bus0 + FROM grid.egon_etrago_link + WHERE scn_name = '{scn}' + AND carrier = '{corresponding_carriers[scn][key]}') + AND bus_id NOT IN + (SELECT bus1 + FROM grid.egon_etrago_link + WHERE scn_name = '{scn}' + AND carrier = '{corresponding_carriers[scn][key]}') + ; + """, + warning=False, + ) + if not isolated_gas_buses_abroad.empty: + logger.info(f"Isolated {key} buses abroad:") + logger.info(isolated_gas_buses_abroad) + + # Loads + logger.info(f"LOADS") + + ( + Norway_global_demand_1y, + normalized_ch4_demandTS, + ) = import_ch4_demandTS() + input_CH4_demand_abroad = calc_global_ch4_demand( + Norway_global_demand_1y + ) + input_CH4_demand = input_CH4_demand_abroad["GlobD_2035"].sum() + + ## CH4 + output_CH4_demand = db.select_dataframe( + f"""SELECT (SUM( + (SELECT SUM(p) + FROM UNNEST(b.p_set) p)))::numeric as load_mwh + FROM grid.egon_etrago_load a + JOIN grid.egon_etrago_load_timeseries b + ON (a.load_id = b.load_id) + JOIN grid.egon_etrago_bus c + ON (a.bus=c.bus_id) + AND b.scn_name = '{scn}' + AND a.scn_name = '{scn}' + AND c.scn_name = '{scn}' + AND c.country != 'DE' + AND a.carrier = 'CH4'; + """, + warning=False, + )["load_mwh"].values[0] + + e_demand_CH4 = ( + round( + (output_CH4_demand - input_CH4_demand) / input_CH4_demand, + 2, + ) + * 100 + ) + logger.info(f"Deviation CH4 load: {e_demand_CH4} %") + + ## H2_for_industry + input_power_to_h2_demand_abroad = calc_global_power_to_h2_demand() + input_H2_demand = input_power_to_h2_demand_abroad["GlobD_2035"].sum() + + output_H2_demand = db.select_dataframe( + f"""SELECT SUM(p_set::numeric) as p_set_abroad + FROM grid.egon_etrago_load + WHERE scn_name = '{scn}' + AND carrier = 'H2_for_industry' + AND bus IN + (SELECT bus_id + FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND country != 'DE' + AND carrier = 'AC'); + """, + warning=False, + )["p_set_abroad"].values[0] + + e_demand_H2 = ( + round( + (output_H2_demand - input_H2_demand) / input_H2_demand, + 2, + ) + * 100 + ) + logger.info(f"Deviation H2_for_industry load: {e_demand_H2} %") + + # Generators + logger.info(f"GENERATORS ") + CH4_gen = calc_capacities() + input_CH4_gen = CH4_gen["cap_2035"].sum() + + output_CH4_gen = db.select_dataframe( + f"""SELECT SUM(p_nom::numeric) as p_nom_abroad + FROM grid.egon_etrago_generator + WHERE scn_name = '{scn}' + AND carrier = 'CH4' + AND bus IN + (SELECT bus_id + FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND country != 'DE' + AND carrier = 'CH4'); + """, + warning=False, + )["p_nom_abroad"].values[0] + + e_gen = ( + round( + (output_CH4_gen - input_CH4_gen) / input_CH4_gen, + 2, + ) + * 100 + ) + logger.info(f"Deviation CH4 generators: {e_gen} %") + + # Stores + logger.info(f"STORES") + ch4_input_capacities = calc_ch4_storage_capacities() + input_CH4_stores = ch4_input_capacities["e_nom"].sum() + + output_CH4_stores = db.select_dataframe( + f"""SELECT SUM(e_nom::numeric) as e_nom_abroad + FROM grid.egon_etrago_store + WHERE scn_name = '{scn}' + AND carrier = 'CH4' + AND bus IN + (SELECT bus_id + FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND country != 'DE' + AND carrier = 'CH4'); + """, + warning=False, + )["e_nom_abroad"].values[0] + + e_stores = ( + round( + (output_CH4_stores - input_CH4_stores) / input_CH4_stores, + 2, + ) + * 100 + ) + logger.info(f"Deviation CH4 stores: {e_stores} %") + + # Links + logger.info(f"LINKS") + ch4_grid_input_capacities = calculate_ch4_grid_capacities() + input_CH4_grid = ch4_grid_input_capacities["p_nom"].sum() + + grid_carrier = "CH4" + output_gas_grid = db.select_dataframe( + f"""SELECT SUM(p_nom::numeric) as p_nom + FROM grid.egon_etrago_link + WHERE scn_name = '{scn}' + AND carrier = '{grid_carrier}' + AND (bus0 IN + (SELECT bus_id + FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND country != 'DE' + AND carrier = '{grid_carrier}') + OR bus1 IN + (SELECT bus_id + FROM grid.egon_etrago_bus + WHERE scn_name = '{scn}' + AND country != 'DE' + AND carrier = '{grid_carrier}')) + ; + """, + warning=False, + )["p_nom"].values[0] + + e_gas_grid = ( + round( + (output_gas_grid - input_CH4_grid) / input_CH4_grid, + 2, + ) + * 100 + ) + logger.info( + f"Deviation of the capacity of the crossbordering CH4 grid: {e_gas_grid} %" + ) + + else: + print("Testmode is on, skipping sanity check.") From 09a922f5f3ac8f53078e26c7ee22e2ad08ea233e Mon Sep 17 00:00:00 2001 From: AmeliaNadal Date: Thu, 29 Dec 2022 15:03:09 +0100 Subject: [PATCH 10/16] Correct typo and formating --- src/egon/data/datasets/gas_neighbours/gas_abroad.py | 2 +- src/egon/data/datasets/sanity_checks.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/egon/data/datasets/gas_neighbours/gas_abroad.py b/src/egon/data/datasets/gas_neighbours/gas_abroad.py index a5b2bf3a5..dd2297774 100755 --- a/src/egon/data/datasets/gas_neighbours/gas_abroad.py +++ b/src/egon/data/datasets/gas_neighbours/gas_abroad.py @@ -18,7 +18,7 @@ def insert_gas_grid_capacities(Neighbouring_pipe_capacities_list, scn_name): For eGon2035, all the CH4 crossbordering pipelines are inserted there (no H2 grid in this scenario). For eGon100RE, only the the crossbordering pipelines with Germany - are inserted there (the other ones are inerted in PypsaEurSec), + are inserted there (the other ones are inserted in PypsaEurSec), but in this scenario there are H2 and CH4 pipelines. Parameters diff --git a/src/egon/data/datasets/sanity_checks.py b/src/egon/data/datasets/sanity_checks.py index 0d6081d4d..ae72439d5 100755 --- a/src/egon/data/datasets/sanity_checks.py +++ b/src/egon/data/datasets/sanity_checks.py @@ -1529,6 +1529,7 @@ def sanity_check_gas_buses(scn): def sanity_check_CH4_stores(scn): """Execute sanity checks for the CH4 stores in Germany + Returns print statements as sanity checks for the CH4 stores capacity in Germany. The deviation is calculated between: * the sum of the capacities of the stores with carrier 'CH4' @@ -1539,10 +1540,12 @@ def sanity_check_CH4_stores(scn): allocated to H2 in eGon100RE) and * the sum of the capacities of the stores in the source document (Storages from the SciGRID_gas data) + Parameters ---------- scn_name : str Name of the scenario + """ output_CH4_stores = db.select_dataframe( f"""SELECT SUM(e_nom::numeric) as e_nom_germany @@ -1618,6 +1621,7 @@ def sanity_check_CH4_stores(scn): def sanity_check_H2_saltcavern_stores(scn): """Execute sanity checks for the H2 saltcavern stores in Germany + Returns print as sanity checks for the H2 saltcavern potential storage capacity in Germany. The deviation is calculated between: * the sum of the of the H2 saltcavern potential storage capacity @@ -1628,10 +1632,12 @@ def sanity_check_H2_saltcavern_stores(scn): total hydrogen storage potential of the corresponding federal state (data from InSpEE-DS report). This test works also in test mode. + Parameters ---------- scn_name : str Name of the scenario + """ output_H2_stores = db.select_dataframe( f"""SELECT SUM(e_nom_max::numeric) as e_nom_max_germany @@ -1837,14 +1843,17 @@ def sanity_check_CH4_grid(scn): In eGon100RE, the sum is reduced by the share of the grid that is allocated to hydrogen (share calculated by PyPSA-eur-sec). This test works also in test mode. + Parameters ---------- scn_name : str Name of the scenario + Returns ------- scn_name : float Sum of the power (p_nom) of all the pipelines in Germany + """ grid_carrier = "CH4" output_gas_grid = db.select_dataframe( From ef8eb38248d4063b9b93daea23d8f1e37f7af45f Mon Sep 17 00:00:00 2001 From: AmeliaNadal Date: Thu, 29 Dec 2022 15:22:40 +0100 Subject: [PATCH 11/16] Increase dataset version --- src/egon/data/datasets/sanity_checks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/egon/data/datasets/sanity_checks.py b/src/egon/data/datasets/sanity_checks.py index ae72439d5..7ecc6c5e6 100755 --- a/src/egon/data/datasets/sanity_checks.py +++ b/src/egon/data/datasets/sanity_checks.py @@ -79,7 +79,7 @@ class SanityChecks(Dataset): #: name: str = "SanityChecks" #: - version: str = "0.0.6" + version: str = "0.0.7" def __init__(self, dependencies): super().__init__( From 91acb6638f82ff3ec4a08ab66a17b4dd489cfb57 Mon Sep 17 00:00:00 2001 From: AmeliaNadal Date: Wed, 1 Feb 2023 15:39:01 +0100 Subject: [PATCH 12/16] Add sanity checks abroads --- src/egon/data/datasets/sanity_checks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/egon/data/datasets/sanity_checks.py b/src/egon/data/datasets/sanity_checks.py index 7e165bea0..11378e757 100755 --- a/src/egon/data/datasets/sanity_checks.py +++ b/src/egon/data/datasets/sanity_checks.py @@ -97,6 +97,7 @@ def __init__(self, dependencies): sanitycheck_pv_rooftop_buildings, sanitycheck_home_batteries, etrago_eGon2035_gas_DE, + etrago_eGon2035_gas_abroad, sanitycheck_dsm, }, ) From fd246f6c2bd19732dbfacee8cc7c03da0902d2a9 Mon Sep 17 00:00:00 2001 From: AmeliaNadal Date: Wed, 1 Feb 2023 15:39:37 +0100 Subject: [PATCH 13/16] Correct typo --- src/egon/data/datasets/sanity_checks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/egon/data/datasets/sanity_checks.py b/src/egon/data/datasets/sanity_checks.py index 11378e757..19946abe9 100755 --- a/src/egon/data/datasets/sanity_checks.py +++ b/src/egon/data/datasets/sanity_checks.py @@ -2147,10 +2147,10 @@ def etrago_eGon2035_gas_abroad(): * Buses * Loads: for the carriers 'CH4' and 'H2_for_industry' the deviation is calculated between the sum of the loads in the - database and the sum in the sources document (TYND) + database and the sum in the sources document (TYNDP) * Generators: the deviation is calculated between the sums of the nominal powers of the methane generators abroad in the database - and of the ones in the sources document (TYNP) + and of the ones in the sources document (TYNDP) * Stores: the deviation for methane stores abroad is calculated between the sum of the capacities in the data base and the one of the source document (SciGRID_gas data) From 3ee36a800d8097fdd8282cf22be5cb3adf5b62f1 Mon Sep 17 00:00:00 2001 From: AmeliaNadal Date: Fri, 10 Feb 2023 11:08:40 +0100 Subject: [PATCH 14/16] Remove f-string without interpolation --- src/egon/data/datasets/sanity_checks.py | 34 ++++++++++++------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/egon/data/datasets/sanity_checks.py b/src/egon/data/datasets/sanity_checks.py index 19946abe9..56874ce8a 100755 --- a/src/egon/data/datasets/sanity_checks.py +++ b/src/egon/data/datasets/sanity_checks.py @@ -1448,7 +1448,7 @@ def sanity_check_gas_buses(scn): Name of the scenario """ - logger.info(f"BUSES") + logger.info("BUSES") # Are gas buses isolated? corresponding_carriers = { @@ -1707,7 +1707,7 @@ def sanity_check_gas_one_port(scn): warning=False, ) if not isolated_one_port_c.empty: - logger.info(f"Isolated loads:") + logger.info("Isolated loads:") logger.info(isolated_one_port_c) ## CH4_for_industry abroad @@ -1728,7 +1728,7 @@ def sanity_check_gas_one_port(scn): warning=False, ) if not isolated_one_port_c.empty: - logger.info(f"Isolated loads:") + logger.info("Isolated loads:") logger.info(isolated_one_port_c) ## H2_for_industry @@ -1755,7 +1755,7 @@ def sanity_check_gas_one_port(scn): warning=False, ) if not isolated_one_port_c.empty: - logger.info(f"Isolated loads:") + logger.info("Isolated loads:") logger.info(isolated_one_port_c) # Genrators @@ -1775,7 +1775,7 @@ def sanity_check_gas_one_port(scn): warning=False, ) if not isolated_one_port_c.empty: - logger.info(f"Isolated generators:") + logger.info("Isolated generators:") logger.info(isolated_one_port_c) # Stores @@ -1801,7 +1801,7 @@ def sanity_check_gas_one_port(scn): warning=False, ) if not isolated_one_port_c.empty: - logger.info(f"Isolated stores:") + logger.info("Isolated stores:") logger.info(isolated_one_port_c) ## H2_overground @@ -1828,7 +1828,7 @@ def sanity_check_gas_one_port(scn): warning=False, ) if not isolated_one_port_c.empty: - logger.info(f"Isolated stores:") + logger.info("Isolated stores:") logger.info(isolated_one_port_c) # elif scn == "eGon2035": @@ -1957,7 +1957,7 @@ def sanity_check_gas_links(scn): warning=False, ) if not link_with_missing_bus.empty: - logger.info(f"Links with missing bus:") + logger.info("Links with missing bus:") logger.info(link_with_missing_bus) @@ -1999,7 +1999,7 @@ def etrago_eGon2035_gas_DE(): sanity_check_gas_buses(scn) # Loads - logger.info(f"LOADS") + logger.info("LOADS") path = Path(".") / "datasets" / "gas_data" / "demand" corr_file = path / "region_corr.json" @@ -2053,7 +2053,7 @@ def etrago_eGon2035_gas_DE(): logger.info(f"Deviation {carrier}: {e_demand} %") # Generators - logger.info(f"GENERATORS") + logger.info("GENERATORS") carrier_generator = "CH4" output_gas_generation = db.select_dataframe( @@ -2123,7 +2123,7 @@ def etrago_eGon2035_gas_DE(): ) # Stores - logger.info(f"STORES") + logger.info("STORES") sanity_check_CH4_stores(scn) sanity_check_H2_saltcavern_stores(scn) @@ -2131,7 +2131,7 @@ def etrago_eGon2035_gas_DE(): sanity_check_gas_one_port(scn) # Links - logger.info(f"LINKS") + logger.info("LINKS") sanity_check_CH4_grid(scn) sanity_check_gas_links(scn) @@ -2164,7 +2164,7 @@ def etrago_eGon2035_gas_abroad(): logger.info(f"Gas sanity checks abroad for scenario {scn}") # Buses - logger.info(f"BUSES") + logger.info("BUSES") # Are gas buses isolated? corresponding_carriers = { @@ -2203,7 +2203,7 @@ def etrago_eGon2035_gas_abroad(): logger.info(isolated_gas_buses_abroad) # Loads - logger.info(f"LOADS") + logger.info("LOADS") ( Norway_global_demand_1y, @@ -2271,7 +2271,7 @@ def etrago_eGon2035_gas_abroad(): logger.info(f"Deviation H2_for_industry load: {e_demand_H2} %") # Generators - logger.info(f"GENERATORS ") + logger.info("GENERATORS ") CH4_gen = calc_capacities() input_CH4_gen = CH4_gen["cap_2035"].sum() @@ -2300,7 +2300,7 @@ def etrago_eGon2035_gas_abroad(): logger.info(f"Deviation CH4 generators: {e_gen} %") # Stores - logger.info(f"STORES") + logger.info("STORES") ch4_input_capacities = calc_ch4_storage_capacities() input_CH4_stores = ch4_input_capacities["e_nom"].sum() @@ -2329,7 +2329,7 @@ def etrago_eGon2035_gas_abroad(): logger.info(f"Deviation CH4 stores: {e_stores} %") # Links - logger.info(f"LINKS") + logger.info("LINKS") ch4_grid_input_capacities = calculate_ch4_grid_capacities() input_CH4_grid = ch4_grid_input_capacities["p_nom"].sum() From e9d3c5c53411968a2d968604845d69aa2ed94994 Mon Sep 17 00:00:00 2001 From: AmeliaNadal Date: Fri, 10 Feb 2023 11:10:29 +0100 Subject: [PATCH 15/16] Adapt sanity check to account for changes of PR #1101 --- src/egon/data/datasets/sanity_checks.py | 46 ++++--------------------- 1 file changed, 6 insertions(+), 40 deletions(-) diff --git a/src/egon/data/datasets/sanity_checks.py b/src/egon/data/datasets/sanity_checks.py index 56874ce8a..beeffc0ca 100755 --- a/src/egon/data/datasets/sanity_checks.py +++ b/src/egon/data/datasets/sanity_checks.py @@ -1539,9 +1539,8 @@ def sanity_check_CH4_stores(scn): * the sum of: * the capacity the gas grid allocated to CH4 (total capacity in eGon2035 and capacity reduced the share of the grid - allocated to H2 in eGon100RE) and - * the sum of the capacities of the stores in the source - document (Storages from the SciGRID_gas data) + allocated to H2 in eGon100RE) + * the total capacity of the CH4 stores in Germany (source: GIE) Parameters ---------- @@ -1564,38 +1563,6 @@ def sanity_check_CH4_stores(scn): warning=False, )["e_nom_germany"].values[0] - target_file = ( - Path(".") / "datasets" / "gas_data" / "data" / "IGGIELGN_Storages.csv" - ) - - CH4_storages_list = pd.read_csv( - target_file, - delimiter=";", - decimal=".", - usecols=["country_code", "param"], - ) - - CH4_storages_list = CH4_storages_list[ - CH4_storages_list["country_code"].str.match("DE") - ] - - max_workingGas_M_m3 = [] - end_year = [] - for index, row in CH4_storages_list.iterrows(): - param = ast.literal_eval(row["param"]) - end_year.append(param["end_year"]) - max_workingGas_M_m3.append(param["max_workingGas_M_m3"]) - CH4_storages_list["max_workingGas_M_m3"] = max_workingGas_M_m3 - CH4_storages_list["end_year"] = [ - float("inf") if x == None else x for x in end_year - ] - - # Remove unused storage units - CH4_storages_list = CH4_storages_list[ - CH4_storages_list["end_year"] - >= get_sector_parameters("global", scn)["population_year"] - ] - if scn == "eGon2035": grid_cap = 130000 elif scn == "eGon100RE": @@ -1605,11 +1572,10 @@ def sanity_check_CH4_stores(scn): "retrofitted_CH4pipeline-to-H2pipeline_share" ] ) - conv_factor = 10830 # gross calorific value = 39 MJ/m3 (eurogas.org) - input_CH4_stores = ( - conv_factor * sum(CH4_storages_list["max_workingGas_M_m3"].to_list()) - + grid_cap - ) + + stores_cap_D = 266424202 # MWh GIE https://www.gie.eu/transparency/databases/storage-database/ + + input_CH4_stores = stores_cap_D + grid_cap e_CH4_stores = ( round( From 92af37659494a461c4bd797b6de8004a5b49a104 Mon Sep 17 00:00:00 2001 From: IlkaCu Date: Fri, 30 Jun 2023 09:51:05 +0200 Subject: [PATCH 16/16] Apply black --- src/egon/data/datasets/gas_grid.py | 6 ++---- src/egon/data/datasets/hydrogen_etrago/storage.py | 3 ++- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/egon/data/datasets/gas_grid.py b/src/egon/data/datasets/gas_grid.py index b73611b61..6dc95e572 100755 --- a/src/egon/data/datasets/gas_grid.py +++ b/src/egon/data/datasets/gas_grid.py @@ -293,6 +293,7 @@ def insert_CH4_nodes_list(gas_nodes_list): dtype={"geom": Geometry()}, ) + def define_gas_buses_abroad(scn_name="eGon2035"): """ Define central CH4 buses in foreign countries for eGon2035 @@ -833,10 +834,7 @@ def insert_gas_pipeline_list(gas_pipelines_list, scn_name="eGon2035"): ] engine = db.engine() gas_pipelines_list = gas_pipelines_list.drop( - columns=[ - "country_0", - "country_1", - ] + columns=["country_0", "country_1"] ) # Clean db diff --git a/src/egon/data/datasets/hydrogen_etrago/storage.py b/src/egon/data/datasets/hydrogen_etrago/storage.py index 4e9787b96..dccf659c6 100755 --- a/src/egon/data/datasets/hydrogen_etrago/storage.py +++ b/src/egon/data/datasets/hydrogen_etrago/storage.py @@ -112,7 +112,7 @@ def insert_H2_saltcavern_storage(scn_name="eGon2035"): f""" SELECT * FROM {sources['H2_AC_map']['schema']}. - {sources['H2_AC_map']['table']}""", + {sources['H2_AC_map']['table']}""" ) storage_potentials["storage_potential"] = ( @@ -367,6 +367,7 @@ def calculate_and_map_saltcavern_storage_potential(): return potential_areas + def write_saltcavern_potential(): """Write saltcavern potentials in the database""" potential_areas = calculate_and_map_saltcavern_storage_potential()