From 1758076815049e5e38b52859ba81132c21c446d2 Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Thu, 11 May 2023 20:34:11 +0200 Subject: [PATCH 01/24] test param in add_electricity --- rules/build_electricity.smk | 8 ++++++ scripts/add_electricity.py | 50 ++++++++++++++++++------------------- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 133ceb6ea..194e186d9 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -273,6 +273,14 @@ rule add_electricity: nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", output: RESOURCES + "networks/elec.nc", + params: + costs=snakemake.config["costs"] + electricity=snakemake.config["electricity"] + renewable=snakemake.config["renewable"] + conventional=snakemake.config.get("conventional", {}) + countries=snakemake.config["countries"] + scaling_factor=snakemake.config["load"]["scaling_factor"] + length_factor=snakemake.config["lines"]["length_factor"] log: LOGS + "add_electricity.log", benchmark: diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 1d32bce18..bed5ee6b9 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -686,15 +686,15 @@ def estimate_renewable_capacities(n, config): ) -def add_nice_carrier_names(n, config): +def add_nice_carrier_names(n, plotting): carrier_i = n.carriers.index nice_names = ( - pd.Series(config["plotting"]["nice_names"]) + pd.Series(plotting["nice_names"]) .reindex(carrier_i) .fillna(carrier_i.to_series().str.title()) ) n.carriers["nice_name"] = nice_names - colors = pd.Series(config["plotting"]["tech_colors"]).reindex(carrier_i) + colors = pd.Series(plotting["tech_colors"]).reindex(carrier_i) if colors.isna().any(): missing_i = list(colors.index[colors.isna()]) logger.warning(f"tech_colors for carriers {missing_i} not defined in config.") @@ -713,23 +713,23 @@ def add_nice_carrier_names(n, config): costs = load_costs( snakemake.input.tech_costs, - snakemake.config["costs"], - snakemake.config["electricity"], + snakemake.param["costs"], + snakemake.param["electricity"], Nyears, ) ppl = load_powerplants(snakemake.input.powerplants) - if "renewable_carriers" in snakemake.config["electricity"]: - renewable_carriers = set(snakemake.config["electricity"]["renewable_carriers"]) + if "renewable_carriers" in snakemake.param["electricity"]: + renewable_carriers = set(snakemake.param["electricity"]["renewable_carriers"]) else: logger.warning( "Missing key `renewable_carriers` under config entry `electricity`. " "In future versions, this will raise an error. " "Falling back to carriers listed under `renewable`." ) - renewable_carriers = snakemake.config["renewable"] + renewable_carriers = snakemake.param["renewable"] - extendable_carriers = snakemake.config["electricity"]["extendable_carriers"] + extendable_carriers = snakemake.param["electricity"]["extendable_carriers"] if not (set(renewable_carriers) & set(extendable_carriers["Generator"])): logger.warning( "No renewables found in config entry `extendable_carriers`. " @@ -737,18 +737,18 @@ def add_nice_carrier_names(n, config): "Falling back to all renewables." ) - conventional_carriers = snakemake.config["electricity"]["conventional_carriers"] + conventional_carriers = snakemake.param["electricity"]["conventional_carriers"] attach_load( n, snakemake.input.regions, snakemake.input.load, snakemake.input.nuts3_shapes, - snakemake.config["countries"], - snakemake.config["load"]["scaling_factor"], + snakemake.param["countries"], + snakemake.param["load"]["scaling_factor"], ) - update_transmission_costs(n, costs, snakemake.config["lines"]["length_factor"]) + update_transmission_costs(n, costs, snakemake.param["lines"]["length_factor"]) conventional_inputs = { k: v for k, v in snakemake.input.items() if k.startswith("conventional_") @@ -759,7 +759,7 @@ def add_nice_carrier_names(n, config): ppl, conventional_carriers, extendable_carriers, - snakemake.config.get("conventional", {}), + snakemake.param.get("conventional", {}), conventional_inputs, ) @@ -769,11 +769,11 @@ def add_nice_carrier_names(n, config): snakemake.input, renewable_carriers, extendable_carriers, - snakemake.config["lines"]["length_factor"], + snakemake.param["lines"]["length_factor"], ) if "hydro" in renewable_carriers: - conf = snakemake.config["renewable"]["hydro"] + conf = snakemake.param["renewable"]["hydro"] attach_hydro( n, costs, @@ -784,7 +784,7 @@ def add_nice_carrier_names(n, config): **conf, ) - if "estimate_renewable_capacities" not in snakemake.config["electricity"]: + if "estimate_renewable_capacities" not in snakemake.param["electricity"]: logger.warning( "Missing key `estimate_renewable_capacities` under config entry `electricity`. " "In future versions, this will raise an error. " @@ -792,18 +792,18 @@ def add_nice_carrier_names(n, config): ) if ( "estimate_renewable_capacities_from_capacity_stats" - in snakemake.config["electricity"] + in snakemake.param["electricity"] ): estimate_renewable_caps = { "enable": True, - **snakemake.config["electricity"][ + **snakemake.param["electricity"][ "estimate_renewable_capacities_from_capacity_stats" ], } else: estimate_renewable_caps = {"enable": False} else: - estimate_renewable_caps = snakemake.config["electricity"][ + estimate_renewable_caps = snakemake.param["electricity"][ "estimate_renewable_capacities" ] if "enable" not in estimate_renewable_caps: @@ -819,21 +819,21 @@ def add_nice_carrier_names(n, config): "Falling back to whether `renewable_capacities_from_opsd` is non-empty." ) from_opsd = bool( - snakemake.config["electricity"].get("renewable_capacities_from_opsd", False) + snakemake.param["electricity"].get("renewable_capacities_from_opsd", False) ) estimate_renewable_caps["from_opsd"] = from_opsd if estimate_renewable_caps["enable"]: if estimate_renewable_caps["from_opsd"]: - tech_map = snakemake.config["electricity"]["estimate_renewable_capacities"][ + tech_map = snakemake.param["electricity"]["estimate_renewable_capacities"][ "technology_mapping" ] attach_OPSD_renewables(n, tech_map) - estimate_renewable_capacities(n, snakemake.config) + estimate_renewable_capacities(n, snakemake.param) update_p_nom_max(n) - add_nice_carrier_names(n, snakemake.config) + add_nice_carrier_names(n, snakemake.param["plotting"]) - n.meta = snakemake.config + n.meta = snakemake.param n.export_to_netcdf(snakemake.output[0]) From aa50ea44cc550433b8db38bb241c359b8ccc92ce Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Thu, 11 May 2023 20:59:37 +0200 Subject: [PATCH 02/24] fix add_electricity mistake --- rules/build_electricity.smk | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 194e186d9..8b7780db1 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -274,13 +274,13 @@ rule add_electricity: output: RESOURCES + "networks/elec.nc", params: - costs=snakemake.config["costs"] - electricity=snakemake.config["electricity"] - renewable=snakemake.config["renewable"] - conventional=snakemake.config.get("conventional", {}) - countries=snakemake.config["countries"] - scaling_factor=snakemake.config["load"]["scaling_factor"] - length_factor=snakemake.config["lines"]["length_factor"] + costs=snakemake.config["costs"], + electricity=snakemake.config["electricity"], + renewable=snakemake.config["renewable"], + conventional=snakemake.config.get("conventional", {}), + countries=snakemake.config["countries"], + scaling_factor=snakemake.config["load"]["scaling_factor"], + length_factor=snakemake.config["lines"]["length_factor"], log: LOGS + "add_electricity.log", benchmark: From 1fc48d8753f93b2b2d2f24c2516cf16777d72cb4 Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Mon, 15 May 2023 10:33:17 +0200 Subject: [PATCH 03/24] using python algorithm to edit rules and script --- rules/build_electricity.smk | 55 +++++++++++++--- rules/build_sector.smk | 65 +++++++++++++++++++ rules/postprocess.smk | 13 ++++ rules/retrieve.smk | 2 + rules/solve_electricity.smk | 4 ++ rules/solve_myopic.smk | 12 ++++ rules/solve_overnight.smk | 2 + scripts/_helpers.py | 2 +- scripts/add_brownfield.py | 8 +-- scripts/add_electricity.py | 50 +++++++------- scripts/add_existing_baseyear.py | 20 +++--- scripts/add_extra_components.py | 4 +- scripts/build_ammonia_production.py | 2 +- scripts/build_biomass_potentials.py | 2 +- scripts/build_bus_regions.py | 2 +- scripts/build_cop_profiles.py | 2 +- scripts/build_cutout.py | 4 +- scripts/build_electricity_demand.py | 12 ++-- scripts/build_energy_totals.py | 12 ++-- scripts/build_gas_input_locations.py | 2 +- scripts/build_gas_network.py | 2 +- scripts/build_heat_demand.py | 2 +- scripts/build_hydro_profile.py | 4 +- scripts/build_industrial_distribution_key.py | 6 +- ...ustrial_energy_demand_per_country_today.py | 6 +- ...build_industrial_production_per_country.py | 10 +-- ...ustrial_production_per_country_tomorrow.py | 2 +- scripts/build_industry_sector_ratios.py | 4 +- scripts/build_population_layouts.py | 2 +- scripts/build_powerplants.py | 6 +- scripts/build_renewable_profiles.py | 4 +- scripts/build_retro_cost.py | 4 +- scripts/build_sequestration_potentials.py | 2 +- scripts/build_shapes.py | 4 +- scripts/build_solar_thermal_profiles.py | 4 +- scripts/build_temperature_profiles.py | 2 +- scripts/build_transport_demand.py | 4 +- scripts/cluster_gas_network.py | 2 +- scripts/cluster_network.py | 18 ++--- scripts/make_summary.py | 22 +++---- scripts/plot_network.py | 16 ++--- scripts/plot_summary.py | 32 ++++----- scripts/prepare_network.py | 20 +++--- scripts/prepare_sector_network.py | 42 ++++++------ scripts/retrieve_databundle.py | 4 +- scripts/simplify_network.py | 8 +-- scripts/solve_network.py | 2 +- scripts/solve_operations_network.py | 2 +- 48 files changed, 324 insertions(+), 187 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 8b7780db1..bb000f2b5 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -19,6 +19,10 @@ if config["enable"].get("prepare_links_p_nom", False): rule build_electricity_demand: + params: + snapshots=config["snapshots"], + countries=config["countries"], + load=config["load"], input: ancient("data/load_raw.csv"), output: @@ -34,6 +38,9 @@ rule build_electricity_demand: rule build_powerplants: + params: + electricity=config["electricity"], + countries=config["countries"], input: base_network=RESOURCES + "networks/base.nc", custom_powerplants="data/custom_powerplants.csv", @@ -79,6 +86,8 @@ rule base_network: rule build_shapes: + params: + countries=config["countries"], input: naturalearth=ancient("data/bundle/naturalearth/ne_10m_admin_0_countries.shp"), eez=ancient("data/bundle/eez/World_EEZ_v8_2014.shp"), @@ -104,6 +113,8 @@ rule build_shapes: rule build_bus_regions: + params: + countries=config["countries"], input: country_shapes=RESOURCES + "country_shapes.geojson", offshore_shapes=RESOURCES + "offshore_shapes.geojson", @@ -125,6 +136,9 @@ rule build_bus_regions: if config["enable"].get("build_cutout", False): rule build_cutout: + params: + snapshots=config["snapshots"], + atlite=config["atlite"], input: regions_onshore=RESOURCES + "regions_onshore.geojson", regions_offshore=RESOURCES + "regions_offshore.geojson", @@ -186,6 +200,8 @@ rule build_ship_raster: rule build_renewable_profiles: + params: + renewable=config["renewable"], input: base_network=RESOURCES + "networks/base.nc", corine=ancient("data/bundle/corine/g250_clc06_V18_5.tif"), @@ -235,6 +251,9 @@ rule build_renewable_profiles: rule build_hydro_profile: + params: + countries=config["countries"], + renewable=config["renewable"], input: country_shapes=RESOURCES + "country_shapes.geojson", eia_hydro_generation="data/eia_hydro_annual_generation.csv", @@ -252,6 +271,13 @@ rule build_hydro_profile: rule add_electricity: + params: + lines=config["lines"], + load=config["load"], + countries=config["countries"], + renewable=config["renewable"], + electricity=config["electricity"], + costs=config["costs"], input: **{ f"profile_{tech}": RESOURCES + f"profile_{tech}.nc" @@ -273,14 +299,6 @@ rule add_electricity: nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", output: RESOURCES + "networks/elec.nc", - params: - costs=snakemake.config["costs"], - electricity=snakemake.config["electricity"], - renewable=snakemake.config["renewable"], - conventional=snakemake.config.get("conventional", {}), - countries=snakemake.config["countries"], - scaling_factor=snakemake.config["load"]["scaling_factor"], - length_factor=snakemake.config["lines"]["length_factor"], log: LOGS + "add_electricity.log", benchmark: @@ -295,6 +313,10 @@ rule add_electricity: rule simplify_network: + params: + clustering=config["clustering"], + electricity=config["electricity"], + costs=config["costs"], input: network=RESOURCES + "networks/elec.nc", tech_costs=COSTS, @@ -320,6 +342,14 @@ rule simplify_network: rule cluster_network: + params: + solving=config["solving"], + electricity=config["electricity"], + costs=config["costs"], + lines=config["lines"], + renewable=config["renewable"], + clustering=config["clustering"], + enable=config["enable"], input: network=RESOURCES + "networks/elec_s{simpl}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson", @@ -351,6 +381,9 @@ rule cluster_network: rule add_extra_components: + params: + costs=config["costs"], + electricity=config["electricity"], input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}.nc", tech_costs=COSTS, @@ -370,6 +403,12 @@ rule add_extra_components: rule prepare_network: + params: + links=config["links"], + solving=config["solving"], + lines=config["lines"], + electricity=config["electricity"], + costs=config["costs"], input: RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc", tech_costs=COSTS, diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 1b724d1a3..d375b7b92 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -4,6 +4,8 @@ rule build_population_layouts: + params: + logging=config["logging"], input: nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", urban_percent="data/urban_percent.csv", @@ -70,6 +72,8 @@ rule build_simplified_population_layouts: if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: rule build_gas_network: + params: + logging=config["logging"], input: gas_network="data/gas_network/scigrid-gas/data/IGGIELGN_PipeSegments.geojson", output: @@ -84,6 +88,8 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: "../scripts/build_gas_network.py" rule build_gas_input_locations: + params: + logging=config["logging"], input: lng=HTTP.remote( "https://globalenergymonitor.org/wp-content/uploads/2022/09/Europe-Gas-Tracker-August-2022.xlsx", @@ -110,6 +116,8 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: "../scripts/build_gas_input_locations.py" rule cluster_gas_network: + params: + logging=config["logging"], input: cleaned_gas_network=RESOURCES + "gas_network.csv", regions_onshore=RESOURCES @@ -140,6 +148,8 @@ if not (config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]): rule build_heat_demands: + params: + snapshots=config["snapshots"], input: pop_layout=RESOURCES + "pop_layout_{scope}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -160,6 +170,8 @@ rule build_heat_demands: rule build_temperature_profiles: + params: + snapshots=config["snapshots"], input: pop_layout=RESOURCES + "pop_layout_{scope}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -181,6 +193,8 @@ rule build_temperature_profiles: rule build_cop_profiles: + params: + sector=config["sector"], input: temp_soil_total=RESOURCES + "temp_soil_total_elec_s{simpl}_{clusters}.nc", temp_soil_rural=RESOURCES + "temp_soil_rural_elec_s{simpl}_{clusters}.nc", @@ -208,6 +222,9 @@ rule build_cop_profiles: rule build_solar_thermal_profiles: + params: + snapshots=config["snapshots"], + solar_thermal=config["solar_thermal"], input: pop_layout=RESOURCES + "pop_layout_{scope}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -228,6 +245,11 @@ rule build_solar_thermal_profiles: rule build_energy_totals: + params: + run=config["run"], + countries=config["countries"], + energy=config["energy"], + logging=config["logging"], input: nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", co2="data/eea/UNFCCC_v23.csv", @@ -253,6 +275,8 @@ rule build_energy_totals: rule build_biomass_potentials: + params: + biomass=config["biomass"], input: enspreso_biomass=HTTP.remote( "https://cidportal.jrc.ec.europa.eu/ftp/jrc-opendata/ENSPRESO/ENSPRESO_BIOMASS.xlsx", @@ -315,6 +339,8 @@ if not config["sector"]["biomass_transport"]: if config["sector"]["regional_co2_sequestration_potential"]["enable"]: rule build_sequestration_potentials: + params: + sector=config["sector"], input: sequestration_potential=HTTP.remote( "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson", @@ -368,6 +394,8 @@ rule build_salt_cavern_potentials: rule build_ammonia_production: + params: + countries=config["countries"], input: usgs="data/myb1-2017-nitro.xls", output: @@ -386,6 +414,9 @@ rule build_ammonia_production: rule build_industry_sector_ratios: + params: + industry=config["industry"], + sector=config["sector"], input: ammonia_production=RESOURCES + "ammonia_production.csv", idees="data/jrc-idees-2015", @@ -405,6 +436,11 @@ rule build_industry_sector_ratios: rule build_industrial_production_per_country: + params: + run=config["run"], + industry=config["industry"], + countries=config["countries"], + logging=config["logging"], input: ammonia_production=RESOURCES + "ammonia_production.csv", jrc="data/jrc-idees-2015", @@ -426,6 +462,8 @@ rule build_industrial_production_per_country: rule build_industrial_production_per_country_tomorrow: + params: + industry=config["industry"], input: industrial_production_per_country=RESOURCES + "industrial_production_per_country.csv", @@ -450,6 +488,10 @@ rule build_industrial_production_per_country_tomorrow: rule build_industrial_distribution_key: + params: + industry=config["industry"], + countries=config["countries"], + logging=config["logging"], input: regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", @@ -524,6 +566,10 @@ rule build_industrial_energy_demand_per_node: rule build_industrial_energy_demand_per_country_today: + params: + run=config["run"], + countries=config["countries"], + industry=config["industry"], input: jrc="data/jrc-idees-2015", ammonia_production=RESOURCES + "ammonia_production.csv", @@ -570,6 +616,9 @@ rule build_industrial_energy_demand_per_node_today: if config["sector"]["retrofitting"]["retro_endogen"]: rule build_retro_cost: + params: + sector=config["sector"], + countries=config["countries"], input: building_stock="data/retro/data_building_stock.csv", data_tabula="data/retro/tabula-calculator-calcsetbuilding.csv", @@ -640,6 +689,9 @@ rule build_shipping_demand: rule build_transport_demand: + params: + snapshots=config["snapshots"], + sector=config["sector"], input: clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", pop_weighted_energy_totals=RESOURCES @@ -666,6 +718,19 @@ rule build_transport_demand: rule prepare_sector_network: params: + co2_budget=config["co2_budget"], + solving=config["solving"], + existing_capacities=config["existing_capacities"], + foresight=config["foresight"], + costs=config["costs"], + logging=config["logging"], + sector=config["sector"], + industry=config["industry"], + pypsa_eur=config["pypsa_eur"], + lines=config["lines"], + scenario=config["scenario"], + countries=config["countries"], + energy=config["energy"], RDIR=RDIR, input: **build_retro_cost_output, diff --git a/rules/postprocess.smk b/rules/postprocess.smk index fae0f8564..1dfdd0987 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -9,6 +9,10 @@ localrules: rule plot_network: + params: + logging=config["logging"], + foresight=config["foresight"], + plotting=config["plotting"], input: overrides="data/override_component_attrs", network=RESULTS @@ -67,6 +71,11 @@ rule copy_conda_env: rule make_summary: params: + foresight=config["foresight"], + costs=config["costs"], + snapshots=config["snapshots"], + logging=config["logging"], + scenario=config["scenario"], RDIR=RDIR, input: overrides="data/override_component_attrs", @@ -114,6 +123,10 @@ rule make_summary: rule plot_summary: params: + logging=config["logging"], + countries=config["countries"], + scenario=config["scenario"], + plotting=config["plotting"], RDIR=RDIR, input: costs=RESULTS + "csvs/costs.csv", diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 0a96406ac..4bfbd6c65 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -19,6 +19,8 @@ if config["enable"].get("retrieve_databundle", True): datafiles.extend(["natura/Natura2000_end2015.shp", "GEBCO_2014_2D.nc"]) rule retrieve_databundle: + params: + tutorial=config["tutorial"], output: expand("data/bundle/{file}", file=datafiles), log: diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index 8ddeca92f..400220c1a 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -4,6 +4,8 @@ rule solve_network: + params: + solving=config["solving"], input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: @@ -30,6 +32,8 @@ rule solve_network: rule solve_operations_network: + params: + solving=config["solving"], input: network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index f10d81572..2ecba9996 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -4,6 +4,12 @@ rule add_existing_baseyear: + params: + scenario=config["scenario"], + sector=config["sector"], + logging=config["logging"], + existing_capacities=config["existing_capacities"], + costs=config["costs"], input: overrides="data/override_component_attrs", network=RESULTS @@ -42,6 +48,10 @@ rule add_existing_baseyear: rule add_brownfield: + params: + logging=config["logging"], + sector=config["sector"], + existing_capacities=config["existing_capacities"], input: overrides="data/override_component_attrs", network=RESULTS @@ -74,6 +84,8 @@ ruleorder: add_existing_baseyear > add_brownfield rule solve_sector_network_myopic: + params: + solving=config["solving"], input: overrides="data/override_component_attrs", network=RESULTS diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index c2e103e53..c36084714 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -4,6 +4,8 @@ rule solve_sector_network: + params: + solving=config["solving"], input: overrides="data/override_component_attrs", network=RESULTS diff --git a/scripts/_helpers.py b/scripts/_helpers.py index a67fb1057..2cc9a05a4 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -82,7 +82,7 @@ def load_network(import_name=None, custom_components=None): As in pypsa.Network(import_name) custom_components : dict Dictionary listing custom components. - For using ``snakemake.config['override_components']`` + For using ``snakemake.params['override_components']`` in ``config/config.yaml`` define: .. code:: yaml diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index e08b86d76..1d4e3a806 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -49,7 +49,7 @@ def add_brownfield(n, n_p, year): ) ] - threshold = snakemake.config["existing_capacities"]["threshold_capacity"] + threshold = snakemake.params["existing_capacities"]["threshold_capacity"] if not chp_heat.empty: threshold_chp_heat = ( @@ -87,7 +87,7 @@ def add_brownfield(n, n_p, year): # deal with gas network pipe_carrier = ["gas pipeline"] - if snakemake.config["sector"]["H2_retrofit"]: + if snakemake.params["sector"]["H2_retrofit"]: # drop capacities of previous year to avoid duplicating to_drop = n.links.carrier.isin(pipe_carrier) & (n.links.build_year != year) n.mremove("Link", n.links.loc[to_drop].index) @@ -98,7 +98,7 @@ def add_brownfield(n, n_p, year): & (n.links.build_year != year) ].index gas_pipes_i = n.links[n.links.carrier.isin(pipe_carrier)].index - CH4_per_H2 = 1 / snakemake.config["sector"]["H2_retrofit_capacity_per_CH4"] + CH4_per_H2 = 1 / snakemake.params["sector"]["H2_retrofit_capacity_per_CH4"] fr = "H2 pipeline retrofitted" to = "gas pipeline" # today's pipe capacity @@ -139,7 +139,7 @@ def add_brownfield(n, n_p, year): planning_horizons=2030, ) - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index bed5ee6b9..c3e847cea 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -686,15 +686,15 @@ def estimate_renewable_capacities(n, config): ) -def add_nice_carrier_names(n, plotting): +def add_nice_carrier_names(n, config): carrier_i = n.carriers.index nice_names = ( - pd.Series(plotting["nice_names"]) + pd.Series(config["plotting"]["nice_names"]) .reindex(carrier_i) .fillna(carrier_i.to_series().str.title()) ) n.carriers["nice_name"] = nice_names - colors = pd.Series(plotting["tech_colors"]).reindex(carrier_i) + colors = pd.Series(config["plotting"]["tech_colors"]).reindex(carrier_i) if colors.isna().any(): missing_i = list(colors.index[colors.isna()]) logger.warning(f"tech_colors for carriers {missing_i} not defined in config.") @@ -713,23 +713,23 @@ def add_nice_carrier_names(n, plotting): costs = load_costs( snakemake.input.tech_costs, - snakemake.param["costs"], - snakemake.param["electricity"], + snakemake.params["costs"], + snakemake.params["electricity"], Nyears, ) ppl = load_powerplants(snakemake.input.powerplants) - if "renewable_carriers" in snakemake.param["electricity"]: - renewable_carriers = set(snakemake.param["electricity"]["renewable_carriers"]) + if "renewable_carriers" in snakemake.params["electricity"]: + renewable_carriers = set(snakemake.params["electricity"]["renewable_carriers"]) else: logger.warning( "Missing key `renewable_carriers` under config entry `electricity`. " "In future versions, this will raise an error. " "Falling back to carriers listed under `renewable`." ) - renewable_carriers = snakemake.param["renewable"] + renewable_carriers = snakemake.params["renewable"] - extendable_carriers = snakemake.param["electricity"]["extendable_carriers"] + extendable_carriers = snakemake.params["electricity"]["extendable_carriers"] if not (set(renewable_carriers) & set(extendable_carriers["Generator"])): logger.warning( "No renewables found in config entry `extendable_carriers`. " @@ -737,18 +737,18 @@ def add_nice_carrier_names(n, plotting): "Falling back to all renewables." ) - conventional_carriers = snakemake.param["electricity"]["conventional_carriers"] + conventional_carriers = snakemake.params["electricity"]["conventional_carriers"] attach_load( n, snakemake.input.regions, snakemake.input.load, snakemake.input.nuts3_shapes, - snakemake.param["countries"], - snakemake.param["load"]["scaling_factor"], + snakemake.params["countries"], + snakemake.params["load"]["scaling_factor"], ) - update_transmission_costs(n, costs, snakemake.param["lines"]["length_factor"]) + update_transmission_costs(n, costs, snakemake.params["lines"]["length_factor"]) conventional_inputs = { k: v for k, v in snakemake.input.items() if k.startswith("conventional_") @@ -759,7 +759,7 @@ def add_nice_carrier_names(n, plotting): ppl, conventional_carriers, extendable_carriers, - snakemake.param.get("conventional", {}), + snakemake.config.get("conventional", {}), conventional_inputs, ) @@ -769,11 +769,11 @@ def add_nice_carrier_names(n, plotting): snakemake.input, renewable_carriers, extendable_carriers, - snakemake.param["lines"]["length_factor"], + snakemake.params["lines"]["length_factor"], ) if "hydro" in renewable_carriers: - conf = snakemake.param["renewable"]["hydro"] + conf = snakemake.params["renewable"]["hydro"] attach_hydro( n, costs, @@ -784,7 +784,7 @@ def add_nice_carrier_names(n, plotting): **conf, ) - if "estimate_renewable_capacities" not in snakemake.param["electricity"]: + if "estimate_renewable_capacities" not in snakemake.params["electricity"]: logger.warning( "Missing key `estimate_renewable_capacities` under config entry `electricity`. " "In future versions, this will raise an error. " @@ -792,18 +792,18 @@ def add_nice_carrier_names(n, plotting): ) if ( "estimate_renewable_capacities_from_capacity_stats" - in snakemake.param["electricity"] + in snakemake.params["electricity"] ): estimate_renewable_caps = { "enable": True, - **snakemake.param["electricity"][ + **snakemake.params["electricity"][ "estimate_renewable_capacities_from_capacity_stats" ], } else: estimate_renewable_caps = {"enable": False} else: - estimate_renewable_caps = snakemake.param["electricity"][ + estimate_renewable_caps = snakemake.params["electricity"][ "estimate_renewable_capacities" ] if "enable" not in estimate_renewable_caps: @@ -819,21 +819,21 @@ def add_nice_carrier_names(n, plotting): "Falling back to whether `renewable_capacities_from_opsd` is non-empty." ) from_opsd = bool( - snakemake.param["electricity"].get("renewable_capacities_from_opsd", False) + snakemake.params["electricity"].get("renewable_capacities_from_opsd", False) ) estimate_renewable_caps["from_opsd"] = from_opsd if estimate_renewable_caps["enable"]: if estimate_renewable_caps["from_opsd"]: - tech_map = snakemake.param["electricity"]["estimate_renewable_capacities"][ + tech_map = snakemake.params["electricity"]["estimate_renewable_capacities"][ "technology_mapping" ] attach_OPSD_renewables(n, tech_map) - estimate_renewable_capacities(n, snakemake.param) + estimate_renewable_capacities(n, snakemake.config) update_p_nom_max(n) - add_nice_carrier_names(n, snakemake.param["plotting"]) + add_nice_carrier_names(n, snakemake.config) - n.meta = snakemake.param + n.meta = snakemake.config n.export_to_netcdf(snakemake.output[0]) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 5bc0960d0..a24b078de 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -157,7 +157,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas # Fill missing DateOut dateout = ( df_agg.loc[biomass_i, "DateIn"] - + snakemake.config["costs"]["fill_values"]["lifetime"] + + snakemake.params["costs"]["fill_values"]["lifetime"] ) df_agg.loc[biomass_i, "DateOut"] = df_agg.loc[biomass_i, "DateOut"].fillna(dateout) @@ -218,7 +218,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas capacity = df.loc[grouping_year, generator] capacity = capacity[~capacity.isna()] capacity = capacity[ - capacity > snakemake.config["existing_capacities"]["threshold_capacity"] + capacity > snakemake.params["existing_capacities"]["threshold_capacity"] ] suffix = "-ac" if generator == "offwind" else "" name_suffix = f" {generator}{suffix}-{grouping_year}" @@ -582,7 +582,7 @@ def add_heating_capacities_installed_before_baseyear( ) # delete links with capacities below threshold - threshold = snakemake.config["existing_capacities"]["threshold_capacity"] + threshold = snakemake.params["existing_capacities"]["threshold_capacity"] n.mremove( "Link", [ @@ -608,14 +608,14 @@ def add_heating_capacities_installed_before_baseyear( planning_horizons=2020, ) - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) - options = snakemake.config["sector"] + options = snakemake.params["sector"] opts = snakemake.wildcards.sector_opts.split("-") - baseyear = snakemake.config["scenario"]["planning_horizons"][0] + baseyear = snakemake.params["scenario"]["planning_horizons"][0] overrides = override_component_attrs(snakemake.input.overrides) n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) @@ -626,14 +626,14 @@ def add_heating_capacities_installed_before_baseyear( Nyears = n.snapshot_weightings.generators.sum() / 8760.0 costs = prepare_costs( snakemake.input.costs, - snakemake.config["costs"], + snakemake.params["costs"], Nyears, ) - grouping_years_power = snakemake.config["existing_capacities"][ + grouping_years_power = snakemake.params["existing_capacities"][ "grouping_years_power" ] - grouping_years_heat = snakemake.config["existing_capacities"]["grouping_years_heat"] + grouping_years_heat = snakemake.params["existing_capacities"]["grouping_years_heat"] add_power_capacities_installed_before_baseyear( n, grouping_years_power, costs, baseyear ) @@ -650,7 +650,7 @@ def add_heating_capacities_installed_before_baseyear( .to_pandas() .reindex(index=n.snapshots) ) - default_lifetime = snakemake.config["costs"]["fill_values"]["lifetime"] + default_lifetime = snakemake.params["costs"]["fill_values"]["lifetime"] add_heating_capacities_installed_before_baseyear( n, baseyear, diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index 020370e54..08178c0a7 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -235,11 +235,11 @@ def attach_hydrogen_pipelines(n, costs, elec_opts): configure_logging(snakemake) n = pypsa.Network(snakemake.input.network) - elec_config = snakemake.config["electricity"] + elec_config = snakemake.params["electricity"] Nyears = n.snapshot_weightings.objective.sum() / 8760.0 costs = load_costs( - snakemake.input.tech_costs, snakemake.config["costs"], elec_config, Nyears + snakemake.input.tech_costs, snakemake.params["costs"], elec_config, Nyears ) attach_storageunits(n, costs, elec_config) diff --git a/scripts/build_ammonia_production.py b/scripts/build_ammonia_production.py index d78d627eb..6f03324fd 100644 --- a/scripts/build_ammonia_production.py +++ b/scripts/build_ammonia_production.py @@ -30,7 +30,7 @@ ammonia.index = cc.convert(ammonia.index, to="iso2") years = [str(i) for i in range(2013, 2018)] - countries = ammonia.index.intersection(snakemake.config["countries"]) + countries = ammonia.index.intersection(snakemake.params["countries"]) ammonia = ammonia.loc[countries, years].astype(float) # convert from ktonN to ktonNH3 diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py index 21d0e623a..35218e2ce 100644 --- a/scripts/build_biomass_potentials.py +++ b/scripts/build_biomass_potentials.py @@ -210,7 +210,7 @@ def convert_nuts2_to_regions(bio_nuts2, regions): snakemake = mock_snakemake("build_biomass_potentials", simpl="", clusters="5") - config = snakemake.config["biomass"] + config = snakemake.params["biomass"] year = config["year"] scenario = config["scenario"] diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index 6dc3b5a4d..e93787921 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -116,7 +116,7 @@ def voronoi_partition_pts(points, outline): snakemake = mock_snakemake("build_bus_regions") configure_logging(snakemake) - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] n = pypsa.Network(snakemake.input.base_network) diff --git a/scripts/build_cop_profiles.py b/scripts/build_cop_profiles.py index 5d36cd5bf..7128ec0d7 100644 --- a/scripts/build_cop_profiles.py +++ b/scripts/build_cop_profiles.py @@ -39,7 +39,7 @@ def coefficient_of_performance(delta_T, source="air"): for source in ["air", "soil"]: source_T = xr.open_dataarray(snakemake.input[f"temp_{source}_{area}"]) - delta_T = snakemake.config["sector"]["heat_pump_sink_T"] - source_T + delta_T = snakemake.params["sector"]["heat_pump_sink_T"] - source_T cop = coefficient_of_performance(delta_T, source) diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index 365797d2b..2f61f0174 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -106,9 +106,9 @@ snakemake = mock_snakemake("build_cutout", cutout="europe-2013-era5") configure_logging(snakemake) - cutout_params = snakemake.config["atlite"]["cutouts"][snakemake.wildcards.cutout] + cutout_params = snakemake.params["atlite"]["cutouts"][snakemake.wildcards.cutout] - snapshots = pd.date_range(freq="h", **snakemake.config["snapshots"]) + snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"]) time = [snapshots[0], snapshots[-1]] cutout_params["time"] = slice(*cutout_params.get("time", time)) diff --git a/scripts/build_electricity_demand.py b/scripts/build_electricity_demand.py index b86b4a5fa..4ef56d1df 100755 --- a/scripts/build_electricity_demand.py +++ b/scripts/build_electricity_demand.py @@ -279,16 +279,16 @@ def manual_adjustment(load, fn_load, powerstatistics): configure_logging(snakemake) - powerstatistics = snakemake.config["load"]["power_statistics"] - interpolate_limit = snakemake.config["load"]["interpolate_limit"] - countries = snakemake.config["countries"] - snapshots = pd.date_range(freq="h", **snakemake.config["snapshots"]) + powerstatistics = snakemake.params["load"]["power_statistics"] + interpolate_limit = snakemake.params["load"]["interpolate_limit"] + countries = snakemake.params["countries"] + snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"]) years = slice(snapshots[0], snapshots[-1]) - time_shift = snakemake.config["load"]["time_shift_for_large_gaps"] + time_shift = snakemake.params["load"]["time_shift_for_large_gaps"] load = load_timeseries(snakemake.input[0], years, countries, powerstatistics) - if snakemake.config["load"]["manual_adjustments"]: + if snakemake.params["load"]["manual_adjustments"]: load = manual_adjustment(load, snakemake.input[0], powerstatistics) logger.info(f"Linearly interpolate gaps of size {interpolate_limit} and less.") diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 45fc960f7..6cedff977 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -373,7 +373,7 @@ def idees_per_country(ct, year, base_dir): def build_idees(countries, year): nprocesses = snakemake.threads - disable_progress = snakemake.config["run"].get("disable_progressbar", False) + disable_progress = snakemake.params["run"].get("disable_progressbar", False) func = partial(idees_per_country, year=year, base_dir=snakemake.input.idees) tqdm_kwargs = dict( @@ -735,18 +735,18 @@ def build_transport_data(countries, population, idees): snakemake = mock_snakemake("build_energy_totals") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) - config = snakemake.config["energy"] + config = snakemake.params["energy"] nuts3 = gpd.read_file(snakemake.input.nuts3_shapes).set_index("index") population = nuts3["pop"].groupby(nuts3.country).sum() - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] idees_countries = pd.Index(countries).intersection(eu28) data_year = config["energy_totals_year"] - report_year = snakemake.config["energy"]["eurostat_report_year"] + report_year = snakemake.params["energy"]["eurostat_report_year"] input_eurostat = snakemake.input.eurostat eurostat = build_eurostat(input_eurostat, countries, report_year, data_year) swiss = build_swiss(data_year) @@ -756,7 +756,7 @@ def build_transport_data(countries, population, idees): energy.to_csv(snakemake.output.energy_name) base_year_emissions = config["base_emissions_year"] - emissions_scope = snakemake.config["energy"]["emissions"] + emissions_scope = snakemake.params["energy"]["emissions"] eea_co2 = build_eea_co2(snakemake.input.co2, base_year_emissions, emissions_scope) eurostat_co2 = build_eurostat_co2( input_eurostat, countries, report_year, base_year_emissions diff --git a/scripts/build_gas_input_locations.py b/scripts/build_gas_input_locations.py index a3b945abc..59f31f5cf 100644 --- a/scripts/build_gas_input_locations.py +++ b/scripts/build_gas_input_locations.py @@ -86,7 +86,7 @@ def build_gas_input_locations(lng_fn, entry_fn, prod_fn, countries): clusters="37", ) - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) regions = load_bus_regions( snakemake.input.regions_onshore, snakemake.input.regions_offshore diff --git a/scripts/build_gas_network.py b/scripts/build_gas_network.py index 23f58caa4..a70f15ff4 100644 --- a/scripts/build_gas_network.py +++ b/scripts/build_gas_network.py @@ -147,7 +147,7 @@ def prepare_dataset( snakemake = mock_snakemake("build_gas_network") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) gas_network = load_dataset(snakemake.input.gas_network) diff --git a/scripts/build_heat_demand.py b/scripts/build_heat_demand.py index 56ceb4b74..655df28f4 100644 --- a/scripts/build_heat_demand.py +++ b/scripts/build_heat_demand.py @@ -27,7 +27,7 @@ cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) client = Client(cluster, asynchronous=True) - time = pd.date_range(freq="h", **snakemake.config["snapshots"]) + time = pd.date_range(freq="h", **snakemake.params["snapshots"]) cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) clustered_regions = ( diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 0e8cfa270..5453ac5cc 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -130,10 +130,10 @@ def get_eia_annual_hydro_generation(fn, countries): snakemake = mock_snakemake("build_hydro_profile") configure_logging(snakemake) - config_hydro = snakemake.config["renewable"]["hydro"] + config_hydro = snakemake.params["renewable"]["hydro"] cutout = atlite.Cutout(snakemake.input.cutout) - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] country_shapes = ( gpd.read_file(snakemake.input.country_shapes) .set_index("name")["geometry"] diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index 69daf64d5..30c652d88 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -73,7 +73,7 @@ def prepare_hotmaps_database(regions): df[["srid", "coordinates"]] = df.geom.str.split(";", expand=True) - if snakemake.config["industry"].get("hotmaps_locate_missing", False): + if snakemake.params["industry"].get("hotmaps_locate_missing", False): df = locate_missing_industrial_sites(df) # remove those sites without valid locations @@ -141,9 +141,9 @@ def build_nodal_distribution_key(hotmaps, regions, countries): clusters=48, ) - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] regions = gpd.read_file(snakemake.input.regions_onshore).set_index("name") diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index 703997b1e..4fec95cc0 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -153,7 +153,7 @@ def add_non_eu28_industrial_energy_demand(countries, demand): def industrial_energy_demand(countries, year): nprocesses = snakemake.threads - disable_progress = snakemake.config["run"].get("disable_progressbar", False) + disable_progress = snakemake.params["run"].get("disable_progressbar", False) func = partial( industrial_energy_demand_per_country, year=year, jrc_dir=snakemake.input.jrc ) @@ -178,9 +178,9 @@ def industrial_energy_demand(countries, year): snakemake = mock_snakemake("build_industrial_energy_demand_per_country_today") - config = snakemake.config["industry"] + config = snakemake.params["industry"] year = config.get("reference_year", 2015) - countries = pd.Index(snakemake.config["countries"]) + countries = pd.Index(snakemake.params["countries"]) demand = industrial_energy_demand(countries.intersection(eu28), year) diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index 437806b35..eb1b16cb0 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -217,7 +217,7 @@ def get_sector_data(sector, country): def industry_production(countries, year, eurostat_dir, jrc_dir): nprocesses = snakemake.threads - disable_progress = snakemake.config["run"].get("disable_progressbar", False) + disable_progress = snakemake.params["run"].get("disable_progressbar", False) func = partial( industry_production_per_country, @@ -277,13 +277,13 @@ def separate_basic_chemicals(demand, year): snakemake = mock_snakemake("build_industrial_production_per_country") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] - year = snakemake.config["industry"]["reference_year"] + year = snakemake.params["industry"]["reference_year"] - config = snakemake.config["industry"] + config = snakemake.params["industry"] jrc_dir = snakemake.input.jrc eurostat_dir = snakemake.input.eurostat diff --git a/scripts/build_industrial_production_per_country_tomorrow.py b/scripts/build_industrial_production_per_country_tomorrow.py index 6c445608b..b9ac9b168 100644 --- a/scripts/build_industrial_production_per_country_tomorrow.py +++ b/scripts/build_industrial_production_per_country_tomorrow.py @@ -15,7 +15,7 @@ snakemake = mock_snakemake("build_industrial_production_per_country_tomorrow") - config = snakemake.config["industry"] + config = snakemake.params["industry"] investment_year = int(snakemake.wildcards.planning_horizons) diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py index 54f2cfdc3..c3e2bd49e 100644 --- a/scripts/build_industry_sector_ratios.py +++ b/scripts/build_industry_sector_ratios.py @@ -439,7 +439,7 @@ def chemicals_industry(): sector = "Ammonia" df[sector] = 0.0 - if snakemake.config["sector"].get("ammonia", False): + if snakemake.params["sector"].get("ammonia", False): df.loc["ammonia", sector] = config["MWh_NH3_per_tNH3"] else: df.loc["hydrogen", sector] = config["MWh_H2_per_tNH3_electrolysis"] @@ -1468,7 +1468,7 @@ def other_industrial_sectors(): # TODO make config option year = 2015 - config = snakemake.config["industry"] + config = snakemake.params["industry"] df = pd.concat( [ diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index e864d9259..bc35dbcfe 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -23,7 +23,7 @@ snakemake = mock_snakemake("build_population_layouts") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) cutout = atlite.Cutout(snakemake.input.cutout) diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index 9ca67a53f..bd0ee74e9 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -115,7 +115,7 @@ def replace_natural_gas_fueltype(df): configure_logging(snakemake) n = pypsa.Network(snakemake.input.base_network) - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] ppl = ( pm.powerplants(from_url=True) @@ -134,12 +134,12 @@ def replace_natural_gas_fueltype(df): ppl = ppl.query('not (Country in @available_countries and Fueltype == "Bioenergy")') ppl = pd.concat([ppl, opsd]) - ppl_query = snakemake.config["electricity"]["powerplants_filter"] + ppl_query = snakemake.params["electricity"]["powerplants_filter"] if isinstance(ppl_query, str): ppl.query(ppl_query, inplace=True) # add carriers from own powerplant files: - custom_ppl_query = snakemake.config["electricity"]["custom_powerplants"] + custom_ppl_query = snakemake.params["electricity"]["custom_powerplants"] ppl = add_custom_powerplants( ppl, snakemake.input.custom_powerplants, custom_ppl_query ) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index c0288aeee..55541522b 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -203,8 +203,8 @@ configure_logging(snakemake) nprocesses = int(snakemake.threads) - noprogress = snakemake.config["run"].get("disable_progressbar", True) - config = snakemake.config["renewable"][snakemake.wildcards.technology] + noprogress = snakemake.params["run"].get("disable_progressbar", True) + config = snakemake.params["renewable"][snakemake.wildcards.technology] resource = config["resource"] # pv panel config / wind turbine config correction_factor = config.get("correction_factor", 1.0) capacity_per_sqkm = config["capacity_per_sqkm"] diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py index 9dbfc3755..c52d4eb6d 100644 --- a/scripts/build_retro_cost.py +++ b/scripts/build_retro_cost.py @@ -305,7 +305,7 @@ def prepare_building_stock_data(): u_values.set_index(["country_code", "subsector", "bage", "type"], inplace=True) # only take in config.yaml specified countries into account - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] area_tot = area_tot.loc[countries] return u_values, country_iso_dic, countries, area_tot, area @@ -1040,7 +1040,7 @@ def sample_dE_costs_area( # ******** config ********************************************************* - retro_opts = snakemake.config["sector"]["retrofitting"] + retro_opts = snakemake.params["sector"]["retrofitting"] interest_rate = retro_opts["interest_rate"] annualise_cost = retro_opts["annualise_cost"] # annualise the investment costs tax_weighting = retro_opts[ diff --git a/scripts/build_sequestration_potentials.py b/scripts/build_sequestration_potentials.py index 012effe86..5c388b2e8 100644 --- a/scripts/build_sequestration_potentials.py +++ b/scripts/build_sequestration_potentials.py @@ -41,7 +41,7 @@ def allocate_sequestration_potential( "build_sequestration_potentials", simpl="", clusters="181" ) - cf = snakemake.config["sector"]["regional_co2_sequestration_potential"] + cf = snakemake.params["sector"]["regional_co2_sequestration_potential"] gdf = gpd.read_file(snakemake.input.sequestration_potential[0]) diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 50d21e12e..f529517c5 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -255,12 +255,12 @@ def nuts3(country_shapes, nuts3, nuts3pop, nuts3gdp, ch_cantons, ch_popgdp): configure_logging(snakemake) country_shapes = countries( - snakemake.input.naturalearth, snakemake.config["countries"] + snakemake.input.naturalearth, snakemake.params["countries"] ) country_shapes.reset_index().to_file(snakemake.output.country_shapes) offshore_shapes = eez( - country_shapes, snakemake.input.eez, snakemake.config["countries"] + country_shapes, snakemake.input.eez, snakemake.params["countries"] ) offshore_shapes.reset_index().to_file(snakemake.output.offshore_shapes) diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py index f4eb15575..180007b74 100644 --- a/scripts/build_solar_thermal_profiles.py +++ b/scripts/build_solar_thermal_profiles.py @@ -27,9 +27,9 @@ cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) client = Client(cluster, asynchronous=True) - config = snakemake.config["solar_thermal"] + config = snakemake.params["solar_thermal"] - time = pd.date_range(freq="h", **snakemake.config["snapshots"]) + time = pd.date_range(freq="h", **snakemake.params["snapshots"]) cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) clustered_regions = ( diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py index 8f6d6c6c9..ee06aebb4 100644 --- a/scripts/build_temperature_profiles.py +++ b/scripts/build_temperature_profiles.py @@ -27,7 +27,7 @@ cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) client = Client(cluster, asynchronous=True) - time = pd.date_range(freq="h", **snakemake.config["snapshots"]) + time = pd.date_range(freq="h", **snakemake.params["snapshots"]) cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) clustered_regions = ( diff --git a/scripts/build_transport_demand.py b/scripts/build_transport_demand.py index 6b8bd04f9..dc2b94b9d 100644 --- a/scripts/build_transport_demand.py +++ b/scripts/build_transport_demand.py @@ -175,9 +175,9 @@ def bev_dsm_profile(snapshots, nodes, options): snakemake.input.pop_weighted_energy_totals, index_col=0 ) - options = snakemake.config["sector"] + options = snakemake.params["sector"] - snapshots = pd.date_range(freq="h", **snakemake.config["snapshots"], tz="UTC") + snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"], tz="UTC") nyears = len(snapshots) / 8760 diff --git a/scripts/cluster_gas_network.py b/scripts/cluster_gas_network.py index e7554dff3..ba11ce1b2 100755 --- a/scripts/cluster_gas_network.py +++ b/scripts/cluster_gas_network.py @@ -110,7 +110,7 @@ def aggregate_parallel_pipes(df): snakemake = mock_snakemake("cluster_gas_network", simpl="", clusters="37") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) fn = snakemake.input.cleaned_gas_network df = pd.read_csv(fn, index_col=0) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 7572d3b3e..78798804f 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -468,18 +468,18 @@ def plot_busmap_for_n_clusters(n, n_clusters, fn=None): [ tech for tech in n.generators.carrier.unique() - if tech in snakemake.config["renewable"] + if tech in snakemake.params["renewable"] ] ) - exclude_carriers = snakemake.config["clustering"]["cluster_network"].get( + exclude_carriers = snakemake.params["clustering"]["cluster_network"].get( "exclude_carriers", [] ) aggregate_carriers = set(n.generators.carrier) - set(exclude_carriers) if snakemake.wildcards.clusters.endswith("m"): n_clusters = int(snakemake.wildcards.clusters[:-1]) conventional = set( - snakemake.config["electricity"].get("conventional_carriers", []) + snakemake.params["electricity"].get("conventional_carriers", []) ) aggregate_carriers = conventional.intersection(aggregate_carriers) elif snakemake.wildcards.clusters == "all": @@ -495,13 +495,13 @@ def plot_busmap_for_n_clusters(n, n_clusters, fn=None): n, busmap, linemap, linemap, pd.Series(dtype="O") ) else: - line_length_factor = snakemake.config["lines"]["length_factor"] + line_length_factor = snakemake.params["lines"]["length_factor"] Nyears = n.snapshot_weightings.objective.sum() / 8760 hvac_overhead_cost = load_costs( snakemake.input.tech_costs, - snakemake.config["costs"], - snakemake.config["electricity"], + snakemake.params["costs"], + snakemake.params["electricity"], Nyears, ).at["HVAC overhead", "capital_cost"] @@ -512,7 +512,7 @@ def consense(x): ).all() or x.isnull().all(), "The `potential` configuration option must agree for all renewable carriers, for now!" return v - aggregation_strategies = snakemake.config["clustering"].get( + aggregation_strategies = snakemake.params["clustering"].get( "aggregation_strategies", {} ) # translate str entries of aggregation_strategies to pd.Series functions: @@ -521,7 +521,7 @@ def consense(x): for p in aggregation_strategies.keys() } - custom_busmap = snakemake.config["enable"].get("custom_busmap", False) + custom_busmap = snakemake.params["enable"].get("custom_busmap", False) if custom_busmap: custom_busmap = pd.read_csv( snakemake.input.custom_busmap, index_col=0, squeeze=True @@ -539,7 +539,7 @@ def consense(x): aggregate_carriers, line_length_factor, aggregation_strategies, - snakemake.config["solving"]["solver"]["name"], + snakemake.params["solving"]["solver"]["name"], cluster_config.get("algorithm", "hac"), cluster_config.get("feature", "solar+onwind-time"), hvac_overhead_cost, diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 3d7439428..560745252 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -198,7 +198,7 @@ def calculate_costs(n, label, costs): def calculate_cumulative_cost(): - planning_horizons = snakemake.config["scenario"]["planning_horizons"] + planning_horizons = snakemake.params["scenario"]["planning_horizons"] cumulative_cost = pd.DataFrame( index=df["costs"].sum().index, @@ -682,25 +682,25 @@ def to_csv(df): snakemake = mock_snakemake("make_summary") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) networks_dict = { (cluster, ll, opt + sector_opt, planning_horizon): "results/" + snakemake.params.RDIR + f"/postnetworks/elec_s{simpl}_{cluster}_l{ll}_{opt}_{sector_opt}_{planning_horizon}.nc" - for simpl in snakemake.config["scenario"]["simpl"] - for cluster in snakemake.config["scenario"]["clusters"] - for opt in snakemake.config["scenario"]["opts"] - for sector_opt in snakemake.config["scenario"]["sector_opts"] - for ll in snakemake.config["scenario"]["ll"] - for planning_horizon in snakemake.config["scenario"]["planning_horizons"] + for simpl in snakemake.params["scenario"]["simpl"] + for cluster in snakemake.params["scenario"]["clusters"] + for opt in snakemake.params["scenario"]["opts"] + for sector_opt in snakemake.params["scenario"]["sector_opts"] + for ll in snakemake.params["scenario"]["ll"] + for planning_horizon in snakemake.params["scenario"]["planning_horizons"] } - Nyears = len(pd.date_range(freq="h", **snakemake.config["snapshots"])) / 8760 + Nyears = len(pd.date_range(freq="h", **snakemake.params["snapshots"])) / 8760 costs_db = prepare_costs( snakemake.input.costs, - snakemake.config["costs"], + snakemake.params["costs"], Nyears, ) @@ -710,7 +710,7 @@ def to_csv(df): to_csv(df) - if snakemake.config["foresight"] == "myopic": + if snakemake.params["foresight"] == "myopic": cumulative_cost = calculate_cumulative_cost() cumulative_cost.to_csv( "results/" + snakemake.params.RDIR + "/csvs/cumulative_cost.csv" diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 0a22b2e57..8aac9db1f 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -70,7 +70,7 @@ def plot_map( transmission=False, with_legend=True, ): - tech_colors = snakemake.config["plotting"]["tech_colors"] + tech_colors = snakemake.params["plotting"]["tech_colors"] n = network.copy() assign_location(n) @@ -116,7 +116,7 @@ def plot_map( costs = costs.stack() # .sort_index() # hack because impossible to drop buses... - eu_location = snakemake.config["plotting"].get( + eu_location = snakemake.params["plotting"].get( "eu_node_location", dict(x=-5.5, y=46) ) n.buses.loc["EU gas", "x"] = eu_location["x"] @@ -315,7 +315,7 @@ def plot_h2_map(network, regions): h2_new = n.links[n.links.carrier == "H2 pipeline"] h2_retro = n.links[n.links.carrier == "H2 pipeline retrofitted"] - if snakemake.config["foresight"] == "myopic": + if snakemake.params["foresight"] == "myopic": # sum capacitiy for pipelines from different investment periods h2_new = group_pipes(h2_new) @@ -558,7 +558,7 @@ def plot_ch4_map(network): link_widths_used = max_usage / linewidth_factor link_widths_used[max_usage < line_lower_threshold] = 0.0 - tech_colors = snakemake.config["plotting"]["tech_colors"] + tech_colors = snakemake.params["plotting"]["tech_colors"] pipe_colors = { "gas pipeline": "#f08080", @@ -700,7 +700,7 @@ def plot_map_without(network): # hack because impossible to drop buses... if "EU gas" in n.buses.index: - eu_location = snakemake.config["plotting"].get( + eu_location = snakemake.params["plotting"].get( "eu_node_location", dict(x=-5.5, y=46) ) n.buses.loc["EU gas", "x"] = eu_location["x"] @@ -876,7 +876,7 @@ def plot_series(network, carrier="AC", name="test"): stacked=True, linewidth=0.0, color=[ - snakemake.config["plotting"]["tech_colors"][i.replace(suffix, "")] + snakemake.params["plotting"]["tech_colors"][i.replace(suffix, "")] for i in new_columns ], ) @@ -930,14 +930,14 @@ def plot_series(network, carrier="AC", name="test"): planning_horizons="2030", ) - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) overrides = override_component_attrs(snakemake.input.overrides) n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) regions = gpd.read_file(snakemake.input.regions).set_index("name") - map_opts = snakemake.config["plotting"]["map"] + map_opts = snakemake.params["plotting"]["map"] if map_opts["boundaries"] is None: map_opts["boundaries"] = regions.total_bounds[[0, 2, 1, 3]] + [-1, 1, -1, 1] diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index cfa8e3617..e18ff61b5 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -142,10 +142,10 @@ def plot_costs(): df = df.groupby(df.index.map(rename_techs)).sum() - to_drop = df.index[df.max(axis=1) < snakemake.config["plotting"]["costs_threshold"]] + to_drop = df.index[df.max(axis=1) < snakemake.params["plotting"]["costs_threshold"]] logger.info( - f"Dropping technology with costs below {snakemake.config['plotting']['costs_threshold']} EUR billion per year" + f"Dropping technology with costs below {snakemake.params['plotting']['costs_threshold']} EUR billion per year" ) logger.debug(df.loc[to_drop]) @@ -165,7 +165,7 @@ def plot_costs(): kind="bar", ax=ax, stacked=True, - color=[snakemake.config["plotting"]["tech_colors"][i] for i in new_index], + color=[snakemake.params["plotting"]["tech_colors"][i] for i in new_index], ) handles, labels = ax.get_legend_handles_labels() @@ -173,7 +173,7 @@ def plot_costs(): handles.reverse() labels.reverse() - ax.set_ylim([0, snakemake.config["plotting"]["costs_max"]]) + ax.set_ylim([0, snakemake.params["plotting"]["costs_max"]]) ax.set_ylabel("System Cost [EUR billion per year]") @@ -201,11 +201,11 @@ def plot_energy(): df = df.groupby(df.index.map(rename_techs)).sum() to_drop = df.index[ - df.abs().max(axis=1) < snakemake.config["plotting"]["energy_threshold"] + df.abs().max(axis=1) < snakemake.params["plotting"]["energy_threshold"] ] logger.info( - f"Dropping all technology with energy consumption or production below {snakemake.config['plotting']['energy_threshold']} TWh/a" + f"Dropping all technology with energy consumption or production below {snakemake.params['plotting']['energy_threshold']} TWh/a" ) logger.debug(df.loc[to_drop]) @@ -227,7 +227,7 @@ def plot_energy(): kind="bar", ax=ax, stacked=True, - color=[snakemake.config["plotting"]["tech_colors"][i] for i in new_index], + color=[snakemake.params["plotting"]["tech_colors"][i] for i in new_index], ) handles, labels = ax.get_legend_handles_labels() @@ -237,8 +237,8 @@ def plot_energy(): ax.set_ylim( [ - snakemake.config["plotting"]["energy_min"], - snakemake.config["plotting"]["energy_max"], + snakemake.params["plotting"]["energy_min"], + snakemake.params["plotting"]["energy_max"], ] ) @@ -287,7 +287,7 @@ def plot_balances(): df = df.groupby(df.index.map(rename_techs)).sum() to_drop = df.index[ - df.abs().max(axis=1) < snakemake.config["plotting"]["energy_threshold"] / 10 + df.abs().max(axis=1) < snakemake.params["plotting"]["energy_threshold"] / 10 ] if v[0] in co2_carriers: @@ -296,7 +296,7 @@ def plot_balances(): units = "TWh/a" logger.debug( - f"Dropping technology energy balance smaller than {snakemake.config['plotting']['energy_threshold']/10} {units}" + f"Dropping technology energy balance smaller than {snakemake.params['plotting']['energy_threshold']/10} {units}" ) logger.debug(df.loc[to_drop]) @@ -317,7 +317,7 @@ def plot_balances(): kind="bar", ax=ax, stacked=True, - color=[snakemake.config["plotting"]["tech_colors"][i] for i in new_index], + color=[snakemake.params["plotting"]["tech_colors"][i] for i in new_index], ) handles, labels = ax.get_legend_handles_labels() @@ -455,10 +455,10 @@ def plot_carbon_budget_distribution(input_eurostat): ax1 = plt.subplot(gs1[0, 0]) ax1.set_ylabel("CO$_2$ emissions (Gt per year)", fontsize=22) ax1.set_ylim([0, 5]) - ax1.set_xlim([1990, snakemake.config["scenario"]["planning_horizons"][-1] + 1]) + ax1.set_xlim([1990, snakemake.params["scenario"]["planning_horizons"][-1] + 1]) path_cb = "results/" + snakemake.params.RDIR + "/csvs/" - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] e_1990 = co2_emissions_year(countries, input_eurostat, opts, year=1990) CO2_CAP = pd.read_csv(path_cb + "carbon_budget_distribution.csv", index_col=0) @@ -545,7 +545,7 @@ def plot_carbon_budget_distribution(input_eurostat): snakemake = mock_snakemake("plot_summary") - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) n_header = 4 @@ -555,7 +555,7 @@ def plot_carbon_budget_distribution(input_eurostat): plot_balances() - for sector_opts in snakemake.config["scenario"]["sector_opts"]: + for sector_opts in snakemake.params["scenario"]["sector_opts"]: opts = sector_opts.split("-") for o in opts: if "cb" in o: diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 14a003aeb..46e4e74aa 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -253,12 +253,12 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): Nyears = n.snapshot_weightings.objective.sum() / 8760.0 costs = load_costs( snakemake.input.tech_costs, - snakemake.config["costs"], - snakemake.config["electricity"], + snakemake.params["costs"], + snakemake.params["electricity"], Nyears, ) - set_line_s_max_pu(n, snakemake.config["lines"]["s_max_pu"]) + set_line_s_max_pu(n, snakemake.params["lines"]["s_max_pu"]) for o in opts: m = re.match(r"^\d+h$", o, re.IGNORECASE) @@ -269,7 +269,7 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): for o in opts: m = re.match(r"^\d+seg$", o, re.IGNORECASE) if m is not None: - solver_name = snakemake.config["solving"]["solver"]["name"] + solver_name = snakemake.params["solving"]["solver"]["name"] n = apply_time_segmentation(n, m.group(0)[:-3], solver_name) break @@ -277,11 +277,11 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): if "Co2L" in o: m = re.findall("[0-9]*\.?[0-9]+$", o) if len(m) > 0: - co2limit = float(m[0]) * snakemake.config["electricity"]["co2base"] + co2limit = float(m[0]) * snakemake.params["electricity"]["co2base"] add_co2limit(n, co2limit, Nyears) logger.info("Setting CO2 limit according to wildcard value.") else: - add_co2limit(n, snakemake.config["electricity"]["co2limit"], Nyears) + add_co2limit(n, snakemake.params["electricity"]["co2limit"], Nyears) logger.info("Setting CO2 limit according to config value.") break @@ -293,7 +293,7 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): add_gaslimit(n, limit, Nyears) logger.info("Setting gas usage limit according to wildcard value.") else: - add_gaslimit(n, snakemake.config["electricity"].get("gaslimit"), Nyears) + add_gaslimit(n, snakemake.params["electricity"].get("gaslimit"), Nyears) logger.info("Setting gas usage limit according to config value.") break @@ -322,7 +322,7 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): add_emission_prices(n, dict(co2=float(m[0]))) else: logger.info("Setting emission prices according to config value.") - add_emission_prices(n, snakemake.config["costs"]["emission_prices"]) + add_emission_prices(n, snakemake.params["costs"]["emission_prices"]) break ll_type, factor = snakemake.wildcards.ll[0], snakemake.wildcards.ll[1:] @@ -330,8 +330,8 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): set_line_nom_max( n, - s_nom_max_set=snakemake.config["lines"].get("s_nom_max,", np.inf), - p_nom_max_set=snakemake.config["links"].get("p_nom_max,", np.inf), + s_nom_max_set=snakemake.params["lines"].get("s_nom_max,", np.inf), + p_nom_max_set=snakemake.params["links"].get("p_nom_max,", np.inf), ) if "ATK" in opts: diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 1cb7146e4..c69f72909 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -200,12 +200,12 @@ def co2_emissions_year( """ Calculate CO2 emissions in one specific year (e.g. 1990 or 2018). """ - emissions_scope = snakemake.config["energy"]["emissions"] + emissions_scope = snakemake.params["energy"]["emissions"] eea_co2 = build_eea_co2(snakemake.input.co2, year, emissions_scope) # TODO: read Eurostat data from year > 2014 # this only affects the estimation of CO2 emissions for BA, RS, AL, ME, MK - report_year = snakemake.config["energy"]["eurostat_report_year"] + report_year = snakemake.params["energy"]["eurostat_report_year"] if year > 2014: eurostat_co2 = build_eurostat_co2( input_eurostat, countries, report_year, year=2014 @@ -241,7 +241,7 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year): carbon_budget = float(o[o.find("cb") + 2 : o.find("ex")]) r = float(o[o.find("ex") + 2 :]) - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] e_1990 = co2_emissions_year( countries, input_eurostat, opts, emissions_scope, report_year, year=1990 @@ -252,7 +252,7 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year): countries, input_eurostat, opts, emissions_scope, report_year, year=2018 ) - planning_horizons = snakemake.config["scenario"]["planning_horizons"] + planning_horizons = snakemake.params["scenario"]["planning_horizons"] t_0 = planning_horizons[0] if "be" in o: @@ -391,7 +391,7 @@ def update_wind_solar_costs(n, costs): with xr.open_dataset(profile) as ds: underwater_fraction = ds["underwater_fraction"].to_pandas() connection_cost = ( - snakemake.config["lines"]["length_factor"] + snakemake.params["lines"]["length_factor"] * ds["average_distance"].to_pandas() * ( underwater_fraction @@ -483,8 +483,8 @@ def remove_elec_base_techs(n): batteries and H2) from base electricity-only network, since they're added here differently using links. """ - for c in n.iterate_components(snakemake.config["pypsa_eur"]): - to_keep = snakemake.config["pypsa_eur"][c.name] + for c in n.iterate_components(snakemake.params["pypsa_eur"]): + to_keep = snakemake.params["pypsa_eur"][c.name] to_remove = pd.Index(c.df.carrier.unique()).symmetric_difference(to_keep) if to_remove.empty: continue @@ -674,7 +674,7 @@ def add_dac(n, costs): def add_co2limit(n, nyears=1.0, limit=0.0): logger.info(f"Adding CO2 budget limit as per unit of 1990 levels of {limit}") - countries = snakemake.config["countries"] + countries = snakemake.params["countries"] sectors = emission_sectors_from_opts(opts) @@ -787,7 +787,7 @@ def add_ammonia(n, costs): nodes = pop_layout.index - cf_industry = snakemake.config["industry"] + cf_industry = snakemake.params["industry"] n.add("Carrier", "NH3") @@ -1102,7 +1102,7 @@ def add_storage_and_grids(n, costs): lifetime=costs.at["OCGT", "lifetime"], ) - cavern_types = snakemake.config["sector"]["hydrogen_underground_storage_locations"] + cavern_types = snakemake.params["sector"]["hydrogen_underground_storage_locations"] h2_caverns = pd.read_csv(snakemake.input.h2_cavern, index_col=0) if not h2_caverns.empty and options["hydrogen_underground_storage"]: @@ -3266,11 +3266,11 @@ def set_temporal_aggregation(n, opts, solver_name): planning_horizons="2030", ) - logging.basicConfig(level=snakemake.config["logging"]["level"]) + logging.basicConfig(level=snakemake.params["logging"]["level"]) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) - options = snakemake.config["sector"] + options = snakemake.params["sector"] opts = snakemake.wildcards.sector_opts.split("-") @@ -3285,7 +3285,7 @@ def set_temporal_aggregation(n, opts, solver_name): costs = prepare_costs( snakemake.input.costs, - snakemake.config["costs"], + snakemake.params["costs"], nyears, ) @@ -3297,10 +3297,10 @@ def set_temporal_aggregation(n, opts, solver_name): spatial = define_spatial(pop_layout.index, options) - if snakemake.config["foresight"] == "myopic": + if snakemake.params["foresight"] == "myopic": add_lifetime_wind_solar(n, costs) - conventional = snakemake.config["existing_capacities"]["conventional_carriers"] + conventional = snakemake.params["existing_capacities"]["conventional_carriers"] for carrier in conventional: add_carrier_buses(n, carrier) @@ -3365,19 +3365,19 @@ def set_temporal_aggregation(n, opts, solver_name): if options["allam_cycle"]: add_allam(n, costs) - solver_name = snakemake.config["solving"]["solver"]["name"] + solver_name = snakemake.params["solving"]["solver"]["name"] n = set_temporal_aggregation(n, opts, solver_name) limit_type = "config" - limit = get(snakemake.config["co2_budget"], investment_year) + limit = get(snakemake.params["co2_budget"], investment_year) for o in opts: if "cb" not in o: continue limit_type = "carbon budget" fn = "results/" + snakemake.params.RDIR + "/csvs/carbon_budget_distribution.csv" if not os.path.exists(fn): - emissions_scope = snakemake.config["energy"]["emissions"] - report_year = snakemake.config["energy"]["eurostat_report_year"] + emissions_scope = snakemake.params["energy"]["emissions"] + report_year = snakemake.params["energy"]["eurostat_report_year"] build_carbon_budget( o, snakemake.input.eurostat, fn, emissions_scope, report_year ) @@ -3412,8 +3412,8 @@ def set_temporal_aggregation(n, opts, solver_name): if options["electricity_grid_connection"]: add_electricity_grid_connection(n, costs) - first_year_myopic = (snakemake.config["foresight"] == "myopic") and ( - snakemake.config["scenario"]["planning_horizons"][0] == investment_year + first_year_myopic = (snakemake.params["foresight"] == "myopic") and ( + snakemake.params["scenario"]["planning_horizons"][0] == investment_year ) if options.get("cluster_heat_buses", False) and not first_year_myopic: diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py index 0c6a7feb5..0271166c7 100644 --- a/scripts/retrieve_databundle.py +++ b/scripts/retrieve_databundle.py @@ -53,7 +53,7 @@ snakemake ) # TODO Make logging compatible with progressbar (see PR #102) - if snakemake.config["tutorial"]: + if snakemake.params["tutorial"]: url = "https://zenodo.org/record/3517921/files/pypsa-eur-tutorial-data-bundle.tar.xz" else: url = "https://zenodo.org/record/3517935/files/pypsa-eur-data-bundle.tar.xz" @@ -63,7 +63,7 @@ to_fn = Path(f"{rootpath}/data") logger.info(f"Downloading databundle from '{url}'.") - disable_progress = snakemake.config["run"].get("disable_progressbar", False) + disable_progress = snakemake.params["run"].get("disable_progressbar", False) progress_retrieve(url, tarball_fn, disable=disable_progress) logger.info("Extracting databundle.") diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 2be8c36a9..bb7f615cc 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -510,7 +510,7 @@ def cluster( n = pypsa.Network(snakemake.input.network) - aggregation_strategies = snakemake.config["clustering"].get( + aggregation_strategies = snakemake.params["clustering"].get( "aggregation_strategies", {} ) # translate str entries of aggregation_strategies to pd.Series functions: @@ -525,8 +525,8 @@ def cluster( technology_costs = load_costs( snakemake.input.tech_costs, - snakemake.config["costs"], - snakemake.config["electricity"], + snakemake.params["costs"], + snakemake.params["electricity"], Nyears, ) @@ -536,7 +536,7 @@ def cluster( busmaps = [trafo_map, simplify_links_map] - cluster_config = snakemake.config["clustering"]["simplify_network"] + cluster_config = snakemake.params["clustering"]["simplify_network"] if cluster_config.get("remove_stubs", True): n, stub_map = remove_stubs( n, diff --git a/scripts/solve_network.py b/scripts/solve_network.py index ff1c0ccf0..c7041e85b 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -663,7 +663,7 @@ def solve_network(n, config, opts="", **kwargs): if "sector_opts" in snakemake.wildcards.keys(): opts += "-" + snakemake.wildcards.sector_opts opts = [o for o in opts.split("-") if o != ""] - solve_opts = snakemake.config["solving"]["options"] + solve_opts = snakemake.params["solving"]["options"] np.random.seed(solve_opts.get("seed", 123)) diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index 25fe0753a..c1b2be6f4 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -42,7 +42,7 @@ opts = (snakemake.wildcards.opts + "-" + snakemake.wildcards.sector_opts).split("-") opts = [o for o in opts if o != ""] - solve_opts = snakemake.config["solving"]["options"] + solve_opts = snakemake.params["solving"]["options"] np.random.seed(solve_opts.get("seed", 123)) From 61893c3c9b9e9f738d795f31ab03be07f6558ebd Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Mon, 15 May 2023 11:26:36 +0200 Subject: [PATCH 04/24] fixing build_electricity.smk and retrieve.smk --- rules/build_electricity.smk | 1 + rules/retrieve.smk | 1 + 2 files changed, 2 insertions(+) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index bb000f2b5..56098ddd1 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -201,6 +201,7 @@ rule build_ship_raster: rule build_renewable_profiles: params: + run=config["run"], renewable=config["renewable"], input: base_network=RESOURCES + "networks/base.nc", diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 4bfbd6c65..bc8756d54 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -20,6 +20,7 @@ if config["enable"].get("retrieve_databundle", True): rule retrieve_databundle: params: + run=config["run"], tutorial=config["tutorial"], output: expand("data/bundle/{file}", file=datafiles), From 8208ac033baff982fbeedb11ef019d4bd581576c Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Wed, 17 May 2023 18:43:30 +0200 Subject: [PATCH 05/24] convert param back to config for "logging" and "run", made "enable" more specific --- rules/build_electricity.smk | 3 +-- rules/build_sector.smk | 15 --------------- rules/postprocess.smk | 3 --- rules/retrieve.smk | 1 - rules/solve_myopic.smk | 2 -- scripts/add_brownfield.py | 2 +- scripts/add_existing_baseyear.py | 2 +- scripts/build_energy_totals.py | 4 ++-- scripts/build_gas_input_locations.py | 2 +- scripts/build_gas_network.py | 2 +- scripts/build_industrial_distribution_key.py | 2 +- ..._industrial_energy_demand_per_country_today.py | 2 +- .../build_industrial_production_per_country.py | 4 ++-- scripts/build_population_layouts.py | 2 +- scripts/build_renewable_profiles.py | 2 +- scripts/cluster_gas_network.py | 2 +- scripts/make_summary.py | 2 +- scripts/plot_network.py | 2 +- scripts/plot_summary.py | 2 +- scripts/prepare_sector_network.py | 2 +- scripts/retrieve_databundle.py | 2 +- 21 files changed, 19 insertions(+), 41 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 56098ddd1..9414c25b4 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -201,7 +201,6 @@ rule build_ship_raster: rule build_renewable_profiles: params: - run=config["run"], renewable=config["renewable"], input: base_network=RESOURCES + "networks/base.nc", @@ -350,7 +349,7 @@ rule cluster_network: lines=config["lines"], renewable=config["renewable"], clustering=config["clustering"], - enable=config["enable"], + enable=config["enable"].get("custom_busmap", False), input: network=RESOURCES + "networks/elec_s{simpl}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson", diff --git a/rules/build_sector.smk b/rules/build_sector.smk index d375b7b92..9faae4b9c 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -4,8 +4,6 @@ rule build_population_layouts: - params: - logging=config["logging"], input: nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", urban_percent="data/urban_percent.csv", @@ -72,8 +70,6 @@ rule build_simplified_population_layouts: if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: rule build_gas_network: - params: - logging=config["logging"], input: gas_network="data/gas_network/scigrid-gas/data/IGGIELGN_PipeSegments.geojson", output: @@ -88,8 +84,6 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: "../scripts/build_gas_network.py" rule build_gas_input_locations: - params: - logging=config["logging"], input: lng=HTTP.remote( "https://globalenergymonitor.org/wp-content/uploads/2022/09/Europe-Gas-Tracker-August-2022.xlsx", @@ -116,8 +110,6 @@ if config["sector"]["gas_network"] or config["sector"]["H2_retrofit"]: "../scripts/build_gas_input_locations.py" rule cluster_gas_network: - params: - logging=config["logging"], input: cleaned_gas_network=RESOURCES + "gas_network.csv", regions_onshore=RESOURCES @@ -246,10 +238,8 @@ rule build_solar_thermal_profiles: rule build_energy_totals: params: - run=config["run"], countries=config["countries"], energy=config["energy"], - logging=config["logging"], input: nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", co2="data/eea/UNFCCC_v23.csv", @@ -437,10 +427,8 @@ rule build_industry_sector_ratios: rule build_industrial_production_per_country: params: - run=config["run"], industry=config["industry"], countries=config["countries"], - logging=config["logging"], input: ammonia_production=RESOURCES + "ammonia_production.csv", jrc="data/jrc-idees-2015", @@ -491,7 +479,6 @@ rule build_industrial_distribution_key: params: industry=config["industry"], countries=config["countries"], - logging=config["logging"], input: regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", @@ -567,7 +554,6 @@ rule build_industrial_energy_demand_per_node: rule build_industrial_energy_demand_per_country_today: params: - run=config["run"], countries=config["countries"], industry=config["industry"], input: @@ -723,7 +709,6 @@ rule prepare_sector_network: existing_capacities=config["existing_capacities"], foresight=config["foresight"], costs=config["costs"], - logging=config["logging"], sector=config["sector"], industry=config["industry"], pypsa_eur=config["pypsa_eur"], diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 1dfdd0987..9eb04ef54 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -10,7 +10,6 @@ localrules: rule plot_network: params: - logging=config["logging"], foresight=config["foresight"], plotting=config["plotting"], input: @@ -74,7 +73,6 @@ rule make_summary: foresight=config["foresight"], costs=config["costs"], snapshots=config["snapshots"], - logging=config["logging"], scenario=config["scenario"], RDIR=RDIR, input: @@ -123,7 +121,6 @@ rule make_summary: rule plot_summary: params: - logging=config["logging"], countries=config["countries"], scenario=config["scenario"], plotting=config["plotting"], diff --git a/rules/retrieve.smk b/rules/retrieve.smk index bc8756d54..4bfbd6c65 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -20,7 +20,6 @@ if config["enable"].get("retrieve_databundle", True): rule retrieve_databundle: params: - run=config["run"], tutorial=config["tutorial"], output: expand("data/bundle/{file}", file=datafiles), diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 2ecba9996..49b93a80a 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -7,7 +7,6 @@ rule add_existing_baseyear: params: scenario=config["scenario"], sector=config["sector"], - logging=config["logging"], existing_capacities=config["existing_capacities"], costs=config["costs"], input: @@ -49,7 +48,6 @@ rule add_existing_baseyear: rule add_brownfield: params: - logging=config["logging"], sector=config["sector"], existing_capacities=config["existing_capacities"], input: diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 1d4e3a806..9330953bc 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -139,7 +139,7 @@ def add_brownfield(n, n_p, year): planning_horizons=2030, ) - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index a24b078de..a4d3748b8 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -608,7 +608,7 @@ def add_heating_capacities_installed_before_baseyear( planning_horizons=2020, ) - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 6cedff977..5099d1403 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -373,7 +373,7 @@ def idees_per_country(ct, year, base_dir): def build_idees(countries, year): nprocesses = snakemake.threads - disable_progress = snakemake.params["run"].get("disable_progressbar", False) + disable_progress = snakemake.config["run"].get("disable_progressbar", False) func = partial(idees_per_country, year=year, base_dir=snakemake.input.idees) tqdm_kwargs = dict( @@ -735,7 +735,7 @@ def build_transport_data(countries, population, idees): snakemake = mock_snakemake("build_energy_totals") - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) config = snakemake.params["energy"] diff --git a/scripts/build_gas_input_locations.py b/scripts/build_gas_input_locations.py index 59f31f5cf..a3b945abc 100644 --- a/scripts/build_gas_input_locations.py +++ b/scripts/build_gas_input_locations.py @@ -86,7 +86,7 @@ def build_gas_input_locations(lng_fn, entry_fn, prod_fn, countries): clusters="37", ) - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) regions = load_bus_regions( snakemake.input.regions_onshore, snakemake.input.regions_offshore diff --git a/scripts/build_gas_network.py b/scripts/build_gas_network.py index a70f15ff4..23f58caa4 100644 --- a/scripts/build_gas_network.py +++ b/scripts/build_gas_network.py @@ -147,7 +147,7 @@ def prepare_dataset( snakemake = mock_snakemake("build_gas_network") - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) gas_network = load_dataset(snakemake.input.gas_network) diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index 30c652d88..9a513673b 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -141,7 +141,7 @@ def build_nodal_distribution_key(hotmaps, regions, countries): clusters=48, ) - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) countries = snakemake.params["countries"] diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index 4fec95cc0..c28351c17 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -153,7 +153,7 @@ def add_non_eu28_industrial_energy_demand(countries, demand): def industrial_energy_demand(countries, year): nprocesses = snakemake.threads - disable_progress = snakemake.params["run"].get("disable_progressbar", False) + disable_progress = snakemake.config["run"].get("disable_progressbar", False) func = partial( industrial_energy_demand_per_country, year=year, jrc_dir=snakemake.input.jrc ) diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index eb1b16cb0..62073ea1f 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -217,7 +217,7 @@ def get_sector_data(sector, country): def industry_production(countries, year, eurostat_dir, jrc_dir): nprocesses = snakemake.threads - disable_progress = snakemake.params["run"].get("disable_progressbar", False) + disable_progress = snakemake.config["run"].get("disable_progressbar", False) func = partial( industry_production_per_country, @@ -277,7 +277,7 @@ def separate_basic_chemicals(demand, year): snakemake = mock_snakemake("build_industrial_production_per_country") - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) countries = snakemake.params["countries"] diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index bc35dbcfe..e864d9259 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -23,7 +23,7 @@ snakemake = mock_snakemake("build_population_layouts") - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) cutout = atlite.Cutout(snakemake.input.cutout) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 55541522b..8fba74e65 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -203,7 +203,7 @@ configure_logging(snakemake) nprocesses = int(snakemake.threads) - noprogress = snakemake.params["run"].get("disable_progressbar", True) + noprogress = snakemake.config["run"].get("disable_progressbar", True) config = snakemake.params["renewable"][snakemake.wildcards.technology] resource = config["resource"] # pv panel config / wind turbine config correction_factor = config.get("correction_factor", 1.0) diff --git a/scripts/cluster_gas_network.py b/scripts/cluster_gas_network.py index ba11ce1b2..e7554dff3 100755 --- a/scripts/cluster_gas_network.py +++ b/scripts/cluster_gas_network.py @@ -110,7 +110,7 @@ def aggregate_parallel_pipes(df): snakemake = mock_snakemake("cluster_gas_network", simpl="", clusters="37") - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) fn = snakemake.input.cleaned_gas_network df = pd.read_csv(fn, index_col=0) diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 560745252..da0712d7a 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -682,7 +682,7 @@ def to_csv(df): snakemake = mock_snakemake("make_summary") - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) networks_dict = { (cluster, ll, opt + sector_opt, planning_horizon): "results/" diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 8aac9db1f..399f46e87 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -930,7 +930,7 @@ def plot_series(network, carrier="AC", name="test"): planning_horizons="2030", ) - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) overrides = override_component_attrs(snakemake.input.overrides) n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index e18ff61b5..4aa37de5e 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -545,7 +545,7 @@ def plot_carbon_budget_distribution(input_eurostat): snakemake = mock_snakemake("plot_summary") - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) n_header = 4 diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index c69f72909..86a9bc821 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3266,7 +3266,7 @@ def set_temporal_aggregation(n, opts, solver_name): planning_horizons="2030", ) - logging.basicConfig(level=snakemake.params["logging"]["level"]) + logging.basicConfig(level=snakemake.config["logging"]["level"]) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py index 0271166c7..de42587d4 100644 --- a/scripts/retrieve_databundle.py +++ b/scripts/retrieve_databundle.py @@ -63,7 +63,7 @@ to_fn = Path(f"{rootpath}/data") logger.info(f"Downloading databundle from '{url}'.") - disable_progress = snakemake.params["run"].get("disable_progressbar", False) + disable_progress = snakemake.config["run"].get("disable_progressbar", False) progress_retrieve(url, tarball_fn, disable=disable_progress) logger.info("Extracting databundle.") From 8af1fe5649feb6df991db0d5b991c66fc9e8c25e Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Wed, 17 May 2023 19:25:45 +0200 Subject: [PATCH 06/24] replace the word "config" to "params" in functions --- rules/build_electricity.smk | 1 + scripts/add_electricity.py | 47 +++++++++-------- scripts/build_biomass_potentials.py | 8 +-- scripts/build_energy_totals.py | 6 +-- scripts/build_hydro_profile.py | 6 +-- ...ustrial_energy_demand_per_country_today.py | 10 ++-- ...build_industrial_production_per_country.py | 8 +-- ...ustrial_production_per_country_tomorrow.py | 14 +++--- scripts/build_industry_sector_ratios.py | 50 +++++++++---------- scripts/build_renewable_profiles.py | 48 +++++++++--------- scripts/prepare_sector_network.py | 4 +- 11 files changed, 101 insertions(+), 101 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 9414c25b4..88e7c548d 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -277,6 +277,7 @@ rule add_electricity: countries=config["countries"], renewable=config["renewable"], electricity=config["electricity"], + conventional=config.get("conventional", {}) costs=config["costs"], input: **{ diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index c3e847cea..a3e033e71 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -135,7 +135,7 @@ def _add_missing_carriers_from_costs(n, costs, carriers): n.import_components_from_dataframe(emissions, "Carrier") -def load_costs(tech_costs, config, elec_config, Nyears=1.0): +def load_costs(tech_costs, params, elec_params, Nyears=1.0): # set all asset costs and other parameters costs = pd.read_csv(tech_costs, index_col=[0, 1]).sort_index() @@ -143,7 +143,7 @@ def load_costs(tech_costs, config, elec_config, Nyears=1.0): costs.loc[costs.unit.str.contains("/kW"), "value"] *= 1e3 costs.unit = costs.unit.str.replace("/kW", "/MW") - fill_values = config["fill_values"] + fill_values = params["fill_values"] costs = costs.value.unstack().fillna(fill_values) costs["capital_cost"] = ( @@ -166,8 +166,8 @@ def load_costs(tech_costs, config, elec_config, Nyears=1.0): costs.at["CCGT", "co2_emissions"] = costs.at["gas", "co2_emissions"] costs.at["solar", "capital_cost"] = ( - config["rooftop_share"] * costs.at["solar-rooftop", "capital_cost"] - + (1 - config["rooftop_share"]) * costs.at["solar-utility", "capital_cost"] + params["rooftop_share"] * costs.at["solar-rooftop", "capital_cost"] + + (1 - params["rooftop_share"]) * costs.at["solar-utility", "capital_cost"] ) def costs_for_storage(store, link1, link2=None, max_hours=1.0): @@ -178,7 +178,7 @@ def costs_for_storage(store, link1, link2=None, max_hours=1.0): dict(capital_cost=capital_cost, marginal_cost=0.0, co2_emissions=0.0) ) - max_hours = elec_config["max_hours"] + max_hours = elec_params["max_hours"] costs.loc["battery"] = costs_for_storage( costs.loc["battery storage"], costs.loc["battery inverter"], @@ -192,7 +192,7 @@ def costs_for_storage(store, link1, link2=None, max_hours=1.0): ) for attr in ("marginal_cost", "capital_cost"): - overwrites = config.get(attr) + overwrites = params.get(attr) if overwrites is not None: overwrites = pd.Series(overwrites) costs.loc[overwrites.index, attr] = overwrites @@ -356,7 +356,7 @@ def attach_conventional_generators( ppl, conventional_carriers, extendable_carriers, - conventional_config, + conventional_params, conventional_inputs, ): carriers = set(conventional_carriers) | set(extendable_carriers["Generator"]) @@ -393,12 +393,12 @@ def attach_conventional_generators( lifetime=(ppl.dateout - ppl.datein).fillna(np.inf), ) - for carrier in conventional_config: + for carrier in conventional_params: # Generators with technology affected idx = n.generators.query("carrier == @carrier").index - for attr in list(set(conventional_config[carrier]) & set(n.generators)): - values = conventional_config[carrier][attr] + for attr in list(set(conventional_params[carrier]) & set(n.generators)): + values = conventional_params[carrier][attr] if f"conventional_{carrier}_{attr}" in conventional_inputs: # Values affecting generators of technology k country-specific @@ -413,7 +413,7 @@ def attach_conventional_generators( n.generators.loc[idx, attr] = values -def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **config): +def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **params): _add_missing_carriers_from_costs(n, costs, carriers) ppl = ( @@ -468,9 +468,9 @@ def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **con ) if "PHS" in carriers and not phs.empty: - # fill missing max hours to config value and + # fill missing max hours to params value and # assume no natural inflow due to lack of data - max_hours = config.get("PHS_max_hours", 6) + max_hours = params.get("PHS_max_hours", 6) phs = phs.replace({"max_hours": {0: max_hours}}) n.madd( "StorageUnit", @@ -486,7 +486,7 @@ def attach_hydro(n, costs, ppl, profile_hydro, hydro_capacities, carriers, **con ) if "hydro" in carriers and not hydro.empty: - hydro_max_hours = config.get("hydro_max_hours") + hydro_max_hours = params.get("hydro_max_hours") assert hydro_max_hours is not None, "No path for hydro capacities given." @@ -636,13 +636,12 @@ def attach_OPSD_renewables(n, tech_map): n.generators.p_nom_min.update(gens.bus.map(caps).dropna()) -def estimate_renewable_capacities(n, config): - year = config["electricity"]["estimate_renewable_capacities"]["year"] - tech_map = config["electricity"]["estimate_renewable_capacities"][ +def estimate_renewable_capacities(n, electricity_params, countries): + year = electricity_params["estimate_renewable_capacities"]["year"] + tech_map = electricity_params["estimate_renewable_capacities"][ "technology_mapping" ] - countries = config["countries"] - expansion_limit = config["electricity"]["estimate_renewable_capacities"][ + expansion_limit = electricity_params["estimate_renewable_capacities"][ "expansion_limit" ] @@ -759,7 +758,7 @@ def add_nice_carrier_names(n, config): ppl, conventional_carriers, extendable_carriers, - snakemake.config.get("conventional", {}), + snakemake.params.get("conventional", {}), conventional_inputs, ) @@ -773,15 +772,15 @@ def add_nice_carrier_names(n, config): ) if "hydro" in renewable_carriers: - conf = snakemake.params["renewable"]["hydro"] + para = snakemake.params["renewable"]["hydro"] attach_hydro( n, costs, ppl, snakemake.input.profile_hydro, snakemake.input.hydro_capacities, - conf.pop("carriers", []), - **conf, + para.pop("carriers", []), + **para, ) if "estimate_renewable_capacities" not in snakemake.params["electricity"]: @@ -829,7 +828,7 @@ def add_nice_carrier_names(n, config): "technology_mapping" ] attach_OPSD_renewables(n, tech_map) - estimate_renewable_capacities(n, snakemake.config) + estimate_renewable_capacities(n, snakemake.params["electricity"],snakemake.params["countries"]) update_p_nom_max(n) diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py index 35218e2ce..0b423ad5f 100644 --- a/scripts/build_biomass_potentials.py +++ b/scripts/build_biomass_potentials.py @@ -210,9 +210,9 @@ def convert_nuts2_to_regions(bio_nuts2, regions): snakemake = mock_snakemake("build_biomass_potentials", simpl="", clusters="5") - config = snakemake.params["biomass"] - year = config["year"] - scenario = config["scenario"] + params = snakemake.params["biomass"] + year = params["year"] + scenario = params["scenario"] enspreso = enspreso_biomass_potentials(year, scenario) @@ -228,7 +228,7 @@ def convert_nuts2_to_regions(bio_nuts2, regions): df.to_csv(snakemake.output.biomass_potentials_all) - grouper = {v: k for k, vv in config["classes"].items() for v in vv} + grouper = {v: k for k, vv in params["classes"].items() for v in vv} df = df.groupby(grouper, axis=1).sum() df *= 1e6 # TWh/a to MWh/a diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 5099d1403..3e3cb4853 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -737,7 +737,7 @@ def build_transport_data(countries, population, idees): logging.basicConfig(level=snakemake.config["logging"]["level"]) - config = snakemake.params["energy"] + params = snakemake.params["energy"] nuts3 = gpd.read_file(snakemake.input.nuts3_shapes).set_index("index") population = nuts3["pop"].groupby(nuts3.country).sum() @@ -745,7 +745,7 @@ def build_transport_data(countries, population, idees): countries = snakemake.params["countries"] idees_countries = pd.Index(countries).intersection(eu28) - data_year = config["energy_totals_year"] + data_year = params["energy_totals_year"] report_year = snakemake.params["energy"]["eurostat_report_year"] input_eurostat = snakemake.input.eurostat eurostat = build_eurostat(input_eurostat, countries, report_year, data_year) @@ -755,7 +755,7 @@ def build_transport_data(countries, population, idees): energy = build_energy_totals(countries, eurostat, swiss, idees) energy.to_csv(snakemake.output.energy_name) - base_year_emissions = config["base_emissions_year"] + base_year_emissions = params["base_emissions_year"] emissions_scope = snakemake.params["energy"]["emissions"] eea_co2 = build_eea_co2(snakemake.input.co2, base_year_emissions, emissions_scope) eurostat_co2 = build_eurostat_co2( diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 5453ac5cc..dd686be38 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -130,7 +130,7 @@ def get_eia_annual_hydro_generation(fn, countries): snakemake = mock_snakemake("build_hydro_profile") configure_logging(snakemake) - config_hydro = snakemake.params["renewable"]["hydro"] + params_hydro = snakemake.params["renewable"]["hydro"] cutout = atlite.Cutout(snakemake.input.cutout) countries = snakemake.params["countries"] @@ -151,7 +151,7 @@ def get_eia_annual_hydro_generation(fn, countries): normalize_using_yearly=eia_stats, ) - if "clip_min_inflow" in config_hydro: - inflow = inflow.where(inflow > config_hydro["clip_min_inflow"], 0) + if "clip_min_inflow" in params_hydro: + inflow = inflow.where(inflow > params_hydro["clip_min_inflow"], 0) inflow.to_netcdf(snakemake.output[0]) diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index c28351c17..9f8c47d08 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -101,8 +101,8 @@ def add_ammonia_energy_demand(demand): def get_ammonia_by_fuel(x): fuels = { - "gas": config["MWh_CH4_per_tNH3_SMR"], - "electricity": config["MWh_elec_per_tNH3_SMR"], + "gas": params["MWh_CH4_per_tNH3_SMR"], + "electricity": params["MWh_elec_per_tNH3_SMR"], } return pd.Series({k: x * v for k, v in fuels.items()}) @@ -112,7 +112,7 @@ def get_ammonia_by_fuel(x): index=demand.index, fill_value=0.0 ) - ammonia = pd.DataFrame({"ammonia": ammonia * config["MWh_NH3_per_tNH3"]}).T + ammonia = pd.DataFrame({"ammonia": ammonia * params["MWh_NH3_per_tNH3"]}).T demand["Ammonia"] = ammonia.unstack().reindex(index=demand.index, fill_value=0.0) @@ -178,8 +178,8 @@ def industrial_energy_demand(countries, year): snakemake = mock_snakemake("build_industrial_energy_demand_per_country_today") - config = snakemake.params["industry"] - year = config.get("reference_year", 2015) + params = snakemake.params["industry"] + year = params.get("reference_year", 2015) countries = pd.Index(snakemake.params["countries"]) demand = industrial_energy_demand(countries.intersection(eu28), year) diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index 62073ea1f..889c9ecd9 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -264,9 +264,9 @@ def separate_basic_chemicals(demand, year): # assume HVC, methanol, chlorine production proportional to non-ammonia basic chemicals distribution_key = demand["Basic chemicals"] / demand["Basic chemicals"].sum() - demand["HVC"] = config["HVC_production_today"] * 1e3 * distribution_key - demand["Chlorine"] = config["chlorine_production_today"] * 1e3 * distribution_key - demand["Methanol"] = config["methanol_production_today"] * 1e3 * distribution_key + demand["HVC"] = params["HVC_production_today"] * 1e3 * distribution_key + demand["Chlorine"] = params["chlorine_production_today"] * 1e3 * distribution_key + demand["Methanol"] = params["methanol_production_today"] * 1e3 * distribution_key demand.drop(columns=["Basic chemicals"], inplace=True) @@ -283,7 +283,7 @@ def separate_basic_chemicals(demand, year): year = snakemake.params["industry"]["reference_year"] - config = snakemake.params["industry"] + params = snakemake.params["industry"] jrc_dir = snakemake.input.jrc eurostat_dir = snakemake.input.eurostat diff --git a/scripts/build_industrial_production_per_country_tomorrow.py b/scripts/build_industrial_production_per_country_tomorrow.py index b9ac9b168..609170aa5 100644 --- a/scripts/build_industrial_production_per_country_tomorrow.py +++ b/scripts/build_industrial_production_per_country_tomorrow.py @@ -15,7 +15,7 @@ snakemake = mock_snakemake("build_industrial_production_per_country_tomorrow") - config = snakemake.params["industry"] + params = snakemake.params["industry"] investment_year = int(snakemake.wildcards.planning_horizons) @@ -25,8 +25,8 @@ keys = ["Integrated steelworks", "Electric arc"] total_steel = production[keys].sum(axis=1) - st_primary_fraction = get(config["St_primary_fraction"], investment_year) - dri_fraction = get(config["DRI_fraction"], investment_year) + st_primary_fraction = get(params["St_primary_fraction"], investment_year) + dri_fraction = get(params["DRI_fraction"], investment_year) int_steel = production["Integrated steelworks"].sum() fraction_persistent_primary = st_primary_fraction * total_steel.sum() / int_steel @@ -51,7 +51,7 @@ key_pri = "Aluminium - primary production" key_sec = "Aluminium - secondary production" - al_primary_fraction = get(config["Al_primary_fraction"], investment_year) + al_primary_fraction = get(params["Al_primary_fraction"], investment_year) fraction_persistent_primary = ( al_primary_fraction * total_aluminium.sum() / production[key_pri].sum() ) @@ -60,15 +60,15 @@ production[key_sec] = total_aluminium - production[key_pri] production["HVC (mechanical recycling)"] = ( - get(config["HVC_mechanical_recycling_fraction"], investment_year) + get(params["HVC_mechanical_recycling_fraction"], investment_year) * production["HVC"] ) production["HVC (chemical recycling)"] = ( - get(config["HVC_chemical_recycling_fraction"], investment_year) + get(params["HVC_chemical_recycling_fraction"], investment_year) * production["HVC"] ) - production["HVC"] *= get(config["HVC_primary_fraction"], investment_year) + production["HVC"] *= get(params["HVC_primary_fraction"], investment_year) fn = snakemake.output.industrial_production_per_country_tomorrow production.to_csv(fn, float_format="%.2f") diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py index c3e2bd49e..2ec007a91 100644 --- a/scripts/build_industry_sector_ratios.py +++ b/scripts/build_industry_sector_ratios.py @@ -185,10 +185,10 @@ def iron_and_steel(): df[sector] = df["Electric arc"] # add H2 consumption for DRI at 1.7 MWh H2 /ton steel - df.at["hydrogen", sector] = config["H2_DRI"] + df.at["hydrogen", sector] = params["H2_DRI"] # add electricity consumption in DRI shaft (0.322 MWh/tSl) - df.at["elec", sector] += config["elec_DRI"] + df.at["elec", sector] += params["elec_DRI"] ## Integrated steelworks # could be used in combination with CCS) @@ -383,19 +383,19 @@ def chemicals_industry(): assert s_emi.index[0] == sector # convert from MtHVC/a to ktHVC/a - s_out = config["HVC_production_today"] * 1e3 + s_out = params["HVC_production_today"] * 1e3 # tCO2/t material df.loc["process emission", sector] += ( s_emi["Process emissions"] - - config["petrochemical_process_emissions"] * 1e3 - - config["NH3_process_emissions"] * 1e3 + - params["petrochemical_process_emissions"] * 1e3 + - params["NH3_process_emissions"] * 1e3 ) / s_out # emissions originating from feedstock, could be non-fossil origin # tCO2/t material df.loc["process emission from feedstock", sector] += ( - config["petrochemical_process_emissions"] * 1e3 + params["petrochemical_process_emissions"] * 1e3 ) / s_out # convert from ktoe/a to GWh/a @@ -405,18 +405,18 @@ def chemicals_industry(): # subtract ammonia energy demand (in ktNH3/a) ammonia = pd.read_csv(snakemake.input.ammonia_production, index_col=0) ammonia_total = ammonia.loc[ammonia.index.intersection(eu28), str(year)].sum() - df.loc["methane", sector] -= ammonia_total * config["MWh_CH4_per_tNH3_SMR"] - df.loc["elec", sector] -= ammonia_total * config["MWh_elec_per_tNH3_SMR"] + df.loc["methane", sector] -= ammonia_total * params["MWh_CH4_per_tNH3_SMR"] + df.loc["elec", sector] -= ammonia_total * params["MWh_elec_per_tNH3_SMR"] # subtract chlorine demand - chlorine_total = config["chlorine_production_today"] - df.loc["hydrogen", sector] -= chlorine_total * config["MWh_H2_per_tCl"] - df.loc["elec", sector] -= chlorine_total * config["MWh_elec_per_tCl"] + chlorine_total = params["chlorine_production_today"] + df.loc["hydrogen", sector] -= chlorine_total * params["MWh_H2_per_tCl"] + df.loc["elec", sector] -= chlorine_total * params["MWh_elec_per_tCl"] # subtract methanol demand - methanol_total = config["methanol_production_today"] - df.loc["methane", sector] -= methanol_total * config["MWh_CH4_per_tMeOH"] - df.loc["elec", sector] -= methanol_total * config["MWh_elec_per_tMeOH"] + methanol_total = params["methanol_production_today"] + df.loc["methane", sector] -= methanol_total * params["MWh_CH4_per_tMeOH"] + df.loc["elec", sector] -= methanol_total * params["MWh_elec_per_tMeOH"] # MWh/t material df.loc[sources, sector] = df.loc[sources, sector] / s_out @@ -427,37 +427,37 @@ def chemicals_industry(): sector = "HVC (mechanical recycling)" df[sector] = 0.0 - df.loc["elec", sector] = config["MWh_elec_per_tHVC_mechanical_recycling"] + df.loc["elec", sector] = params["MWh_elec_per_tHVC_mechanical_recycling"] # HVC chemical recycling sector = "HVC (chemical recycling)" df[sector] = 0.0 - df.loc["elec", sector] = config["MWh_elec_per_tHVC_chemical_recycling"] + df.loc["elec", sector] = params["MWh_elec_per_tHVC_chemical_recycling"] # Ammonia sector = "Ammonia" df[sector] = 0.0 if snakemake.params["sector"].get("ammonia", False): - df.loc["ammonia", sector] = config["MWh_NH3_per_tNH3"] + df.loc["ammonia", sector] = params["MWh_NH3_per_tNH3"] else: - df.loc["hydrogen", sector] = config["MWh_H2_per_tNH3_electrolysis"] - df.loc["elec", sector] = config["MWh_elec_per_tNH3_electrolysis"] + df.loc["hydrogen", sector] = params["MWh_H2_per_tNH3_electrolysis"] + df.loc["elec", sector] = params["MWh_elec_per_tNH3_electrolysis"] # Chlorine sector = "Chlorine" df[sector] = 0.0 - df.loc["hydrogen", sector] = config["MWh_H2_per_tCl"] - df.loc["elec", sector] = config["MWh_elec_per_tCl"] + df.loc["hydrogen", sector] = params["MWh_H2_per_tCl"] + df.loc["elec", sector] = params["MWh_elec_per_tCl"] # Methanol sector = "Methanol" df[sector] = 0.0 - df.loc["methane", sector] = config["MWh_CH4_per_tMeOH"] - df.loc["elec", sector] = config["MWh_elec_per_tMeOH"] + df.loc["methane", sector] = params["MWh_CH4_per_tMeOH"] + df.loc["elec", sector] = params["MWh_elec_per_tMeOH"] # Other chemicals @@ -1465,10 +1465,10 @@ def other_industrial_sectors(): snakemake = mock_snakemake("build_industry_sector_ratios") - # TODO make config option + # TODO make params option year = 2015 - config = snakemake.params["industry"] + params = snakemake.params["industry"] df = pd.concat( [ diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 8fba74e65..91463c23a 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -64,7 +64,7 @@ - ``resources/offshore_shapes.geojson``: confer :ref:`shapes` - ``resources/regions_onshore.geojson``: (if not offshore wind), confer :ref:`busregions` - ``resources/regions_offshore.geojson``: (if offshore wind), :ref:`busregions` -- ``"cutouts/" + config["renewable"][{technology}]['cutout']``: :ref:`cutout` +- ``"cutouts/" + params["renewable"][{technology}]['cutout']``: :ref:`cutout` - ``networks/base.nc``: :ref:`base` Outputs @@ -204,14 +204,14 @@ nprocesses = int(snakemake.threads) noprogress = snakemake.config["run"].get("disable_progressbar", True) - config = snakemake.params["renewable"][snakemake.wildcards.technology] - resource = config["resource"] # pv panel config / wind turbine config - correction_factor = config.get("correction_factor", 1.0) - capacity_per_sqkm = config["capacity_per_sqkm"] - p_nom_max_meth = config.get("potential", "conservative") + params = snakemake.params["renewable"][snakemake.wildcards.technology] + resource = params["resource"] # pv panel params / wind turbine params + correction_factor = params.get("correction_factor", 1.0) + capacity_per_sqkm = params["capacity_per_sqkm"] + p_nom_max_meth = params.get("potential", "conservative") - if isinstance(config.get("corine", {}), list): - config["corine"] = {"grid_codes": config["corine"]} + if isinstance(params.get("corine", {}), list): + params["corine"] = {"grid_codes": params["corine"]} if correction_factor != 1.0: logger.info(f"correction_factor is set as {correction_factor}") @@ -229,13 +229,13 @@ regions = regions.set_index("name").rename_axis("bus") buses = regions.index - res = config.get("excluder_resolution", 100) + res = params.get("excluder_resolution", 100) excluder = atlite.ExclusionContainer(crs=3035, res=res) - if config["natura"]: + if params["natura"]: excluder.add_raster(snakemake.input.natura, nodata=0, allow_no_overlap=True) - corine = config.get("corine", {}) + corine = params.get("corine", {}) if "grid_codes" in corine: codes = corine["grid_codes"] excluder.add_raster(snakemake.input.corine, codes=codes, invert=True, crs=3035) @@ -246,28 +246,28 @@ snakemake.input.corine, codes=codes, buffer=buffer, crs=3035 ) - if "ship_threshold" in config: + if "ship_threshold" in params: shipping_threshold = ( - config["ship_threshold"] * 8760 * 6 + params["ship_threshold"] * 8760 * 6 ) # approximation because 6 years of data which is hourly collected func = functools.partial(np.less, shipping_threshold) excluder.add_raster( snakemake.input.ship_density, codes=func, crs=4326, allow_no_overlap=True ) - if config.get("max_depth"): + if params.get("max_depth"): # lambda not supported for atlite + multiprocessing # use named function np.greater with partially frozen argument instead # and exclude areas where: -max_depth > grid cell depth - func = functools.partial(np.greater, -config["max_depth"]) + func = functools.partial(np.greater, -params["max_depth"]) excluder.add_raster(snakemake.input.gebco, codes=func, crs=4326, nodata=-1000) - if "min_shore_distance" in config: - buffer = config["min_shore_distance"] + if "min_shore_distance" in params: + buffer = params["min_shore_distance"] excluder.add_geometry(snakemake.input.country_shapes, buffer=buffer) - if "max_shore_distance" in config: - buffer = config["max_shore_distance"] + if "max_shore_distance" in params: + buffer = params["max_shore_distance"] excluder.add_geometry( snakemake.input.country_shapes, buffer=buffer, invert=True ) @@ -309,7 +309,7 @@ p_nom_max = capacities / max_cap_factor else: raise AssertionError( - 'Config key `potential` should be one of "simple" ' + 'params key `potential` should be one of "simple" ' f'(default) or "conservative", not "{p_nom_max_meth}"' ) @@ -358,13 +358,13 @@ # select only buses with some capacity and minimal capacity factor ds = ds.sel( bus=( - (ds["profile"].mean("time") > config.get("min_p_max_pu", 0.0)) - & (ds["p_nom_max"] > config.get("min_p_nom_max", 0.0)) + (ds["profile"].mean("time") > params.get("min_p_max_pu", 0.0)) + & (ds["p_nom_max"] > params.get("min_p_nom_max", 0.0)) ) ) - if "clip_p_max_pu" in config: - min_p_max_pu = config["clip_p_max_pu"] + if "clip_p_max_pu" in params: + min_p_max_pu = params["clip_p_max_pu"] ds["profile"] = ds["profile"].where(ds["profile"] >= min_p_max_pu, 0) ds.to_netcdf(snakemake.output.profile) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 86a9bc821..1db9b916a 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -727,7 +727,7 @@ def cycling_shift(df, steps=1): return df -def prepare_costs(cost_file, config, nyears): +def prepare_costs(cost_file, params, nyears): # set all asset costs and other parameters costs = pd.read_csv(cost_file, index_col=[0, 1]).sort_index() @@ -739,7 +739,7 @@ def prepare_costs(cost_file, config, nyears): costs.loc[:, "value"].unstack(level=1).groupby("technology").sum(min_count=1) ) - costs = costs.fillna(config["fill_values"]) + costs = costs.fillna(params["fill_values"]) def annuity_factor(v): return annuity(v["lifetime"], v["discount rate"]) + v["FOM"] / 100 From 6366dc4c55076c084caddcbd67f997c17e9360bd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 17 May 2023 17:26:00 +0000 Subject: [PATCH 07/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/add_electricity.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index a3e033e71..adb8bf31b 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -638,9 +638,7 @@ def attach_OPSD_renewables(n, tech_map): def estimate_renewable_capacities(n, electricity_params, countries): year = electricity_params["estimate_renewable_capacities"]["year"] - tech_map = electricity_params["estimate_renewable_capacities"][ - "technology_mapping" - ] + tech_map = electricity_params["estimate_renewable_capacities"]["technology_mapping"] expansion_limit = electricity_params["estimate_renewable_capacities"][ "expansion_limit" ] @@ -828,7 +826,9 @@ def add_nice_carrier_names(n, config): "technology_mapping" ] attach_OPSD_renewables(n, tech_map) - estimate_renewable_capacities(n, snakemake.params["electricity"],snakemake.params["countries"]) + estimate_renewable_capacities( + n, snakemake.params["electricity"], snakemake.params["countries"] + ) update_p_nom_max(n) From d97a5434630d93c2f41ab64c8304ccc94961bee9 Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Wed, 17 May 2023 19:37:47 +0200 Subject: [PATCH 08/24] missing a coma in build_electricity.smk --- rules/build_electricity.smk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 88e7c548d..1ba1a1c52 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -277,7 +277,7 @@ rule add_electricity: countries=config["countries"], renewable=config["renewable"], electricity=config["electricity"], - conventional=config.get("conventional", {}) + conventional=config.get("conventional", {}), costs=config["costs"], input: **{ From c043100ada790d088ea190e9f8ba634e28a204e1 Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Fri, 19 May 2023 15:42:07 +0200 Subject: [PATCH 09/24] add more params in simplify_network.py --- rules/build_electricity.smk | 8 +++- scripts/add_electricity.py | 2 +- scripts/cluster_network.py | 2 +- scripts/simplify_network.py | 76 +++++++++++++++++++------------------ 4 files changed, 49 insertions(+), 39 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 1ba1a1c52..f33d16862 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -318,6 +318,12 @@ rule simplify_network: clustering=config["clustering"], electricity=config["electricity"], costs=config["costs"], + renewable=config["renewable"], + length_factor=config["lines"]["length_factor"], + p_max_pu=config["links"].get("p_max_pu", 1.0), + exclude_carriers=config["clustering"]["simplify_network"].get("exclude_carriers", []), + focus_weights=config.get("focus_weights", None), + solver_name=config["solving"]["solver"]["name"], input: network=RESOURCES + "networks/elec.nc", tech_costs=COSTS, @@ -350,7 +356,7 @@ rule cluster_network: lines=config["lines"], renewable=config["renewable"], clustering=config["clustering"], - enable=config["enable"].get("custom_busmap", False), + custom_busmap=config["enable"].get("custom_busmap", False), input: network=RESOURCES + "networks/elec_s{simpl}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson", diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index adb8bf31b..3717f0e8a 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -756,7 +756,7 @@ def add_nice_carrier_names(n, config): ppl, conventional_carriers, extendable_carriers, - snakemake.params.get("conventional", {}), + snakemake.params["conventional"], conventional_inputs, ) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 78798804f..d217e5ed8 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -521,7 +521,7 @@ def consense(x): for p in aggregation_strategies.keys() } - custom_busmap = snakemake.params["enable"].get("custom_busmap", False) + custom_busmap = snakemake.params["custom_busmap"] if custom_busmap: custom_busmap = pd.read_csv( snakemake.input.custom_busmap, index_col=0, squeeze=True diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index bb7f615cc..91dde6b19 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -149,17 +149,17 @@ def simplify_network_to_380(n): return n, trafo_map -def _prepare_connection_costs_per_link(n, costs, config): +def _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_param): if n.links.empty: return {} connection_costs_per_link = {} - for tech in config["renewable"]: + for tech in renewable_param: if tech.startswith("offwind"): connection_costs_per_link[tech] = ( n.links.length - * config["lines"]["length_factor"] + * length_factor_param * ( n.links.underwater_fraction * costs.at[tech + "-connection-submarine", "capital_cost"] @@ -172,10 +172,10 @@ def _prepare_connection_costs_per_link(n, costs, config): def _compute_connection_costs_to_bus( - n, busmap, costs, config, connection_costs_per_link=None, buses=None + n, busmap, costs, renewable_param, length_factor_param, connection_costs_per_link=None, buses=None ): if connection_costs_per_link is None: - connection_costs_per_link = _prepare_connection_costs_per_link(n, costs, config) + connection_costs_per_link = _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_param) if buses is None: buses = busmap.index[busmap.index != busmap.values] @@ -265,7 +265,7 @@ def replace_components(n, c, df, pnl): n.mremove(c, df.index[df.bus0.isin(buses_to_del) | df.bus1.isin(buses_to_del)]) -def simplify_links(n, costs, config, output, aggregation_strategies=dict()): +def simplify_links(n, costs, renewable_param, length_factor_param, p_max_pu_param, exclude_carriers_param, output, aggregation_strategies=dict()): ## Complex multi-node links are folded into end-points logger.info("Simplifying connected link components") @@ -315,7 +315,7 @@ def split_links(nodes): busmap = n.buses.index.to_series() - connection_costs_per_link = _prepare_connection_costs_per_link(n, costs, config) + connection_costs_per_link = _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_param) connection_costs_to_bus = pd.DataFrame( 0.0, index=n.buses.index, columns=list(connection_costs_per_link) ) @@ -333,12 +333,11 @@ def split_links(nodes): ) busmap.loc[buses] = b[np.r_[0, m.argmin(axis=0), 1]] connection_costs_to_bus.loc[buses] += _compute_connection_costs_to_bus( - n, busmap, costs, config, connection_costs_per_link, buses + n, busmap, costs, renewable_param, length_factor_param, connection_costs_per_link, buses ) all_links = [i for _, i in sum(links, [])] - p_max_pu = config["links"].get("p_max_pu", 1.0) lengths = n.links.loc[all_links, "length"] name = lengths.idxmax() + "+{}".format(len(links) - 1) params = dict( @@ -354,8 +353,8 @@ def split_links(nodes): / lengths.sum() * n.links.loc[all_links, "underwater_fraction"] ), - p_max_pu=p_max_pu, - p_min_pu=-p_max_pu, + p_max_pu=p_max_pu_param, + p_min_pu=-p_max_pu_param, underground=False, under_construction=False, ) @@ -377,33 +376,29 @@ def split_links(nodes): logger.debug("Collecting all components using the busmap") - exclude_carriers = config["clustering"]["simplify_network"].get( - "exclude_carriers", [] - ) - _aggregate_and_move_components( n, busmap, connection_costs_to_bus, output, aggregation_strategies=aggregation_strategies, - exclude_carriers=exclude_carriers, + exclude_carriers=exclude_carriers_param, ) return n, busmap -def remove_stubs(n, costs, config, output, aggregation_strategies=dict()): +def remove_stubs(n, costs, renewable_param, length_factor_param, clustering_param, exclude_carriers_param, output, aggregation_strategies=dict()): logger.info("Removing stubs") - across_borders = config["clustering"]["simplify_network"].get( + across_borders = clustering_param["simplify_network"].get( "remove_stubs_across_borders", True ) matching_attrs = [] if across_borders else ["country"] busmap = busmap_by_stubs(n, matching_attrs) - connection_costs_to_bus = _compute_connection_costs_to_bus(n, busmap, costs, config) + connection_costs_to_bus = _compute_connection_costs_to_bus(n, busmap, costs, renewable_param, length_factor_param) - exclude_carriers = config["clustering"]["simplify_network"].get( + exclude_carriers = clustering_param["simplify_network"].get( "exclude_carriers", [] ) @@ -473,17 +468,15 @@ def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None): def cluster( - n, n_clusters, config, algorithm="hac", feature=None, aggregation_strategies=dict() + n, n_clusters, focus_weights_param, renewable_param, solver_name_param, algorithm="hac", feature=None, aggregation_strategies=dict() ): logger.info(f"Clustering to {n_clusters} buses") - focus_weights = config.get("focus_weights", None) - renewable_carriers = pd.Index( [ tech for tech in n.generators.carrier.unique() - if tech.split("-", 2)[0] in config["renewable"] + if tech.split("-", 2)[0] in renewable_param ] ) @@ -492,10 +485,10 @@ def cluster( n_clusters, custom_busmap=False, aggregation_strategies=aggregation_strategies, - solver_name=config["solving"]["solver"]["name"], + solver_name=solver_name_param, algorithm=algorithm, feature=feature, - focus_weights=focus_weights, + focus_weights=focus_weights_param, ) return clustering.network, clustering.busmap @@ -531,23 +524,32 @@ def cluster( ) n, simplify_links_map = simplify_links( - n, technology_costs, snakemake.config, snakemake.output, aggregation_strategies + n, technology_costs, + snakemake.params['renewable'], + snakemake.params['length_factor'], + snakemake.params['p_max_pu'], + snakemake.params['exclude_carriers'], + snakemake.output, + aggregation_strategies ) busmaps = [trafo_map, simplify_links_map] - cluster_config = snakemake.params["clustering"]["simplify_network"] - if cluster_config.get("remove_stubs", True): + cluster_param = snakemake.params["clustering"]["simplify_network"] + if cluster_param.get("remove_stubs", True): n, stub_map = remove_stubs( n, technology_costs, - snakemake.config, + snakemake.params['renewable'], + snakemake.params['length_factor'], + snakemake.params["clustering"], + snakemake.params['exclude_carriers'], snakemake.output, aggregation_strategies=aggregation_strategies, ) busmaps.append(stub_map) - if cluster_config.get("to_substations", False): + if cluster_param.get("to_substations", False): n, substation_map = aggregate_to_substations(n, aggregation_strategies) busmaps.append(substation_map) @@ -558,10 +560,10 @@ def cluster( .get("cluster_network", {}) .get("algorithm", "hac") == "hac" - or cluster_config.get("algorithm", "hac") == "hac" + or cluster_param.get("algorithm", "hac") == "hac" ): carriers = ( - cluster_config.get("feature", "solar+onwind-time").split("-")[0].split("+") + cluster_param.get("feature", "solar+onwind-time").split("-")[0].split("+") ) for carrier in carriers: buses_i = list( @@ -577,9 +579,11 @@ def cluster( n, cluster_map = cluster( n, int(snakemake.wildcards.simpl), - snakemake.config, - cluster_config.get("algorithm", "hac"), - cluster_config.get("feature", None), + snakemake.params['focus_weights'], + snakemake.params['renewable'], + snakemake.params['solver_name'], + cluster_param.get("algorithm", "hac"), + cluster_param.get("feature", None), aggregation_strategies, ) busmaps.append(cluster_map) From 216a02fba12fe445e36fae1c019820715379c437 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 19 May 2023 13:42:43 +0000 Subject: [PATCH 10/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/build_electricity.smk | 4 +- scripts/simplify_network.py | 90 +++++++++++++++++++++++++++---------- 2 files changed, 69 insertions(+), 25 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index f33d16862..45436c5df 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -321,7 +321,9 @@ rule simplify_network: renewable=config["renewable"], length_factor=config["lines"]["length_factor"], p_max_pu=config["links"].get("p_max_pu", 1.0), - exclude_carriers=config["clustering"]["simplify_network"].get("exclude_carriers", []), + exclude_carriers=config["clustering"]["simplify_network"].get( + "exclude_carriers", [] + ), focus_weights=config.get("focus_weights", None), solver_name=config["solving"]["solver"]["name"], input: diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 91dde6b19..689e80843 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -172,10 +172,18 @@ def _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_ def _compute_connection_costs_to_bus( - n, busmap, costs, renewable_param, length_factor_param, connection_costs_per_link=None, buses=None + n, + busmap, + costs, + renewable_param, + length_factor_param, + connection_costs_per_link=None, + buses=None, ): if connection_costs_per_link is None: - connection_costs_per_link = _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_param) + connection_costs_per_link = _prepare_connection_costs_per_link( + n, costs, renewable_param, length_factor_param + ) if buses is None: buses = busmap.index[busmap.index != busmap.values] @@ -265,7 +273,16 @@ def replace_components(n, c, df, pnl): n.mremove(c, df.index[df.bus0.isin(buses_to_del) | df.bus1.isin(buses_to_del)]) -def simplify_links(n, costs, renewable_param, length_factor_param, p_max_pu_param, exclude_carriers_param, output, aggregation_strategies=dict()): +def simplify_links( + n, + costs, + renewable_param, + length_factor_param, + p_max_pu_param, + exclude_carriers_param, + output, + aggregation_strategies=dict(), +): ## Complex multi-node links are folded into end-points logger.info("Simplifying connected link components") @@ -315,7 +332,9 @@ def split_links(nodes): busmap = n.buses.index.to_series() - connection_costs_per_link = _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_param) + connection_costs_per_link = _prepare_connection_costs_per_link( + n, costs, renewable_param, length_factor_param + ) connection_costs_to_bus = pd.DataFrame( 0.0, index=n.buses.index, columns=list(connection_costs_per_link) ) @@ -333,7 +352,13 @@ def split_links(nodes): ) busmap.loc[buses] = b[np.r_[0, m.argmin(axis=0), 1]] connection_costs_to_bus.loc[buses] += _compute_connection_costs_to_bus( - n, busmap, costs, renewable_param, length_factor_param, connection_costs_per_link, buses + n, + busmap, + costs, + renewable_param, + length_factor_param, + connection_costs_per_link, + buses, ) all_links = [i for _, i in sum(links, [])] @@ -387,7 +412,16 @@ def split_links(nodes): return n, busmap -def remove_stubs(n, costs, renewable_param, length_factor_param, clustering_param, exclude_carriers_param, output, aggregation_strategies=dict()): +def remove_stubs( + n, + costs, + renewable_param, + length_factor_param, + clustering_param, + exclude_carriers_param, + output, + aggregation_strategies=dict(), +): logger.info("Removing stubs") across_borders = clustering_param["simplify_network"].get( @@ -396,12 +430,12 @@ def remove_stubs(n, costs, renewable_param, length_factor_param, clustering_para matching_attrs = [] if across_borders else ["country"] busmap = busmap_by_stubs(n, matching_attrs) - connection_costs_to_bus = _compute_connection_costs_to_bus(n, busmap, costs, renewable_param, length_factor_param) - - exclude_carriers = clustering_param["simplify_network"].get( - "exclude_carriers", [] + connection_costs_to_bus = _compute_connection_costs_to_bus( + n, busmap, costs, renewable_param, length_factor_param ) + exclude_carriers = clustering_param["simplify_network"].get("exclude_carriers", []) + _aggregate_and_move_components( n, busmap, @@ -468,7 +502,14 @@ def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None): def cluster( - n, n_clusters, focus_weights_param, renewable_param, solver_name_param, algorithm="hac", feature=None, aggregation_strategies=dict() + n, + n_clusters, + focus_weights_param, + renewable_param, + solver_name_param, + algorithm="hac", + feature=None, + aggregation_strategies=dict(), ): logger.info(f"Clustering to {n_clusters} buses") @@ -524,13 +565,14 @@ def cluster( ) n, simplify_links_map = simplify_links( - n, technology_costs, - snakemake.params['renewable'], - snakemake.params['length_factor'], - snakemake.params['p_max_pu'], - snakemake.params['exclude_carriers'], - snakemake.output, - aggregation_strategies + n, + technology_costs, + snakemake.params["renewable"], + snakemake.params["length_factor"], + snakemake.params["p_max_pu"], + snakemake.params["exclude_carriers"], + snakemake.output, + aggregation_strategies, ) busmaps = [trafo_map, simplify_links_map] @@ -540,10 +582,10 @@ def cluster( n, stub_map = remove_stubs( n, technology_costs, - snakemake.params['renewable'], - snakemake.params['length_factor'], + snakemake.params["renewable"], + snakemake.params["length_factor"], snakemake.params["clustering"], - snakemake.params['exclude_carriers'], + snakemake.params["exclude_carriers"], snakemake.output, aggregation_strategies=aggregation_strategies, ) @@ -579,9 +621,9 @@ def cluster( n, cluster_map = cluster( n, int(snakemake.wildcards.simpl), - snakemake.params['focus_weights'], - snakemake.params['renewable'], - snakemake.params['solver_name'], + snakemake.params["focus_weights"], + snakemake.params["renewable"], + snakemake.params["solver_name"], cluster_param.get("algorithm", "hac"), cluster_param.get("feature", None), aggregation_strategies, From 1c73bb03426beaaaedc10df7ee83d8f036395a4d Mon Sep 17 00:00:00 2001 From: virio-andreyana Date: Sat, 27 May 2023 12:22:53 +0200 Subject: [PATCH 11/24] add params for solve_network --- rules/solve_electricity.smk | 6 ++++++ rules/solve_myopic.smk | 6 ++++++ rules/solve_overnight.smk | 6 ++++++ scripts/solve_network.py | 38 ++++++++++++++++++------------------- 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index 2e9c4b44d..f0ce55c83 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -6,6 +6,12 @@ rule solve_network: params: solving=config["solving"], + config_parts={ + "foresight":config["foresight"], + "planning_horizons":config["scenario"]["planning_horizons"], + "co2_sequestration_potential":config["sector"].get("co2_sequestration_potential", 200), + "H2_retrofit_capacity_per_CH4":config["sector"]["H2_retrofit_capacity_per_CH4"] + }, input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index ec7638cf9..c682f7809 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -84,6 +84,12 @@ ruleorder: add_existing_baseyear > add_brownfield rule solve_sector_network_myopic: params: solving=config["solving"], + config_parts={ + "foresight":config["foresight"], + "planning_horizons":config["scenario"]["planning_horizons"], + "co2_sequestration_potential":config["sector"].get("co2_sequestration_potential", 200), + "H2_retrofit_capacity_per_CH4":config["sector"]["H2_retrofit_capacity_per_CH4"] + }, input: overrides="data/override_component_attrs", network=RESULTS diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index b657eb2bc..050372bdc 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -6,6 +6,12 @@ rule solve_sector_network: params: solving=config["solving"], + config_parts={ + "foresight":config["foresight"], + "planning_horizons":config["scenario"]["planning_horizons"], + "co2_sequestration_potential":config["sector"].get("co2_sequestration_potential", 200), + "H2_retrofit_capacity_per_CH4":config["sector"]["H2_retrofit_capacity_per_CH4"] + }, input: overrides="data/override_component_attrs", network=RESULTS diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 180c83c6a..d615fc858 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -44,14 +44,14 @@ from pypsa.descriptors import get_switchable_as_dense as get_as_dense -def add_land_use_constraint(n, config): +def add_land_use_constraint(n, param, config): if "m" in snakemake.wildcards.clusters: - _add_land_use_constraint_m(n, config) + _add_land_use_constraint_m(n, param, config) else: - _add_land_use_constraint(n, config) + _add_land_use_constraint(n, param) -def _add_land_use_constraint(n, config): +def _add_land_use_constraint(n, param): # warning: this will miss existing offwind which is not classed AC-DC and has carrier 'offwind' for carrier in ["solar", "onwind", "offwind-ac", "offwind-dc"]: @@ -80,10 +80,10 @@ def _add_land_use_constraint(n, config): n.generators.p_nom_max.clip(lower=0, inplace=True) -def _add_land_use_constraint_m(n, config): +def _add_land_use_constraint_m(n, param, config): # if generators clustering is lower than network clustering, land_use accounting is at generators clusters - planning_horizons = config["scenario"]["planning_horizons"] + planning_horizons = param["planning_horizons"] grouping_years = config["existing_capacities"]["grouping_years"] current_horizon = snakemake.wildcards.planning_horizons @@ -141,7 +141,7 @@ def add_co2_sequestration_limit(n, limit=200): ) -def prepare_network(n, solve_opts=None, config=None): +def prepare_network(n, solve_opts=None, config=None, param=None): if "clip_p_max_pu" in solve_opts: for df in ( n.generators_t.p_max_pu, @@ -191,11 +191,11 @@ def prepare_network(n, solve_opts=None, config=None): n.set_snapshots(n.snapshots[:nhours]) n.snapshot_weightings[:] = 8760.0 / nhours - if config["foresight"] == "myopic": - add_land_use_constraint(n, config) + if param["foresight"] == "myopic": + add_land_use_constraint(n, param, config) if n.stores.carrier.eq("co2 stored").any(): - limit = config["sector"].get("co2_sequestration_potential", 200) + limit = param["co2_sequestration_potential"] add_co2_sequestration_limit(n, limit=limit) return n @@ -222,7 +222,7 @@ def add_CCL_constraints(n, config): agg_p_nom_limits: data/agg_p_nom_minmax.csv """ agg_p_nom_minmax = pd.read_csv( - config["electricity"]["agg_p_nom_limits"], index_col=[0, 1] + config['electricity']["agg_p_nom_limits"], index_col=[0, 1] ) logger.info("Adding generation capacity constraints per carrier and country") p_nom = n.model["Generator-p_nom"] @@ -370,7 +370,7 @@ def add_SAFE_constraints(n, config): Which sets a reserve margin of 10% above the peak demand. """ peakdemand = n.loads_t.p_set.sum(axis=1).max() - margin = 1.0 + config["electricity"]["SAFE_reservemargin"] + margin = 1.0 + config['electricity']["SAFE_reservemargin"] reserve_margin = peakdemand * margin # TODO: do not take this from the plotting config! conv_techs = config["plotting"]["conv_techs"] @@ -590,13 +590,13 @@ def extra_functionality(n, snapshots): add_pipe_retrofit_constraint(n) -def solve_network(n, config, opts="", **kwargs): - set_of_options = config["solving"]["solver"]["options"] +def solve_network(n, config, solving_param, opts="", **kwargs): + set_of_options = solving_param["solver"]["options"] solver_options = ( - config["solving"]["solver_options"][set_of_options] if set_of_options else {} + solving_param["solver_options"][set_of_options] if set_of_options else {} ) - solver_name = config["solving"]["solver"]["name"] - cf_solving = config["solving"]["options"] + solver_name = solving_param["solver"]["name"] + cf_solving = solving_param["options"] track_iterations = cf_solving.get("track_iterations", False) min_iterations = cf_solving.get("min_iterations", 4) max_iterations = cf_solving.get("max_iterations", 6) @@ -675,10 +675,10 @@ def solve_network(n, config, opts="", **kwargs): else: n = pypsa.Network(snakemake.input.network) - n = prepare_network(n, solve_opts, config=snakemake.config) + n = prepare_network(n, solve_opts, config=snakemake.config, param=snakemake.params["config_parts"]) n = solve_network( - n, config=snakemake.config, opts=opts, log_fn=snakemake.log.solver + n, config=snakemake.config, solving_param=snakemake.params["solving"], opts=opts, log_fn=snakemake.log.solver ) n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) From ead87afce1ab0d7179b9012f88df1089e609bc61 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 27 May 2023 10:23:28 +0000 Subject: [PATCH 12/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/solve_electricity.smk | 14 +++++++++----- rules/solve_myopic.smk | 14 +++++++++----- rules/solve_overnight.smk | 14 +++++++++----- scripts/solve_network.py | 14 ++++++++++---- 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index f0ce55c83..30520dc1b 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -7,11 +7,15 @@ rule solve_network: params: solving=config["solving"], config_parts={ - "foresight":config["foresight"], - "planning_horizons":config["scenario"]["planning_horizons"], - "co2_sequestration_potential":config["sector"].get("co2_sequestration_potential", 200), - "H2_retrofit_capacity_per_CH4":config["sector"]["H2_retrofit_capacity_per_CH4"] - }, + "foresight": config["foresight"], + "planning_horizons": config["scenario"]["planning_horizons"], + "co2_sequestration_potential": config["sector"].get( + "co2_sequestration_potential", 200 + ), + "H2_retrofit_capacity_per_CH4": config["sector"][ + "H2_retrofit_capacity_per_CH4" + ], + }, input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index c682f7809..22555f75c 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -85,11 +85,15 @@ rule solve_sector_network_myopic: params: solving=config["solving"], config_parts={ - "foresight":config["foresight"], - "planning_horizons":config["scenario"]["planning_horizons"], - "co2_sequestration_potential":config["sector"].get("co2_sequestration_potential", 200), - "H2_retrofit_capacity_per_CH4":config["sector"]["H2_retrofit_capacity_per_CH4"] - }, + "foresight": config["foresight"], + "planning_horizons": config["scenario"]["planning_horizons"], + "co2_sequestration_potential": config["sector"].get( + "co2_sequestration_potential", 200 + ), + "H2_retrofit_capacity_per_CH4": config["sector"][ + "H2_retrofit_capacity_per_CH4" + ], + }, input: overrides="data/override_component_attrs", network=RESULTS diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index 050372bdc..a523f1325 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -7,11 +7,15 @@ rule solve_sector_network: params: solving=config["solving"], config_parts={ - "foresight":config["foresight"], - "planning_horizons":config["scenario"]["planning_horizons"], - "co2_sequestration_potential":config["sector"].get("co2_sequestration_potential", 200), - "H2_retrofit_capacity_per_CH4":config["sector"]["H2_retrofit_capacity_per_CH4"] - }, + "foresight": config["foresight"], + "planning_horizons": config["scenario"]["planning_horizons"], + "co2_sequestration_potential": config["sector"].get( + "co2_sequestration_potential", 200 + ), + "H2_retrofit_capacity_per_CH4": config["sector"][ + "H2_retrofit_capacity_per_CH4" + ], + }, input: overrides="data/override_component_attrs", network=RESULTS diff --git a/scripts/solve_network.py b/scripts/solve_network.py index d615fc858..6e07e340b 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -222,7 +222,7 @@ def add_CCL_constraints(n, config): agg_p_nom_limits: data/agg_p_nom_minmax.csv """ agg_p_nom_minmax = pd.read_csv( - config['electricity']["agg_p_nom_limits"], index_col=[0, 1] + config["electricity"]["agg_p_nom_limits"], index_col=[0, 1] ) logger.info("Adding generation capacity constraints per carrier and country") p_nom = n.model["Generator-p_nom"] @@ -370,7 +370,7 @@ def add_SAFE_constraints(n, config): Which sets a reserve margin of 10% above the peak demand. """ peakdemand = n.loads_t.p_set.sum(axis=1).max() - margin = 1.0 + config['electricity']["SAFE_reservemargin"] + margin = 1.0 + config["electricity"]["SAFE_reservemargin"] reserve_margin = peakdemand * margin # TODO: do not take this from the plotting config! conv_techs = config["plotting"]["conv_techs"] @@ -675,10 +675,16 @@ def solve_network(n, config, solving_param, opts="", **kwargs): else: n = pypsa.Network(snakemake.input.network) - n = prepare_network(n, solve_opts, config=snakemake.config, param=snakemake.params["config_parts"]) + n = prepare_network( + n, solve_opts, config=snakemake.config, param=snakemake.params["config_parts"] + ) n = solve_network( - n, config=snakemake.config, solving_param=snakemake.params["solving"], opts=opts, log_fn=snakemake.log.solver + n, + config=snakemake.config, + solving_param=snakemake.params["solving"], + opts=opts, + log_fn=snakemake.log.solver, ) n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) From e581ca930c24de2c3e3fa4d0d6bdac0fbfa06b30 Mon Sep 17 00:00:00 2001 From: virio-andreyana Date: Sat, 27 May 2023 15:50:37 +0200 Subject: [PATCH 13/24] made params more specific --- rules/build_electricity.smk | 29 +++++--- rules/build_sector.smk | 72 ++++++++++++++++---- rules/postprocess.smk | 3 +- rules/solve_electricity.smk | 2 +- rules/solve_myopic.smk | 7 +- scripts/add_brownfield.py | 6 +- scripts/add_electricity.py | 11 ++- scripts/add_existing_baseyear.py | 2 +- scripts/add_extra_components.py | 23 +++---- scripts/build_cop_profiles.py | 2 +- scripts/build_cutout.py | 2 +- scripts/build_hydro_profile.py | 2 +- scripts/build_industrial_distribution_key.py | 2 +- scripts/build_industry_sector_ratios.py | 2 +- scripts/build_powerplants.py | 4 +- scripts/build_retro_cost.py | 2 +- scripts/build_sequestration_potentials.py | 2 +- scripts/cluster_network.py | 8 +-- scripts/plot_summary.py | 4 +- scripts/prepare_network.py | 10 +-- scripts/prepare_sector_network.py | 14 ++-- scripts/simplify_network.py | 2 +- scripts/solve_operations_network.py | 2 +- 23 files changed, 131 insertions(+), 82 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 45436c5df..9b3a03fb1 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -39,7 +39,8 @@ rule build_electricity_demand: rule build_powerplants: params: - electricity=config["electricity"], + powerplants_filter=config["electricity"]["powerplants_filter"], + custom_powerplants=config["electricity"]["custom_powerplants"], countries=config["countries"], input: base_network=RESOURCES + "networks/base.nc", @@ -138,7 +139,7 @@ if config["enable"].get("build_cutout", False): rule build_cutout: params: snapshots=config["snapshots"], - atlite=config["atlite"], + cutouts=config["atlite"]["cutouts"], input: regions_onshore=RESOURCES + "regions_onshore.geojson", regions_offshore=RESOURCES + "regions_offshore.geojson", @@ -252,6 +253,7 @@ rule build_renewable_profiles: rule build_hydro_profile: params: + hydro=config["renewable"]["hydro"], countries=config["countries"], renewable=config["renewable"], input: @@ -272,8 +274,8 @@ rule build_hydro_profile: rule add_electricity: params: - lines=config["lines"], - load=config["load"], + length_factor=config["lines"]["length_factor"], + scaling_factor=config["load"]["scaling_factor"], countries=config["countries"], renewable=config["renewable"], electricity=config["electricity"], @@ -316,7 +318,7 @@ rule add_electricity: rule simplify_network: params: clustering=config["clustering"], - electricity=config["electricity"], + max_hours=config["electricity"]["max_hours"], costs=config["costs"], renewable=config["renewable"], length_factor=config["lines"]["length_factor"], @@ -352,10 +354,11 @@ rule simplify_network: rule cluster_network: params: - solving=config["solving"], - electricity=config["electricity"], + solver_name=config["solving"]["solver"]["name"], + max_hours=config["electricity"]["max_hours"], + conventional_carriers=config["electricity"].get("conventional_carriers", []), costs=config["costs"], - lines=config["lines"], + length_factor=config["lines"]["length_factor"], renewable=config["renewable"], clustering=config["clustering"], custom_busmap=config["enable"].get("custom_busmap", False), @@ -392,7 +395,8 @@ rule cluster_network: rule add_extra_components: params: costs=config["costs"], - electricity=config["electricity"], + ext_carriers=config["electricity"]["extendable_carriers"], + max_hours=config["electricity"]["max_hours"], input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}.nc", tech_costs=COSTS, @@ -414,9 +418,12 @@ rule add_extra_components: rule prepare_network: params: links=config["links"], - solving=config["solving"], lines=config["lines"], - electricity=config["electricity"], + solver_name=config["solving"]["solver"]["name"], + co2base=config["electricity"]["co2base"], + co2limit=config["electricity"]["co2limit"], + gaslimit=config["electricity"].get("gaslimit"), + max_hours=config["electricity"]["max_hours"], costs=config["costs"], input: RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc", diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 9faae4b9c..da8a60f9f 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -186,7 +186,7 @@ rule build_temperature_profiles: rule build_cop_profiles: params: - sector=config["sector"], + heat_pump_sink_T=config["sector"]["heat_pump_sink_T"], input: temp_soil_total=RESOURCES + "temp_soil_total_elec_s{simpl}_{clusters}.nc", temp_soil_rural=RESOURCES + "temp_soil_rural_elec_s{simpl}_{clusters}.nc", @@ -330,7 +330,7 @@ if config["sector"]["regional_co2_sequestration_potential"]["enable"]: rule build_sequestration_potentials: params: - sector=config["sector"], + co2seq_potential=config["sector"]["regional_co2_sequestration_potential"], input: sequestration_potential=HTTP.remote( "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson", @@ -405,8 +405,27 @@ rule build_ammonia_production: rule build_industry_sector_ratios: params: - industry=config["industry"], - sector=config["sector"], + industry={ + "H2_DRI":config["industry"]["H2_DRI"], + "elec_DRI":config["industry"]["elec_DRI"], + "HVC_production_today":config["industry"]["HVC_production_today"], + "petrochemical_process_emissions":config["industry"]["petrochemical_process_emissions"], + "NH3_process_emissions":config["industry"]["NH3_process_emissions"], + "MWh_CH4_per_tNH3_SMR":config["industry"]["MWh_CH4_per_tNH3_SMR"], + "MWh_elec_per_tNH3_SMR":config["industry"]["MWh_elec_per_tNH3_SMR"], + "chlorine_production_today":config["industry"]["chlorine_production_today"], + "MWh_H2_per_tCl":config["industry"]["MWh_H2_per_tCl"], + "MWh_elec_per_tCl":config["industry"]["MWh_elec_per_tCl"], + "methanol_production_today":config["industry"]["methanol_production_today"], + "MWh_CH4_per_tMeOH":config["industry"]["MWh_CH4_per_tMeOH"], + "MWh_elec_per_tMeOH":config["industry"]["MWh_elec_per_tMeOH"], + "MWh_elec_per_tHVC_mechanical_recycling":config["industry"]["MWh_elec_per_tHVC_mechanical_recycling"], + "MWh_elec_per_tHVC_chemical_recycling":config["industry"]["MWh_elec_per_tHVC_chemical_recycling"], + "MWh_NH3_per_tNH3":config["industry"]["MWh_NH3_per_tNH3"], + "MWh_H2_per_tNH3_electrolysis":config["industry"]["MWh_H2_per_tNH3_electrolysis"], + "MWh_elec_per_tNH3_electrolysis":config["industry"]["MWh_elec_per_tNH3_electrolysis"] + }, + sector_amonia=config["sector"].get("ammonia", False), input: ammonia_production=RESOURCES + "ammonia_production.csv", idees="data/jrc-idees-2015", @@ -427,7 +446,12 @@ rule build_industry_sector_ratios: rule build_industrial_production_per_country: params: - industry=config["industry"], + industry={ + "reference_year":config["industry"]["reference_year"], + "HVC_production_today":config["industry"]["HVC_production_today"], + "chlorine_production_today":config["industry"]["chlorine_production_today"], + "methanol_production_today":config["industry"]["methanol_production_today"] + }, countries=config["countries"], input: ammonia_production=RESOURCES + "ammonia_production.csv", @@ -451,7 +475,14 @@ rule build_industrial_production_per_country: rule build_industrial_production_per_country_tomorrow: params: - industry=config["industry"], + industry={ + "St_primary_fraction":config["industry"]["St_primary_fraction"], + "DRI_fraction":config["industry"]["DRI_fraction"], + "Al_primary_fraction":config["industry"]["Al_primary_fraction"], + "HVC_mechanical_recycling_fraction":config["industry"]["HVC_mechanical_recycling_fraction"], + "HVC_chemical_recycling_fraction":config["industry"]["HVC_chemical_recycling_fraction"], + "HVC_primary_fraction":config["industry"]["HVC_primary_fraction"] + }, input: industrial_production_per_country=RESOURCES + "industrial_production_per_country.csv", @@ -477,7 +508,7 @@ rule build_industrial_production_per_country_tomorrow: rule build_industrial_distribution_key: params: - industry=config["industry"], + hotmaps_locate_missing=config["industry"].get("hotmaps_locate_missing", False), countries=config["countries"], input: regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", @@ -555,7 +586,12 @@ rule build_industrial_energy_demand_per_node: rule build_industrial_energy_demand_per_country_today: params: countries=config["countries"], - industry=config["industry"], + industry={ + "reference_year":config["industry"].get("reference_year", 2015), + "MWh_CH4_per_tNH3_SMR":config["industry"]["MWh_CH4_per_tNH3_SMR"], + "MWh_elec_per_tNH3_SMR":config["industry"]["MWh_elec_per_tNH3_SMR"], + "MWh_NH3_per_tNH3":config["industry"]["MWh_NH3_per_tNH3"] + }, input: jrc="data/jrc-idees-2015", ammonia_production=RESOURCES + "ammonia_production.csv", @@ -603,7 +639,7 @@ if config["sector"]["retrofitting"]["retro_endogen"]: rule build_retro_cost: params: - sector=config["sector"], + retrofitting=config["sector"]["retrofitting"], countries=config["countries"], input: building_stock="data/retro/data_building_stock.csv", @@ -705,17 +741,23 @@ rule build_transport_demand: rule prepare_sector_network: params: co2_budget=config["co2_budget"], - solving=config["solving"], - existing_capacities=config["existing_capacities"], + solver_name=config["solving"]["solver"]["name"], + conventional_carriers=config["existing_capacities"]["conventional_carriers"], foresight=config["foresight"], costs=config["costs"], sector=config["sector"], - industry=config["industry"], + industry={ + "MWh_elec_per_tNH3_electrolysis":config["industry"]["MWh_elec_per_tNH3_electrolysis"], + "MWh_NH3_per_tNH3":config["industry"]["MWh_NH3_per_tNH3"], + "MWh_H2_per_tNH3_electrolysis":config["industry"]["MWh_H2_per_tNH3_electrolysis"], + "MWh_NH3_per_MWh_H2_cracker":config["industry"]["MWh_NH3_per_MWh_H2_cracker"] + }, pypsa_eur=config["pypsa_eur"], - lines=config["lines"], - scenario=config["scenario"], + length_factor=config["lines"]["length_factor"], + planning_horizons=config["scenario"]["planning_horizons"], countries=config["countries"], - energy=config["energy"], + emissions_scope=config["energy"]["emissions"], + report_year=config["energy"]["eurostat_report_year"], RDIR=RDIR, input: **build_retro_cost_output, diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 9eb04ef54..ac80cd103 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -122,7 +122,8 @@ rule make_summary: rule plot_summary: params: countries=config["countries"], - scenario=config["scenario"], + planning_horizons=config["scenario"]["planning_horizons"], + sector_opts=config["scenario"]["sector_opts"], plotting=config["plotting"], RDIR=RDIR, input: diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index 30520dc1b..892415c4e 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -41,7 +41,7 @@ rule solve_network: rule solve_operations_network: params: - solving=config["solving"], + options=config["solving"]["options"], input: network=RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 22555f75c..8ec1c4a8f 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -5,7 +5,7 @@ rule add_existing_baseyear: params: - scenario=config["scenario"], + baseyear=config["scenario"]["planning_horizons"][0], sector=config["sector"], existing_capacities=config["existing_capacities"], costs=config["costs"], @@ -48,8 +48,9 @@ rule add_existing_baseyear: rule add_brownfield: params: - sector=config["sector"], - existing_capacities=config["existing_capacities"], + H2_retrofit=config["sector"]["H2_retrofit"], + H2_retrofit_capacity_per_CH4=config["sector"]["H2_retrofit_capacity_per_CH4"], + threshold_capacity=config["existing_capacities"]["threshold_capacity"], input: overrides="data/override_component_attrs", network=RESULTS diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 9330953bc..48dffbb92 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -49,7 +49,7 @@ def add_brownfield(n, n_p, year): ) ] - threshold = snakemake.params["existing_capacities"]["threshold_capacity"] + threshold = snakemake.params["threshold_capacity"] if not chp_heat.empty: threshold_chp_heat = ( @@ -87,7 +87,7 @@ def add_brownfield(n, n_p, year): # deal with gas network pipe_carrier = ["gas pipeline"] - if snakemake.params["sector"]["H2_retrofit"]: + if snakemake.params["H2_retrofit"]: # drop capacities of previous year to avoid duplicating to_drop = n.links.carrier.isin(pipe_carrier) & (n.links.build_year != year) n.mremove("Link", n.links.loc[to_drop].index) @@ -98,7 +98,7 @@ def add_brownfield(n, n_p, year): & (n.links.build_year != year) ].index gas_pipes_i = n.links[n.links.carrier.isin(pipe_carrier)].index - CH4_per_H2 = 1 / snakemake.params["sector"]["H2_retrofit_capacity_per_CH4"] + CH4_per_H2 = 1 / snakemake.params["H2_retrofit_capacity_per_CH4"] fr = "H2 pipeline retrofitted" to = "gas pipeline" # today's pipe capacity diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 69d91b87e..dc5fb7cc6 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -137,7 +137,7 @@ def _add_missing_carriers_from_costs(n, costs, carriers): n.import_components_from_dataframe(emissions, "Carrier") -def load_costs(tech_costs, params, elec_params, Nyears=1.0): +def load_costs(tech_costs, params, max_hours, Nyears=1.0): # set all asset costs and other parameters costs = pd.read_csv(tech_costs, index_col=[0, 1]).sort_index() @@ -180,7 +180,6 @@ def costs_for_storage(store, link1, link2=None, max_hours=1.0): dict(capital_cost=capital_cost, marginal_cost=0.0, co2_emissions=0.0) ) - max_hours = elec_params["max_hours"] costs.loc["battery"] = costs_for_storage( costs.loc["battery storage"], costs.loc["battery inverter"], @@ -728,7 +727,7 @@ def add_nice_carrier_names(n, config): costs = load_costs( snakemake.input.tech_costs, snakemake.params["costs"], - snakemake.params["electricity"], + snakemake.params["electricity"]["max_hours"], Nyears, ) ppl = load_powerplants(snakemake.input.powerplants) @@ -759,10 +758,10 @@ def add_nice_carrier_names(n, config): snakemake.input.load, snakemake.input.nuts3_shapes, snakemake.params["countries"], - snakemake.params["load"]["scaling_factor"], + snakemake.params["scaling_factor"], ) - update_transmission_costs(n, costs, snakemake.params["lines"]["length_factor"]) + update_transmission_costs(n, costs, snakemake.params["length_factor"]) conventional_inputs = { k: v for k, v in snakemake.input.items() if k.startswith("conventional_") @@ -783,7 +782,7 @@ def add_nice_carrier_names(n, config): snakemake.input, renewable_carriers, extendable_carriers, - snakemake.params["lines"]["length_factor"], + snakemake.params["length_factor"], ) if "hydro" in renewable_carriers: diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index a4d3748b8..bc1d3a056 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -615,7 +615,7 @@ def add_heating_capacities_installed_before_baseyear( options = snakemake.params["sector"] opts = snakemake.wildcards.sector_opts.split("-") - baseyear = snakemake.params["scenario"]["planning_horizons"][0] + baseyear = snakemake.params["baseyear"] overrides = override_component_attrs(snakemake.input.overrides) n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index 08178c0a7..ce4e533ee 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -67,9 +67,8 @@ logger = logging.getLogger(__name__) -def attach_storageunits(n, costs, elec_opts): - carriers = elec_opts["extendable_carriers"]["StorageUnit"] - max_hours = elec_opts["max_hours"] +def attach_storageunits(n, costs, ext_carriers, max_hours): + carriers = ext_carriers["StorageUnit"] _add_missing_carriers_from_costs(n, costs, carriers) @@ -99,8 +98,8 @@ def attach_storageunits(n, costs, elec_opts): ) -def attach_stores(n, costs, elec_opts): - carriers = elec_opts["extendable_carriers"]["Store"] +def attach_stores(n, costs, ext_carriers): + carriers = ext_carriers["Store"] _add_missing_carriers_from_costs(n, costs, carriers) @@ -187,8 +186,7 @@ def attach_stores(n, costs, elec_opts): ) -def attach_hydrogen_pipelines(n, costs, elec_opts): - ext_carriers = elec_opts["extendable_carriers"] +def attach_hydrogen_pipelines(n, costs, ext_carriers): as_stores = ext_carriers.get("Store", []) if "H2 pipeline" not in ext_carriers.get("Link", []): @@ -235,16 +233,17 @@ def attach_hydrogen_pipelines(n, costs, elec_opts): configure_logging(snakemake) n = pypsa.Network(snakemake.input.network) - elec_config = snakemake.params["electricity"] + ext_carriers = snakemake.params["ext_carriers"] + max_hours = snakemake.params["max_hours"] Nyears = n.snapshot_weightings.objective.sum() / 8760.0 costs = load_costs( - snakemake.input.tech_costs, snakemake.params["costs"], elec_config, Nyears + snakemake.input.tech_costs, snakemake.params["costs"], max_hours, Nyears ) - attach_storageunits(n, costs, elec_config) - attach_stores(n, costs, elec_config) - attach_hydrogen_pipelines(n, costs, elec_config) + attach_storageunits(n, costs, ext_carriers, max_hours) + attach_stores(n, costs, ext_carriers) + attach_hydrogen_pipelines(n, costs, ext_carriers) add_nice_carrier_names(n, snakemake.config) diff --git a/scripts/build_cop_profiles.py b/scripts/build_cop_profiles.py index 7128ec0d7..983eda2dd 100644 --- a/scripts/build_cop_profiles.py +++ b/scripts/build_cop_profiles.py @@ -39,7 +39,7 @@ def coefficient_of_performance(delta_T, source="air"): for source in ["air", "soil"]: source_T = xr.open_dataarray(snakemake.input[f"temp_{source}_{area}"]) - delta_T = snakemake.params["sector"]["heat_pump_sink_T"] - source_T + delta_T = snakemake.params["heat_pump_sink_T"] - source_T cop = coefficient_of_performance(delta_T, source) diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index 2f61f0174..798588d9d 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -106,7 +106,7 @@ snakemake = mock_snakemake("build_cutout", cutout="europe-2013-era5") configure_logging(snakemake) - cutout_params = snakemake.params["atlite"]["cutouts"][snakemake.wildcards.cutout] + cutout_params = snakemake.params["cutouts"][snakemake.wildcards.cutout] snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"]) time = [snapshots[0], snapshots[-1]] diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index dd686be38..26bf31c66 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -130,7 +130,7 @@ def get_eia_annual_hydro_generation(fn, countries): snakemake = mock_snakemake("build_hydro_profile") configure_logging(snakemake) - params_hydro = snakemake.params["renewable"]["hydro"] + params_hydro = snakemake.params["hydro"] cutout = atlite.Cutout(snakemake.input.cutout) countries = snakemake.params["countries"] diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index 9a513673b..a51ff8b26 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -73,7 +73,7 @@ def prepare_hotmaps_database(regions): df[["srid", "coordinates"]] = df.geom.str.split(";", expand=True) - if snakemake.params["industry"].get("hotmaps_locate_missing", False): + if snakemake.params["hotmaps_locate_missing"]: df = locate_missing_industrial_sites(df) # remove those sites without valid locations diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py index 2ec007a91..fa9e5c188 100644 --- a/scripts/build_industry_sector_ratios.py +++ b/scripts/build_industry_sector_ratios.py @@ -439,7 +439,7 @@ def chemicals_industry(): sector = "Ammonia" df[sector] = 0.0 - if snakemake.params["sector"].get("ammonia", False): + if snakemake.params["sector_amonia"]: df.loc["ammonia", sector] = params["MWh_NH3_per_tNH3"] else: df.loc["hydrogen", sector] = params["MWh_H2_per_tNH3_electrolysis"] diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index bd0ee74e9..6edd4ac48 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -134,12 +134,12 @@ def replace_natural_gas_fueltype(df): ppl = ppl.query('not (Country in @available_countries and Fueltype == "Bioenergy")') ppl = pd.concat([ppl, opsd]) - ppl_query = snakemake.params["electricity"]["powerplants_filter"] + ppl_query = snakemake.params["powerplants_filter"] if isinstance(ppl_query, str): ppl.query(ppl_query, inplace=True) # add carriers from own powerplant files: - custom_ppl_query = snakemake.params["electricity"]["custom_powerplants"] + custom_ppl_query = snakemake.params["custom_powerplants"] ppl = add_custom_powerplants( ppl, snakemake.input.custom_powerplants, custom_ppl_query ) diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py index c52d4eb6d..8473be7a2 100644 --- a/scripts/build_retro_cost.py +++ b/scripts/build_retro_cost.py @@ -1040,7 +1040,7 @@ def sample_dE_costs_area( # ******** config ********************************************************* - retro_opts = snakemake.params["sector"]["retrofitting"] + retro_opts = snakemake.params["retrofitting"] interest_rate = retro_opts["interest_rate"] annualise_cost = retro_opts["annualise_cost"] # annualise the investment costs tax_weighting = retro_opts[ diff --git a/scripts/build_sequestration_potentials.py b/scripts/build_sequestration_potentials.py index 5c388b2e8..9d26b0b9f 100644 --- a/scripts/build_sequestration_potentials.py +++ b/scripts/build_sequestration_potentials.py @@ -41,7 +41,7 @@ def allocate_sequestration_potential( "build_sequestration_potentials", simpl="", clusters="181" ) - cf = snakemake.params["sector"]["regional_co2_sequestration_potential"] + cf = snakemake.params["co2seq_potential"] gdf = gpd.read_file(snakemake.input.sequestration_potential[0]) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index d217e5ed8..3cfbd2146 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -479,7 +479,7 @@ def plot_busmap_for_n_clusters(n, n_clusters, fn=None): if snakemake.wildcards.clusters.endswith("m"): n_clusters = int(snakemake.wildcards.clusters[:-1]) conventional = set( - snakemake.params["electricity"].get("conventional_carriers", []) + snakemake.params["conventional_carriers"] ) aggregate_carriers = conventional.intersection(aggregate_carriers) elif snakemake.wildcards.clusters == "all": @@ -495,13 +495,13 @@ def plot_busmap_for_n_clusters(n, n_clusters, fn=None): n, busmap, linemap, linemap, pd.Series(dtype="O") ) else: - line_length_factor = snakemake.params["lines"]["length_factor"] + line_length_factor = snakemake.params["length_factor"] Nyears = n.snapshot_weightings.objective.sum() / 8760 hvac_overhead_cost = load_costs( snakemake.input.tech_costs, snakemake.params["costs"], - snakemake.params["electricity"], + snakemake.params["max_hours"], Nyears, ).at["HVAC overhead", "capital_cost"] @@ -539,7 +539,7 @@ def consense(x): aggregate_carriers, line_length_factor, aggregation_strategies, - snakemake.params["solving"]["solver"]["name"], + snakemake.params["solver_name"], cluster_config.get("algorithm", "hac"), cluster_config.get("feature", "solar+onwind-time"), hvac_overhead_cost, diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index 4aa37de5e..36f75207a 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -455,7 +455,7 @@ def plot_carbon_budget_distribution(input_eurostat): ax1 = plt.subplot(gs1[0, 0]) ax1.set_ylabel("CO$_2$ emissions (Gt per year)", fontsize=22) ax1.set_ylim([0, 5]) - ax1.set_xlim([1990, snakemake.params["scenario"]["planning_horizons"][-1] + 1]) + ax1.set_xlim([1990, snakemake.params["planning_horizons"][-1] + 1]) path_cb = "results/" + snakemake.params.RDIR + "/csvs/" countries = snakemake.params["countries"] @@ -555,7 +555,7 @@ def plot_carbon_budget_distribution(input_eurostat): plot_balances() - for sector_opts in snakemake.params["scenario"]["sector_opts"]: + for sector_opts in snakemake.params["sector_opts"]: opts = sector_opts.split("-") for o in opts: if "cb" in o: diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 46e4e74aa..51777ef56 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -254,7 +254,7 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): costs = load_costs( snakemake.input.tech_costs, snakemake.params["costs"], - snakemake.params["electricity"], + snakemake.params["max_hours"], Nyears, ) @@ -269,7 +269,7 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): for o in opts: m = re.match(r"^\d+seg$", o, re.IGNORECASE) if m is not None: - solver_name = snakemake.params["solving"]["solver"]["name"] + solver_name = snakemake.params["solver_name"] n = apply_time_segmentation(n, m.group(0)[:-3], solver_name) break @@ -277,11 +277,11 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): if "Co2L" in o: m = re.findall("[0-9]*\.?[0-9]+$", o) if len(m) > 0: - co2limit = float(m[0]) * snakemake.params["electricity"]["co2base"] + co2limit = float(m[0]) * snakemake.params["co2base"] add_co2limit(n, co2limit, Nyears) logger.info("Setting CO2 limit according to wildcard value.") else: - add_co2limit(n, snakemake.params["electricity"]["co2limit"], Nyears) + add_co2limit(n, snakemake.params["co2limit"], Nyears) logger.info("Setting CO2 limit according to config value.") break @@ -293,7 +293,7 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): add_gaslimit(n, limit, Nyears) logger.info("Setting gas usage limit according to wildcard value.") else: - add_gaslimit(n, snakemake.params["electricity"].get("gaslimit"), Nyears) + add_gaslimit(n, snakemake.params["gaslimit"], Nyears) logger.info("Setting gas usage limit according to config value.") break diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 6a71b1e2a..54ab1f944 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -252,7 +252,7 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year): countries, input_eurostat, opts, emissions_scope, report_year, year=2018 ) - planning_horizons = snakemake.params["scenario"]["planning_horizons"] + planning_horizons = snakemake.params["planning_horizons"] t_0 = planning_horizons[0] if "be" in o: @@ -391,7 +391,7 @@ def update_wind_solar_costs(n, costs): with xr.open_dataset(profile) as ds: underwater_fraction = ds["underwater_fraction"].to_pandas() connection_cost = ( - snakemake.params["lines"]["length_factor"] + snakemake.params["length_factor"] * ds["average_distance"].to_pandas() * ( underwater_fraction @@ -3300,7 +3300,7 @@ def set_temporal_aggregation(n, opts, solver_name): if snakemake.params["foresight"] == "myopic": add_lifetime_wind_solar(n, costs) - conventional = snakemake.params["existing_capacities"]["conventional_carriers"] + conventional = snakemake.params["conventional_carriers"] for carrier in conventional: add_carrier_buses(n, carrier) @@ -3365,7 +3365,7 @@ def set_temporal_aggregation(n, opts, solver_name): if options["allam_cycle"]: add_allam(n, costs) - solver_name = snakemake.params["solving"]["solver"]["name"] + solver_name = snakemake.params["solver_name"] n = set_temporal_aggregation(n, opts, solver_name) limit_type = "config" @@ -3376,8 +3376,8 @@ def set_temporal_aggregation(n, opts, solver_name): limit_type = "carbon budget" fn = "results/" + snakemake.params.RDIR + "/csvs/carbon_budget_distribution.csv" if not os.path.exists(fn): - emissions_scope = snakemake.params["energy"]["emissions"] - report_year = snakemake.params["energy"]["eurostat_report_year"] + emissions_scope = snakemake.params["emissions_scope"] + report_year = snakemake.params["report_year"] build_carbon_budget( o, snakemake.input.eurostat, fn, emissions_scope, report_year ) @@ -3413,7 +3413,7 @@ def set_temporal_aggregation(n, opts, solver_name): add_electricity_grid_connection(n, costs) first_year_myopic = (snakemake.params["foresight"] == "myopic") and ( - snakemake.params["scenario"]["planning_horizons"][0] == investment_year + snakemake.params["planning_horizons"][0] == investment_year ) if options.get("cluster_heat_buses", False) and not first_year_myopic: diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 689e80843..83e932a3f 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -560,7 +560,7 @@ def cluster( technology_costs = load_costs( snakemake.input.tech_costs, snakemake.params["costs"], - snakemake.params["electricity"], + snakemake.params["max_hours"], Nyears, ) diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index c268e2ee5..421bc515f 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -41,7 +41,7 @@ opts = (snakemake.wildcards.opts + "-" + snakemake.wildcards.sector_opts).split("-") opts = [o for o in opts if o != ""] - solve_opts = snakemake.params["solving"]["options"] + solve_opts = snakemake.params["options"] np.random.seed(solve_opts.get("seed", 123)) From f3e8fe23122bb47e2d8dfb4c8264e920007ee20b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 27 May 2023 13:50:58 +0000 Subject: [PATCH 14/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/build_sector.smk | 100 ++++++++++++++++++++++++------------- scripts/cluster_network.py | 4 +- 2 files changed, 65 insertions(+), 39 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index da8a60f9f..2d6c82630 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -406,24 +406,38 @@ rule build_ammonia_production: rule build_industry_sector_ratios: params: industry={ - "H2_DRI":config["industry"]["H2_DRI"], - "elec_DRI":config["industry"]["elec_DRI"], - "HVC_production_today":config["industry"]["HVC_production_today"], - "petrochemical_process_emissions":config["industry"]["petrochemical_process_emissions"], - "NH3_process_emissions":config["industry"]["NH3_process_emissions"], - "MWh_CH4_per_tNH3_SMR":config["industry"]["MWh_CH4_per_tNH3_SMR"], - "MWh_elec_per_tNH3_SMR":config["industry"]["MWh_elec_per_tNH3_SMR"], - "chlorine_production_today":config["industry"]["chlorine_production_today"], - "MWh_H2_per_tCl":config["industry"]["MWh_H2_per_tCl"], - "MWh_elec_per_tCl":config["industry"]["MWh_elec_per_tCl"], - "methanol_production_today":config["industry"]["methanol_production_today"], - "MWh_CH4_per_tMeOH":config["industry"]["MWh_CH4_per_tMeOH"], - "MWh_elec_per_tMeOH":config["industry"]["MWh_elec_per_tMeOH"], - "MWh_elec_per_tHVC_mechanical_recycling":config["industry"]["MWh_elec_per_tHVC_mechanical_recycling"], - "MWh_elec_per_tHVC_chemical_recycling":config["industry"]["MWh_elec_per_tHVC_chemical_recycling"], - "MWh_NH3_per_tNH3":config["industry"]["MWh_NH3_per_tNH3"], - "MWh_H2_per_tNH3_electrolysis":config["industry"]["MWh_H2_per_tNH3_electrolysis"], - "MWh_elec_per_tNH3_electrolysis":config["industry"]["MWh_elec_per_tNH3_electrolysis"] + "H2_DRI": config["industry"]["H2_DRI"], + "elec_DRI": config["industry"]["elec_DRI"], + "HVC_production_today": config["industry"]["HVC_production_today"], + "petrochemical_process_emissions": config["industry"][ + "petrochemical_process_emissions" + ], + "NH3_process_emissions": config["industry"]["NH3_process_emissions"], + "MWh_CH4_per_tNH3_SMR": config["industry"]["MWh_CH4_per_tNH3_SMR"], + "MWh_elec_per_tNH3_SMR": config["industry"]["MWh_elec_per_tNH3_SMR"], + "chlorine_production_today": config["industry"][ + "chlorine_production_today" + ], + "MWh_H2_per_tCl": config["industry"]["MWh_H2_per_tCl"], + "MWh_elec_per_tCl": config["industry"]["MWh_elec_per_tCl"], + "methanol_production_today": config["industry"][ + "methanol_production_today" + ], + "MWh_CH4_per_tMeOH": config["industry"]["MWh_CH4_per_tMeOH"], + "MWh_elec_per_tMeOH": config["industry"]["MWh_elec_per_tMeOH"], + "MWh_elec_per_tHVC_mechanical_recycling": config["industry"][ + "MWh_elec_per_tHVC_mechanical_recycling" + ], + "MWh_elec_per_tHVC_chemical_recycling": config["industry"][ + "MWh_elec_per_tHVC_chemical_recycling" + ], + "MWh_NH3_per_tNH3": config["industry"]["MWh_NH3_per_tNH3"], + "MWh_H2_per_tNH3_electrolysis": config["industry"][ + "MWh_H2_per_tNH3_electrolysis" + ], + "MWh_elec_per_tNH3_electrolysis": config["industry"][ + "MWh_elec_per_tNH3_electrolysis" + ], }, sector_amonia=config["sector"].get("ammonia", False), input: @@ -447,10 +461,14 @@ rule build_industry_sector_ratios: rule build_industrial_production_per_country: params: industry={ - "reference_year":config["industry"]["reference_year"], - "HVC_production_today":config["industry"]["HVC_production_today"], - "chlorine_production_today":config["industry"]["chlorine_production_today"], - "methanol_production_today":config["industry"]["methanol_production_today"] + "reference_year": config["industry"]["reference_year"], + "HVC_production_today": config["industry"]["HVC_production_today"], + "chlorine_production_today": config["industry"][ + "chlorine_production_today" + ], + "methanol_production_today": config["industry"][ + "methanol_production_today" + ], }, countries=config["countries"], input: @@ -476,12 +494,16 @@ rule build_industrial_production_per_country: rule build_industrial_production_per_country_tomorrow: params: industry={ - "St_primary_fraction":config["industry"]["St_primary_fraction"], - "DRI_fraction":config["industry"]["DRI_fraction"], - "Al_primary_fraction":config["industry"]["Al_primary_fraction"], - "HVC_mechanical_recycling_fraction":config["industry"]["HVC_mechanical_recycling_fraction"], - "HVC_chemical_recycling_fraction":config["industry"]["HVC_chemical_recycling_fraction"], - "HVC_primary_fraction":config["industry"]["HVC_primary_fraction"] + "St_primary_fraction": config["industry"]["St_primary_fraction"], + "DRI_fraction": config["industry"]["DRI_fraction"], + "Al_primary_fraction": config["industry"]["Al_primary_fraction"], + "HVC_mechanical_recycling_fraction": config["industry"][ + "HVC_mechanical_recycling_fraction" + ], + "HVC_chemical_recycling_fraction": config["industry"][ + "HVC_chemical_recycling_fraction" + ], + "HVC_primary_fraction": config["industry"]["HVC_primary_fraction"], }, input: industrial_production_per_country=RESOURCES @@ -587,10 +609,10 @@ rule build_industrial_energy_demand_per_country_today: params: countries=config["countries"], industry={ - "reference_year":config["industry"].get("reference_year", 2015), - "MWh_CH4_per_tNH3_SMR":config["industry"]["MWh_CH4_per_tNH3_SMR"], - "MWh_elec_per_tNH3_SMR":config["industry"]["MWh_elec_per_tNH3_SMR"], - "MWh_NH3_per_tNH3":config["industry"]["MWh_NH3_per_tNH3"] + "reference_year": config["industry"].get("reference_year", 2015), + "MWh_CH4_per_tNH3_SMR": config["industry"]["MWh_CH4_per_tNH3_SMR"], + "MWh_elec_per_tNH3_SMR": config["industry"]["MWh_elec_per_tNH3_SMR"], + "MWh_NH3_per_tNH3": config["industry"]["MWh_NH3_per_tNH3"], }, input: jrc="data/jrc-idees-2015", @@ -747,10 +769,16 @@ rule prepare_sector_network: costs=config["costs"], sector=config["sector"], industry={ - "MWh_elec_per_tNH3_electrolysis":config["industry"]["MWh_elec_per_tNH3_electrolysis"], - "MWh_NH3_per_tNH3":config["industry"]["MWh_NH3_per_tNH3"], - "MWh_H2_per_tNH3_electrolysis":config["industry"]["MWh_H2_per_tNH3_electrolysis"], - "MWh_NH3_per_MWh_H2_cracker":config["industry"]["MWh_NH3_per_MWh_H2_cracker"] + "MWh_elec_per_tNH3_electrolysis": config["industry"][ + "MWh_elec_per_tNH3_electrolysis" + ], + "MWh_NH3_per_tNH3": config["industry"]["MWh_NH3_per_tNH3"], + "MWh_H2_per_tNH3_electrolysis": config["industry"][ + "MWh_H2_per_tNH3_electrolysis" + ], + "MWh_NH3_per_MWh_H2_cracker": config["industry"][ + "MWh_NH3_per_MWh_H2_cracker" + ], }, pypsa_eur=config["pypsa_eur"], length_factor=config["lines"]["length_factor"], diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 3cfbd2146..b86bb2257 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -478,9 +478,7 @@ def plot_busmap_for_n_clusters(n, n_clusters, fn=None): aggregate_carriers = set(n.generators.carrier) - set(exclude_carriers) if snakemake.wildcards.clusters.endswith("m"): n_clusters = int(snakemake.wildcards.clusters[:-1]) - conventional = set( - snakemake.params["conventional_carriers"] - ) + conventional = set(snakemake.params["conventional_carriers"]) aggregate_carriers = conventional.intersection(aggregate_carriers) elif snakemake.wildcards.clusters == "all": n_clusters = len(n.buses) From b67a00903af6b5d55019da50bd845a217e07d628 Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Wed, 31 May 2023 10:54:27 +0200 Subject: [PATCH 15/24] using higher keys for industries --- rules/build_electricity.smk | 1 - rules/build_sector.smk | 79 +++---------------------------------- 2 files changed, 5 insertions(+), 75 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 9b3a03fb1..34276ba4a 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -255,7 +255,6 @@ rule build_hydro_profile: params: hydro=config["renewable"]["hydro"], countries=config["countries"], - renewable=config["renewable"], input: country_shapes=RESOURCES + "country_shapes.geojson", eia_hydro_generation="data/eia_hydro_annual_generation.csv", diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 2d6c82630..3714cbad5 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -405,40 +405,7 @@ rule build_ammonia_production: rule build_industry_sector_ratios: params: - industry={ - "H2_DRI": config["industry"]["H2_DRI"], - "elec_DRI": config["industry"]["elec_DRI"], - "HVC_production_today": config["industry"]["HVC_production_today"], - "petrochemical_process_emissions": config["industry"][ - "petrochemical_process_emissions" - ], - "NH3_process_emissions": config["industry"]["NH3_process_emissions"], - "MWh_CH4_per_tNH3_SMR": config["industry"]["MWh_CH4_per_tNH3_SMR"], - "MWh_elec_per_tNH3_SMR": config["industry"]["MWh_elec_per_tNH3_SMR"], - "chlorine_production_today": config["industry"][ - "chlorine_production_today" - ], - "MWh_H2_per_tCl": config["industry"]["MWh_H2_per_tCl"], - "MWh_elec_per_tCl": config["industry"]["MWh_elec_per_tCl"], - "methanol_production_today": config["industry"][ - "methanol_production_today" - ], - "MWh_CH4_per_tMeOH": config["industry"]["MWh_CH4_per_tMeOH"], - "MWh_elec_per_tMeOH": config["industry"]["MWh_elec_per_tMeOH"], - "MWh_elec_per_tHVC_mechanical_recycling": config["industry"][ - "MWh_elec_per_tHVC_mechanical_recycling" - ], - "MWh_elec_per_tHVC_chemical_recycling": config["industry"][ - "MWh_elec_per_tHVC_chemical_recycling" - ], - "MWh_NH3_per_tNH3": config["industry"]["MWh_NH3_per_tNH3"], - "MWh_H2_per_tNH3_electrolysis": config["industry"][ - "MWh_H2_per_tNH3_electrolysis" - ], - "MWh_elec_per_tNH3_electrolysis": config["industry"][ - "MWh_elec_per_tNH3_electrolysis" - ], - }, + industry=config["industry"], sector_amonia=config["sector"].get("ammonia", False), input: ammonia_production=RESOURCES + "ammonia_production.csv", @@ -460,16 +427,7 @@ rule build_industry_sector_ratios: rule build_industrial_production_per_country: params: - industry={ - "reference_year": config["industry"]["reference_year"], - "HVC_production_today": config["industry"]["HVC_production_today"], - "chlorine_production_today": config["industry"][ - "chlorine_production_today" - ], - "methanol_production_today": config["industry"][ - "methanol_production_today" - ], - }, + industry=config["industry"], countries=config["countries"], input: ammonia_production=RESOURCES + "ammonia_production.csv", @@ -493,18 +451,7 @@ rule build_industrial_production_per_country: rule build_industrial_production_per_country_tomorrow: params: - industry={ - "St_primary_fraction": config["industry"]["St_primary_fraction"], - "DRI_fraction": config["industry"]["DRI_fraction"], - "Al_primary_fraction": config["industry"]["Al_primary_fraction"], - "HVC_mechanical_recycling_fraction": config["industry"][ - "HVC_mechanical_recycling_fraction" - ], - "HVC_chemical_recycling_fraction": config["industry"][ - "HVC_chemical_recycling_fraction" - ], - "HVC_primary_fraction": config["industry"]["HVC_primary_fraction"], - }, + industry=config["industry"], input: industrial_production_per_country=RESOURCES + "industrial_production_per_country.csv", @@ -608,12 +555,7 @@ rule build_industrial_energy_demand_per_node: rule build_industrial_energy_demand_per_country_today: params: countries=config["countries"], - industry={ - "reference_year": config["industry"].get("reference_year", 2015), - "MWh_CH4_per_tNH3_SMR": config["industry"]["MWh_CH4_per_tNH3_SMR"], - "MWh_elec_per_tNH3_SMR": config["industry"]["MWh_elec_per_tNH3_SMR"], - "MWh_NH3_per_tNH3": config["industry"]["MWh_NH3_per_tNH3"], - }, + industry=config["industry"], input: jrc="data/jrc-idees-2015", ammonia_production=RESOURCES + "ammonia_production.csv", @@ -768,18 +710,7 @@ rule prepare_sector_network: foresight=config["foresight"], costs=config["costs"], sector=config["sector"], - industry={ - "MWh_elec_per_tNH3_electrolysis": config["industry"][ - "MWh_elec_per_tNH3_electrolysis" - ], - "MWh_NH3_per_tNH3": config["industry"]["MWh_NH3_per_tNH3"], - "MWh_H2_per_tNH3_electrolysis": config["industry"][ - "MWh_H2_per_tNH3_electrolysis" - ], - "MWh_NH3_per_MWh_H2_cracker": config["industry"][ - "MWh_NH3_per_MWh_H2_cracker" - ], - }, + industry=config["industry"], pypsa_eur=config["pypsa_eur"], length_factor=config["lines"]["length_factor"], planning_horizons=config["scenario"]["planning_horizons"], From 83ff639ad57f40f7a61027951843cd7e4b334642 Mon Sep 17 00:00:00 2001 From: virio-andreyana Date: Fri, 2 Jun 2023 12:52:49 +0200 Subject: [PATCH 16/24] amend solve_network --- rules/solve_electricity.smk | 13 +++---------- rules/solve_myopic.smk | 13 +++---------- rules/solve_overnight.smk | 13 +++---------- scripts/solve_network.py | 25 +++++++++++++++---------- 4 files changed, 24 insertions(+), 40 deletions(-) diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index 892415c4e..e7b264f76 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -6,16 +6,9 @@ rule solve_network: params: solving=config["solving"], - config_parts={ - "foresight": config["foresight"], - "planning_horizons": config["scenario"]["planning_horizons"], - "co2_sequestration_potential": config["sector"].get( - "co2_sequestration_potential", 200 - ), - "H2_retrofit_capacity_per_CH4": config["sector"][ - "H2_retrofit_capacity_per_CH4" - ], - }, + foresight=config["foresight"], + planning_horizons=config["scenario"]["planning_horizons"], + co2_sequestration_potential=config["sector"].get("co2_sequestration_potential", 200), input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 8ec1c4a8f..39567b92c 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -85,16 +85,9 @@ ruleorder: add_existing_baseyear > add_brownfield rule solve_sector_network_myopic: params: solving=config["solving"], - config_parts={ - "foresight": config["foresight"], - "planning_horizons": config["scenario"]["planning_horizons"], - "co2_sequestration_potential": config["sector"].get( - "co2_sequestration_potential", 200 - ), - "H2_retrofit_capacity_per_CH4": config["sector"][ - "H2_retrofit_capacity_per_CH4" - ], - }, + foresight=config["foresight"], + planning_horizons=config["scenario"]["planning_horizons"], + co2_sequestration_potential=config["sector"].get("co2_sequestration_potential", 200), input: overrides="data/override_component_attrs", network=RESULTS diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index a523f1325..71ef075e0 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -6,16 +6,9 @@ rule solve_sector_network: params: solving=config["solving"], - config_parts={ - "foresight": config["foresight"], - "planning_horizons": config["scenario"]["planning_horizons"], - "co2_sequestration_potential": config["sector"].get( - "co2_sequestration_potential", 200 - ), - "H2_retrofit_capacity_per_CH4": config["sector"][ - "H2_retrofit_capacity_per_CH4" - ], - }, + foresight=config["foresight"], + planning_horizons=config["scenario"]["planning_horizons"], + co2_sequestration_potential=config["sector"].get("co2_sequestration_potential", 200), input: overrides="data/override_component_attrs", network=RESULTS diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 6e07e340b..90ff3a2e8 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -44,14 +44,14 @@ from pypsa.descriptors import get_switchable_as_dense as get_as_dense -def add_land_use_constraint(n, param, config): +def add_land_use_constraint(n, planning_horizons_param, config): if "m" in snakemake.wildcards.clusters: - _add_land_use_constraint_m(n, param, config) + _add_land_use_constraint_m(n, planning_horizons_param, config) else: - _add_land_use_constraint(n, param) + _add_land_use_constraint(n) -def _add_land_use_constraint(n, param): +def _add_land_use_constraint(n): # warning: this will miss existing offwind which is not classed AC-DC and has carrier 'offwind' for carrier in ["solar", "onwind", "offwind-ac", "offwind-dc"]: @@ -80,7 +80,7 @@ def _add_land_use_constraint(n, param): n.generators.p_nom_max.clip(lower=0, inplace=True) -def _add_land_use_constraint_m(n, param, config): +def _add_land_use_constraint_m(n, planning_horizons_param, config): # if generators clustering is lower than network clustering, land_use accounting is at generators clusters planning_horizons = param["planning_horizons"] @@ -141,7 +141,7 @@ def add_co2_sequestration_limit(n, limit=200): ) -def prepare_network(n, solve_opts=None, config=None, param=None): +def prepare_network(n, solve_opts=None, config=None, foresight_param=None, planning_horizons_param=None, co2_sequestration_potential=None): if "clip_p_max_pu" in solve_opts: for df in ( n.generators_t.p_max_pu, @@ -191,11 +191,11 @@ def prepare_network(n, solve_opts=None, config=None, param=None): n.set_snapshots(n.snapshots[:nhours]) n.snapshot_weightings[:] = 8760.0 / nhours - if param["foresight"] == "myopic": - add_land_use_constraint(n, param, config) + if foresight_param == "myopic": + add_land_use_constraint(n, planning_horizons_param, config) if n.stores.carrier.eq("co2 stored").any(): - limit = param["co2_sequestration_potential"] + limit = co2_sequestration_potential_param add_co2_sequestration_limit(n, limit=limit) return n @@ -676,7 +676,12 @@ def solve_network(n, config, solving_param, opts="", **kwargs): n = pypsa.Network(snakemake.input.network) n = prepare_network( - n, solve_opts, config=snakemake.config, param=snakemake.params["config_parts"] + n, + solve_opts, + config=snakemake.config, + foresight_param=snakemake.params["foresight"], + planning_horizons_param=snakemake.params["planning_horizons"], + co2_sequestration_potential=snakemake.params["co2_sequestration_potential"] ) n = solve_network( From 0f5bb8b893afc4955f608d9a206035f63e2bb6fe Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 2 Jun 2023 10:53:09 +0000 Subject: [PATCH 17/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/solve_electricity.smk | 4 +++- rules/solve_myopic.smk | 4 +++- rules/solve_overnight.smk | 4 +++- scripts/solve_network.py | 11 +++++++++-- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index e7b264f76..f73a5b0c5 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -8,7 +8,9 @@ rule solve_network: solving=config["solving"], foresight=config["foresight"], planning_horizons=config["scenario"]["planning_horizons"], - co2_sequestration_potential=config["sector"].get("co2_sequestration_potential", 200), + co2_sequestration_potential=config["sector"].get( + "co2_sequestration_potential", 200 + ), input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", output: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 39567b92c..086375c2f 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -87,7 +87,9 @@ rule solve_sector_network_myopic: solving=config["solving"], foresight=config["foresight"], planning_horizons=config["scenario"]["planning_horizons"], - co2_sequestration_potential=config["sector"].get("co2_sequestration_potential", 200), + co2_sequestration_potential=config["sector"].get( + "co2_sequestration_potential", 200 + ), input: overrides="data/override_component_attrs", network=RESULTS diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index 71ef075e0..fc430f4d4 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -8,7 +8,9 @@ rule solve_sector_network: solving=config["solving"], foresight=config["foresight"], planning_horizons=config["scenario"]["planning_horizons"], - co2_sequestration_potential=config["sector"].get("co2_sequestration_potential", 200), + co2_sequestration_potential=config["sector"].get( + "co2_sequestration_potential", 200 + ), input: overrides="data/override_component_attrs", network=RESULTS diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 90ff3a2e8..ab1175abc 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -141,7 +141,14 @@ def add_co2_sequestration_limit(n, limit=200): ) -def prepare_network(n, solve_opts=None, config=None, foresight_param=None, planning_horizons_param=None, co2_sequestration_potential=None): +def prepare_network( + n, + solve_opts=None, + config=None, + foresight_param=None, + planning_horizons_param=None, + co2_sequestration_potential=None, +): if "clip_p_max_pu" in solve_opts: for df in ( n.generators_t.p_max_pu, @@ -681,7 +688,7 @@ def solve_network(n, config, solving_param, opts="", **kwargs): config=snakemake.config, foresight_param=snakemake.params["foresight"], planning_horizons_param=snakemake.params["planning_horizons"], - co2_sequestration_potential=snakemake.params["co2_sequestration_potential"] + co2_sequestration_potential=snakemake.params["co2_sequestration_potential"], ) n = solve_network( From 8dbcca560cd47d58d098dd91e751efd45314b8e7 Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Sat, 3 Jun 2023 12:09:51 +0200 Subject: [PATCH 18/24] minor fixes in solve_network.py --- scripts/solve_network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index ab1175abc..ea5446ad5 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -147,7 +147,7 @@ def prepare_network( config=None, foresight_param=None, planning_horizons_param=None, - co2_sequestration_potential=None, + co2_sequestration_potential_param=None, ): if "clip_p_max_pu" in solve_opts: for df in ( @@ -688,7 +688,7 @@ def solve_network(n, config, solving_param, opts="", **kwargs): config=snakemake.config, foresight_param=snakemake.params["foresight"], planning_horizons_param=snakemake.params["planning_horizons"], - co2_sequestration_potential=snakemake.params["co2_sequestration_potential"], + co2_sequestration_potential_param=snakemake.params["co2_sequestration_potential"], ) n = solve_network( From d6057cb92be6fdb07d02da23e13035b90a57d1fc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 3 Jun 2023 10:10:06 +0000 Subject: [PATCH 19/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/solve_network.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index ea5446ad5..41679a035 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -688,7 +688,9 @@ def solve_network(n, config, solving_param, opts="", **kwargs): config=snakemake.config, foresight_param=snakemake.params["foresight"], planning_horizons_param=snakemake.params["planning_horizons"], - co2_sequestration_potential_param=snakemake.params["co2_sequestration_potential"], + co2_sequestration_potential_param=snakemake.params[ + "co2_sequestration_potential" + ], ) n = solve_network( From 12b93562453ccce90df248c2686040c984a1612b Mon Sep 17 00:00:00 2001 From: virio-andreyana Date: Wed, 14 Jun 2023 11:03:07 +0200 Subject: [PATCH 20/24] add info to release_notes --- doc/release_notes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 3af16477a..8f26ac02d 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -10,6 +10,7 @@ Release Notes Upcoming Release ================ +* ``param:`` section in rule definition are added to track changed settings in ``config.yaml``. The goal is to automatically re-execute rules whose parameters have changed. See `Non-file parameters for rules `_ in the snakemake documentation. * **Important:** The configuration files are now located in the ``config`` directory. This counts for ``config.default.yaml``, ``config.yaml`` as well as the test configuration files which are now located in ``config/test``. Config files that are still in the root directory will be ignored. From 45cac01ea3c49529c4b68714e783bedb60565f00 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 14 Jun 2023 13:28:20 +0200 Subject: [PATCH 21/24] adjust param accessing in cluster, simplify and build_elec script --- rules/build_electricity.smk | 24 +++--- scripts/add_electricity.py | 40 ++++----- scripts/build_renewable_profiles.py | 2 +- scripts/cluster_network.py | 56 +++++------- scripts/simplify_network.py | 129 ++++++++++++---------------- 5 files changed, 108 insertions(+), 143 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 34276ba4a..fd66cc043 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -316,16 +316,14 @@ rule add_electricity: rule simplify_network: params: - clustering=config["clustering"], - max_hours=config["electricity"]["max_hours"], + simplify_network=config["clustering"]["simplify_network"], + aggregation_strategies=config["clustering"].get("aggregation_strategies", {}), + focus_weights=config.get("focus_weights", None), + renewable_carriers=config["electricity"]["renewable_carriers"], costs=config["costs"], - renewable=config["renewable"], + max_hours=config["electricity"]["max_hours"], length_factor=config["lines"]["length_factor"], p_max_pu=config["links"].get("p_max_pu", 1.0), - exclude_carriers=config["clustering"]["simplify_network"].get( - "exclude_carriers", [] - ), - focus_weights=config.get("focus_weights", None), solver_name=config["solving"]["solver"]["name"], input: network=RESOURCES + "networks/elec.nc", @@ -353,14 +351,16 @@ rule simplify_network: rule cluster_network: params: - solver_name=config["solving"]["solver"]["name"], - max_hours=config["electricity"]["max_hours"], + cluster_network=config["clustering"]["cluster_network"], + aggregation_strategies=config["clustering"].get("aggregation_strategies", {}), + custom_busmap=config["enable"].get("custom_busmap", False), + focus_weights=config.get("focus_weights", None), + renewable_carriers=config["electricity"]["renewable_carriers"], conventional_carriers=config["electricity"].get("conventional_carriers", []), costs=config["costs"], + max_hours=config["electricity"]["max_hours"], length_factor=config["lines"]["length_factor"], - renewable=config["renewable"], - clustering=config["clustering"], - custom_busmap=config["enable"].get("custom_busmap", False), + solver_name=config["solving"]["solver"]["name"], input: network=RESOURCES + "networks/elec_s{simpl}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson", diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index dc5fb7cc6..34147af3b 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -726,23 +726,23 @@ def add_nice_carrier_names(n, config): costs = load_costs( snakemake.input.tech_costs, - snakemake.params["costs"], - snakemake.params["electricity"]["max_hours"], + snakemake.params.costs, + snakemake.params.electricity["max_hours"], Nyears, ) ppl = load_powerplants(snakemake.input.powerplants) - if "renewable_carriers" in snakemake.params["electricity"]: - renewable_carriers = set(snakemake.params["electricity"]["renewable_carriers"]) + if "renewable_carriers" in snakemake.params.electricity: + renewable_carriers = set(snakemake.params.electricity["renewable_carriers"]) else: logger.warning( "Missing key `renewable_carriers` under config entry `electricity`. " "In future versions, this will raise an error. " "Falling back to carriers listed under `renewable`." ) - renewable_carriers = snakemake.params["renewable"] + renewable_carriers = snakemake.params.renewable - extendable_carriers = snakemake.params["electricity"]["extendable_carriers"] + extendable_carriers = snakemake.params.electricity["extendable_carriers"] if not (set(renewable_carriers) & set(extendable_carriers["Generator"])): logger.warning( "No renewables found in config entry `extendable_carriers`. " @@ -750,18 +750,18 @@ def add_nice_carrier_names(n, config): "Falling back to all renewables." ) - conventional_carriers = snakemake.params["electricity"]["conventional_carriers"] + conventional_carriers = snakemake.params.electricity["conventional_carriers"] attach_load( n, snakemake.input.regions, snakemake.input.load, snakemake.input.nuts3_shapes, - snakemake.params["countries"], - snakemake.params["scaling_factor"], + snakemake.params.countries, + snakemake.params.scaling_factor, ) - update_transmission_costs(n, costs, snakemake.params["length_factor"]) + update_transmission_costs(n, costs, snakemake.params.length_factor) conventional_inputs = { k: v for k, v in snakemake.input.items() if k.startswith("conventional_") @@ -772,7 +772,7 @@ def add_nice_carrier_names(n, config): ppl, conventional_carriers, extendable_carriers, - snakemake.params["conventional"], + snakemake.params.conventional, conventional_inputs, ) @@ -782,11 +782,11 @@ def add_nice_carrier_names(n, config): snakemake.input, renewable_carriers, extendable_carriers, - snakemake.params["length_factor"], + snakemake.params.length_factor, ) if "hydro" in renewable_carriers: - para = snakemake.params["renewable"]["hydro"] + para = snakemake.params.renewable["hydro"] attach_hydro( n, costs, @@ -797,7 +797,7 @@ def add_nice_carrier_names(n, config): **para, ) - if "estimate_renewable_capacities" not in snakemake.params["electricity"]: + if "estimate_renewable_capacities" not in snakemake.params.electricity: logger.warning( "Missing key `estimate_renewable_capacities` under config entry `electricity`. " "In future versions, this will raise an error. " @@ -805,18 +805,18 @@ def add_nice_carrier_names(n, config): ) if ( "estimate_renewable_capacities_from_capacity_stats" - in snakemake.params["electricity"] + in snakemake.params.electricity ): estimate_renewable_caps = { "enable": True, - **snakemake.params["electricity"][ + **snakemake.params.electricity[ "estimate_renewable_capacities_from_capacity_stats" ], } else: estimate_renewable_caps = {"enable": False} else: - estimate_renewable_caps = snakemake.params["electricity"][ + estimate_renewable_caps = snakemake.params.electricity[ "estimate_renewable_capacities" ] if "enable" not in estimate_renewable_caps: @@ -832,18 +832,18 @@ def add_nice_carrier_names(n, config): "Falling back to whether `renewable_capacities_from_opsd` is non-empty." ) from_opsd = bool( - snakemake.params["electricity"].get("renewable_capacities_from_opsd", False) + snakemake.params.electricity.get("renewable_capacities_from_opsd", False) ) estimate_renewable_caps["from_opsd"] = from_opsd if estimate_renewable_caps["enable"]: if estimate_renewable_caps["from_opsd"]: - tech_map = snakemake.params["electricity"]["estimate_renewable_capacities"][ + tech_map = snakemake.params.electricity["estimate_renewable_capacities"][ "technology_mapping" ] attach_OPSD_renewables(n, tech_map) estimate_renewable_capacities( - n, snakemake.params["electricity"], snakemake.params["countries"] + n, snakemake.params.electricity, snakemake.params.countries ) update_p_nom_max(n) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 91463c23a..9e90d42ba 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -309,7 +309,7 @@ p_nom_max = capacities / max_cap_factor else: raise AssertionError( - 'params key `potential` should be one of "simple" ' + 'Config key `potential` should be one of "simple" ' f'(default) or "conservative", not "{p_nom_max_meth}"' ) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index db0fb2c27..52685af24 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -186,7 +186,7 @@ def get_feature_for_hac(n, buses_i=None, feature=None): if "offwind" in carriers: carriers.remove("offwind") carriers = np.append( - carriers, network.generators.carrier.filter(like="offwind").unique() + carriers, n.generators.carrier.filter(like="offwind").unique() ) if feature.split("-")[1] == "cap": @@ -463,26 +463,18 @@ def plot_busmap_for_n_clusters(n, n_clusters, fn=None): snakemake = mock_snakemake("cluster_network", simpl="", clusters="5") configure_logging(snakemake) - n = pypsa.Network(snakemake.input.network) - - focus_weights = snakemake.config.get("focus_weights", None) + params = snakemake.params + solver_name = snakemake.config["solving"]["solver"]["name"] - renewable_carriers = pd.Index( - [ - tech - for tech in n.generators.carrier.unique() - if tech in snakemake.params["renewable"] - ] - ) + n = pypsa.Network(snakemake.input.network) - exclude_carriers = snakemake.params["clustering"]["cluster_network"].get( - "exclude_carriers", [] - ) + exclude_carriers = params.cluster_network["exclude_carriers"] aggregate_carriers = set(n.generators.carrier) - set(exclude_carriers) if snakemake.wildcards.clusters.endswith("m"): n_clusters = int(snakemake.wildcards.clusters[:-1]) - conventional = set(snakemake.params["conventional_carriers"]) - aggregate_carriers = conventional.intersection(aggregate_carriers) + aggregate_carriers = set(params.conventional_carriers).intersection( + aggregate_carriers + ) elif snakemake.wildcards.clusters == "all": n_clusters = len(n.buses) else: @@ -496,13 +488,12 @@ def plot_busmap_for_n_clusters(n, n_clusters, fn=None): n, busmap, linemap, linemap, pd.Series(dtype="O") ) else: - line_length_factor = snakemake.params["length_factor"] Nyears = n.snapshot_weightings.objective.sum() / 8760 hvac_overhead_cost = load_costs( snakemake.input.tech_costs, - snakemake.params["costs"], - snakemake.params["max_hours"], + params.costs, + params.max_hours, Nyears, ).at["HVAC overhead", "capital_cost"] @@ -513,16 +504,16 @@ def consense(x): ).all() or x.isnull().all(), "The `potential` configuration option must agree for all renewable carriers, for now!" return v - aggregation_strategies = snakemake.params["clustering"].get( - "aggregation_strategies", {} - ) # translate str entries of aggregation_strategies to pd.Series functions: aggregation_strategies = { - p: {k: getattr(pd.Series, v) for k, v in aggregation_strategies[p].items()} - for p in aggregation_strategies.keys() + p: { + k: getattr(pd.Series, v) + for k, v in params.aggregation_strategies[p].items() + } + for p in params.aggregation_strategies.keys() } - custom_busmap = snakemake.params["custom_busmap"] + custom_busmap = params.custom_busmap if custom_busmap: custom_busmap = pd.read_csv( snakemake.input.custom_busmap, index_col=0, squeeze=True @@ -530,21 +521,18 @@ def consense(x): custom_busmap.index = custom_busmap.index.astype(str) logger.info(f"Imported custom busmap from {snakemake.input.custom_busmap}") - cluster_config = snakemake.config.get("clustering", {}).get( - "cluster_network", {} - ) clustering = clustering_for_n_clusters( n, n_clusters, custom_busmap, aggregate_carriers, - line_length_factor, - aggregation_strategies, - snakemake.params["solver_name"], - cluster_config.get("algorithm", "hac"), - cluster_config.get("feature", "solar+onwind-time"), + params.length_factor, + params.aggregation_strategies, + solver_name, + params.cluster_network["algorithm"], + params.cluster_network["feature"], hvac_overhead_cost, - focus_weights, + params.focus_weights, ) update_p_nom_max(clustering.network) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 83e932a3f..79161760a 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -149,17 +149,17 @@ def simplify_network_to_380(n): return n, trafo_map -def _prepare_connection_costs_per_link(n, costs, renewable_param, length_factor_param): +def _prepare_connection_costs_per_link(n, costs, renewable_carriers, length_factor): if n.links.empty: return {} connection_costs_per_link = {} - for tech in renewable_param: + for tech in renewable_carriers: if tech.startswith("offwind"): connection_costs_per_link[tech] = ( n.links.length - * length_factor_param + * length_factor * ( n.links.underwater_fraction * costs.at[tech + "-connection-submarine", "capital_cost"] @@ -175,14 +175,14 @@ def _compute_connection_costs_to_bus( n, busmap, costs, - renewable_param, - length_factor_param, + renewable_carriers, + length_factor, connection_costs_per_link=None, buses=None, ): if connection_costs_per_link is None: connection_costs_per_link = _prepare_connection_costs_per_link( - n, costs, renewable_param, length_factor_param + n, costs, renewable_carriers, length_factor ) if buses is None: @@ -276,10 +276,10 @@ def replace_components(n, c, df, pnl): def simplify_links( n, costs, - renewable_param, - length_factor_param, - p_max_pu_param, - exclude_carriers_param, + renewables, + length_factor, + p_max_pu, + exclude_carriers, output, aggregation_strategies=dict(), ): @@ -333,7 +333,7 @@ def split_links(nodes): busmap = n.buses.index.to_series() connection_costs_per_link = _prepare_connection_costs_per_link( - n, costs, renewable_param, length_factor_param + n, costs, renewables, length_factor ) connection_costs_to_bus = pd.DataFrame( 0.0, index=n.buses.index, columns=list(connection_costs_per_link) @@ -355,8 +355,8 @@ def split_links(nodes): n, busmap, costs, - renewable_param, - length_factor_param, + renewables, + length_factor, connection_costs_per_link, buses, ) @@ -378,8 +378,8 @@ def split_links(nodes): / lengths.sum() * n.links.loc[all_links, "underwater_fraction"] ), - p_max_pu=p_max_pu_param, - p_min_pu=-p_max_pu_param, + p_max_pu=p_max_pu, + p_min_pu=-p_max_pu, underground=False, under_construction=False, ) @@ -407,7 +407,7 @@ def split_links(nodes): connection_costs_to_bus, output, aggregation_strategies=aggregation_strategies, - exclude_carriers=exclude_carriers_param, + exclude_carriers=exclude_carriers, ) return n, busmap @@ -415,34 +415,29 @@ def split_links(nodes): def remove_stubs( n, costs, - renewable_param, - length_factor_param, - clustering_param, - exclude_carriers_param, + renewable_carriers, + length_factor, + simplify_network, output, aggregation_strategies=dict(), ): logger.info("Removing stubs") - across_borders = clustering_param["simplify_network"].get( - "remove_stubs_across_borders", True - ) + across_borders = simplify_network["remove_stubs_across_borders"] matching_attrs = [] if across_borders else ["country"] busmap = busmap_by_stubs(n, matching_attrs) connection_costs_to_bus = _compute_connection_costs_to_bus( - n, busmap, costs, renewable_param, length_factor_param + n, busmap, costs, renewable_carriers, length_factor ) - exclude_carriers = clustering_param["simplify_network"].get("exclude_carriers", []) - _aggregate_and_move_components( n, busmap, connection_costs_to_bus, output, aggregation_strategies=aggregation_strategies, - exclude_carriers=exclude_carriers, + exclude_carriers=simplify_network["exclude_carriers"], ) return n, busmap @@ -504,32 +499,23 @@ def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None): def cluster( n, n_clusters, - focus_weights_param, - renewable_param, - solver_name_param, + focus_weights, + solver_name, algorithm="hac", feature=None, aggregation_strategies=dict(), ): logger.info(f"Clustering to {n_clusters} buses") - renewable_carriers = pd.Index( - [ - tech - for tech in n.generators.carrier.unique() - if tech.split("-", 2)[0] in renewable_param - ] - ) - clustering = clustering_for_n_clusters( n, n_clusters, custom_busmap=False, aggregation_strategies=aggregation_strategies, - solver_name=solver_name_param, + solver_name=solver_name, algorithm=algorithm, feature=feature, - focus_weights=focus_weights_param, + focus_weights=focus_weights, ) return clustering.network, clustering.busmap @@ -542,77 +528,69 @@ def cluster( snakemake = mock_snakemake("simplify_network", simpl="") configure_logging(snakemake) + params = snakemake.params + solver_name = snakemake.config["solving"]["solver"]["name"] + n = pypsa.Network(snakemake.input.network) + Nyears = n.snapshot_weightings.objective.sum() / 8760 - aggregation_strategies = snakemake.params["clustering"].get( - "aggregation_strategies", {} - ) # translate str entries of aggregation_strategies to pd.Series functions: aggregation_strategies = { - p: {k: getattr(pd.Series, v) for k, v in aggregation_strategies[p].items()} - for p in aggregation_strategies.keys() + p: { + k: getattr(pd.Series, v) + for k, v in params.aggregation_strategies[p].items() + } + for p in params.aggregation_strategies.keys() } n, trafo_map = simplify_network_to_380(n) - Nyears = n.snapshot_weightings.objective.sum() / 8760 - technology_costs = load_costs( snakemake.input.tech_costs, - snakemake.params["costs"], - snakemake.params["max_hours"], + params.costs, + params.max_hours, Nyears, ) n, simplify_links_map = simplify_links( n, technology_costs, - snakemake.params["renewable"], - snakemake.params["length_factor"], - snakemake.params["p_max_pu"], - snakemake.params["exclude_carriers"], + params.renewable_carriers, + params.length_factor, + params.p_max_pu, + params.simplify_network["exclude_carriers"], snakemake.output, aggregation_strategies, ) busmaps = [trafo_map, simplify_links_map] - cluster_param = snakemake.params["clustering"]["simplify_network"] - if cluster_param.get("remove_stubs", True): + if params.simplify_network["remove_stubs"]: n, stub_map = remove_stubs( n, technology_costs, - snakemake.params["renewable"], - snakemake.params["length_factor"], - snakemake.params["clustering"], - snakemake.params["exclude_carriers"], + params.renewable_carriers, + params.length_factor, + params.simplify_network, snakemake.output, aggregation_strategies=aggregation_strategies, ) busmaps.append(stub_map) - if cluster_param.get("to_substations", False): + if params.simplify_network["to_substations"]: n, substation_map = aggregate_to_substations(n, aggregation_strategies) busmaps.append(substation_map) # treatment of outliers (nodes without a profile for considered carrier): # all nodes that have no profile of the given carrier are being aggregated to closest neighbor - if ( - snakemake.config.get("clustering", {}) - .get("cluster_network", {}) - .get("algorithm", "hac") - == "hac" - or cluster_param.get("algorithm", "hac") == "hac" - ): - carriers = ( - cluster_param.get("feature", "solar+onwind-time").split("-")[0].split("+") - ) + if params.simplify_network["algorithm"] == "hac": + carriers = params.simplify_network["feature"].split("-")[0].split("+") for carrier in carriers: buses_i = list( set(n.buses.index) - set(n.generators.query("carrier == @carrier").bus) ) logger.info( - f"clustering preparaton (hac): aggregating {len(buses_i)} buses of type {carrier}." + f"clustering preparation (hac): aggregating {len(buses_i)} buses of type {carrier}." ) n, busmap_hac = aggregate_to_substations(n, aggregation_strategies, buses_i) busmaps.append(busmap_hac) @@ -621,11 +599,10 @@ def cluster( n, cluster_map = cluster( n, int(snakemake.wildcards.simpl), - snakemake.params["focus_weights"], - snakemake.params["renewable"], - snakemake.params["solver_name"], - cluster_param.get("algorithm", "hac"), - cluster_param.get("feature", None), + params.focus_weights, + solver_name, + params.simplify_network["algorithm"], + params.simplify_network["feature"], aggregation_strategies, ) busmaps.append(cluster_map) From 1d10073514a66f09e6821364f03a57f99cff2352 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 15 Jun 2023 18:52:25 +0200 Subject: [PATCH 22/24] review params implementation; reproduce objective values in CI --- config/test/config.myopic.yaml | 8 ++ config/test/config.overnight.yaml | 8 ++ rules/build_electricity.smk | 2 +- rules/build_sector.smk | 2 +- scripts/add_brownfield.py | 6 +- scripts/add_electricity.py | 93 +++++-------------- scripts/add_existing_baseyear.py | 20 ++-- scripts/add_extra_components.py | 26 +++--- scripts/build_ammonia_production.py | 2 +- scripts/build_biomass_potentials.py | 2 +- scripts/build_bus_regions.py | 2 +- scripts/build_cop_profiles.py | 2 +- scripts/build_cutout.py | 4 +- scripts/build_electricity_demand.py | 12 +-- scripts/build_energy_totals.py | 8 +- scripts/build_heat_demand.py | 2 +- scripts/build_hydro_profile.py | 4 +- scripts/build_industrial_distribution_key.py | 4 +- ...ustrial_energy_demand_per_country_today.py | 4 +- ...build_industrial_production_per_country.py | 6 +- ...ustrial_production_per_country_tomorrow.py | 2 +- scripts/build_industry_sector_ratios.py | 4 +- scripts/build_powerplants.py | 6 +- scripts/build_renewable_profiles.py | 2 +- scripts/build_retro_cost.py | 4 +- scripts/build_sequestration_potentials.py | 2 +- scripts/build_shapes.py | 6 +- scripts/build_solar_thermal_profiles.py | 4 +- scripts/build_temperature_profiles.py | 2 +- scripts/build_transport_demand.py | 4 +- scripts/make_summary.py | 20 ++-- scripts/plot_network.py | 16 ++-- scripts/plot_summary.py | 24 ++--- scripts/prepare_network.py | 20 ++-- scripts/prepare_sector_network.py | 40 ++++---- scripts/retrieve_databundle.py | 2 +- scripts/solve_network.py | 42 ++++----- scripts/solve_operations_network.py | 2 +- 38 files changed, 188 insertions(+), 231 deletions(-) diff --git a/config/test/config.myopic.yaml b/config/test/config.myopic.yaml index efa031366..0bb85ec6c 100644 --- a/config/test/config.myopic.yaml +++ b/config/test/config.myopic.yaml @@ -31,6 +31,14 @@ snapshots: end: "2013-03-08" electricity: + co2limit: 100.e+6 + + extendable_carriers: + Generator: [OCGT] + StorageUnit: [battery] + Store: [H2] + Link: [H2 pipeline] + renewable_carriers: [solar, onwind, offwind-ac, offwind-dc] atlite: diff --git a/config/test/config.overnight.yaml b/config/test/config.overnight.yaml index fb468ded9..a2a0f5a46 100644 --- a/config/test/config.overnight.yaml +++ b/config/test/config.overnight.yaml @@ -28,6 +28,14 @@ snapshots: end: "2013-03-08" electricity: + co2limit: 100.e+6 + + extendable_carriers: + Generator: [OCGT] + StorageUnit: [battery] + Store: [H2] + Link: [H2 pipeline] + renewable_carriers: [solar, onwind, offwind-ac, offwind-dc] atlite: diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index fd66cc043..9d4315df1 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -394,7 +394,7 @@ rule cluster_network: rule add_extra_components: params: costs=config["costs"], - ext_carriers=config["electricity"]["extendable_carriers"], + extendable_carriers=config["electricity"]["extendable_carriers"], max_hours=config["electricity"]["max_hours"], input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}.nc", diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 3714cbad5..e15394ffb 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -406,7 +406,7 @@ rule build_ammonia_production: rule build_industry_sector_ratios: params: industry=config["industry"], - sector_amonia=config["sector"].get("ammonia", False), + ammonia=config["sector"].get("ammonia", False), input: ammonia_production=RESOURCES + "ammonia_production.csv", idees="data/jrc-idees-2015", diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 48dffbb92..b33d2f195 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -49,7 +49,7 @@ def add_brownfield(n, n_p, year): ) ] - threshold = snakemake.params["threshold_capacity"] + threshold = snakemake.params.threshold_capacity if not chp_heat.empty: threshold_chp_heat = ( @@ -87,7 +87,7 @@ def add_brownfield(n, n_p, year): # deal with gas network pipe_carrier = ["gas pipeline"] - if snakemake.params["H2_retrofit"]: + if snakemake.params.H2_retrofit: # drop capacities of previous year to avoid duplicating to_drop = n.links.carrier.isin(pipe_carrier) & (n.links.build_year != year) n.mremove("Link", n.links.loc[to_drop].index) @@ -98,7 +98,7 @@ def add_brownfield(n, n_p, year): & (n.links.build_year != year) ].index gas_pipes_i = n.links[n.links.carrier.isin(pipe_carrier)].index - CH4_per_H2 = 1 / snakemake.params["H2_retrofit_capacity_per_CH4"] + CH4_per_H2 = 1 / snakemake.params.H2_retrofit_capacity_per_CH4 fr = "H2 pipeline retrofitted" to = "gas pipeline" # today's pipe capacity diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 34147af3b..d98dc7671 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -652,13 +652,7 @@ def attach_OPSD_renewables(n, tech_map): n.generators.p_nom_min.update(gens.bus.map(caps).dropna()) -def estimate_renewable_capacities(n, electricity_params, countries): - year = electricity_params["estimate_renewable_capacities"]["year"] - tech_map = electricity_params["estimate_renewable_capacities"]["technology_mapping"] - expansion_limit = electricity_params["estimate_renewable_capacities"][ - "expansion_limit" - ] - +def estimate_renewable_capacities(n, year, tech_map, expansion_limit, countries): if not len(countries) or not len(tech_map): return @@ -721,48 +715,42 @@ def add_nice_carrier_names(n, config): snakemake = mock_snakemake("add_electricity") configure_logging(snakemake) + params = snakemake.params + n = pypsa.Network(snakemake.input.base_network) Nyears = n.snapshot_weightings.objective.sum() / 8760.0 costs = load_costs( snakemake.input.tech_costs, - snakemake.params.costs, - snakemake.params.electricity["max_hours"], + params.costs, + params.electricity["max_hours"], Nyears, ) ppl = load_powerplants(snakemake.input.powerplants) - if "renewable_carriers" in snakemake.params.electricity: - renewable_carriers = set(snakemake.params.electricity["renewable_carriers"]) + if "renewable_carriers" in params.electricity: + renewable_carriers = set(params.electricity["renewable_carriers"]) else: logger.warning( "Missing key `renewable_carriers` under config entry `electricity`. " "In future versions, this will raise an error. " "Falling back to carriers listed under `renewable`." ) - renewable_carriers = snakemake.params.renewable - - extendable_carriers = snakemake.params.electricity["extendable_carriers"] - if not (set(renewable_carriers) & set(extendable_carriers["Generator"])): - logger.warning( - "No renewables found in config entry `extendable_carriers`. " - "In future versions, these have to be explicitly listed. " - "Falling back to all renewables." - ) - - conventional_carriers = snakemake.params.electricity["conventional_carriers"] + renewable_carriers = params.renewable attach_load( n, snakemake.input.regions, snakemake.input.load, snakemake.input.nuts3_shapes, - snakemake.params.countries, - snakemake.params.scaling_factor, + params.countries, + params.scaling_factor, ) - update_transmission_costs(n, costs, snakemake.params.length_factor) + update_transmission_costs(n, costs, params.length_factor) + extendable_carriers = params.electricity["extendable_carriers"] + conventional_carriers = params.electricity["conventional_carriers"] conventional_inputs = { k: v for k, v in snakemake.input.items() if k.startswith("conventional_") } @@ -772,7 +760,7 @@ def add_nice_carrier_names(n, config): ppl, conventional_carriers, extendable_carriers, - snakemake.params.conventional, + params.conventional, conventional_inputs, ) @@ -782,11 +770,11 @@ def add_nice_carrier_names(n, config): snakemake.input, renewable_carriers, extendable_carriers, - snakemake.params.length_factor, + params.length_factor, ) if "hydro" in renewable_carriers: - para = snakemake.params.renewable["hydro"] + para = params.renewable["hydro"] attach_hydro( n, costs, @@ -797,53 +785,16 @@ def add_nice_carrier_names(n, config): **para, ) - if "estimate_renewable_capacities" not in snakemake.params.electricity: - logger.warning( - "Missing key `estimate_renewable_capacities` under config entry `electricity`. " - "In future versions, this will raise an error. " - "Falling back to whether ``estimate_renewable_capacities_from_capacity_stats`` is in the config." - ) - if ( - "estimate_renewable_capacities_from_capacity_stats" - in snakemake.params.electricity - ): - estimate_renewable_caps = { - "enable": True, - **snakemake.params.electricity[ - "estimate_renewable_capacities_from_capacity_stats" - ], - } - else: - estimate_renewable_caps = {"enable": False} - else: - estimate_renewable_caps = snakemake.params.electricity[ - "estimate_renewable_capacities" - ] - if "enable" not in estimate_renewable_caps: - logger.warning( - "Missing key `enable` under config entry `estimate_renewable_capacities`. " - "In future versions, this will raise an error. Falling back to False." - ) - estimate_renewable_caps = {"enable": False} - if "from_opsd" not in estimate_renewable_caps: - logger.warning( - "Missing key `from_opsd` under config entry `estimate_renewable_capacities`. " - "In future versions, this will raise an error. " - "Falling back to whether `renewable_capacities_from_opsd` is non-empty." - ) - from_opsd = bool( - snakemake.params.electricity.get("renewable_capacities_from_opsd", False) - ) - estimate_renewable_caps["from_opsd"] = from_opsd - + estimate_renewable_caps = params.electricity["estimate_renewable_capacities"] if estimate_renewable_caps["enable"]: + tech_map = estimate_renewable_caps["technology_mapping"] + expansion_limit = estimate_renewable_caps["expansion_limit"] + year = estimate_renewable_caps["year"] + if estimate_renewable_caps["from_opsd"]: - tech_map = snakemake.params.electricity["estimate_renewable_capacities"][ - "technology_mapping" - ] attach_OPSD_renewables(n, tech_map) estimate_renewable_capacities( - n, snakemake.params.electricity, snakemake.params.countries + n, year, tech_map, expansion_limit, params.countries ) update_p_nom_max(n) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index bc1d3a056..52cb585ed 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -157,7 +157,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas # Fill missing DateOut dateout = ( df_agg.loc[biomass_i, "DateIn"] - + snakemake.params["costs"]["fill_values"]["lifetime"] + + snakemake.params.costs["fill_values"]["lifetime"] ) df_agg.loc[biomass_i, "DateOut"] = df_agg.loc[biomass_i, "DateOut"].fillna(dateout) @@ -218,7 +218,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas capacity = df.loc[grouping_year, generator] capacity = capacity[~capacity.isna()] capacity = capacity[ - capacity > snakemake.params["existing_capacities"]["threshold_capacity"] + capacity > snakemake.params.existing_capacities["threshold_capacity"] ] suffix = "-ac" if generator == "offwind" else "" name_suffix = f" {generator}{suffix}-{grouping_year}" @@ -582,7 +582,7 @@ def add_heating_capacities_installed_before_baseyear( ) # delete links with capacities below threshold - threshold = snakemake.params["existing_capacities"]["threshold_capacity"] + threshold = snakemake.params.existing_capacities["threshold_capacity"] n.mremove( "Link", [ @@ -612,10 +612,10 @@ def add_heating_capacities_installed_before_baseyear( update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) - options = snakemake.params["sector"] + options = snakemake.params.sector opts = snakemake.wildcards.sector_opts.split("-") - baseyear = snakemake.params["baseyear"] + baseyear = snakemake.params.baseyear overrides = override_component_attrs(snakemake.input.overrides) n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) @@ -626,14 +626,12 @@ def add_heating_capacities_installed_before_baseyear( Nyears = n.snapshot_weightings.generators.sum() / 8760.0 costs = prepare_costs( snakemake.input.costs, - snakemake.params["costs"], + snakemake.params.costs, Nyears, ) - grouping_years_power = snakemake.params["existing_capacities"][ - "grouping_years_power" - ] - grouping_years_heat = snakemake.params["existing_capacities"]["grouping_years_heat"] + grouping_years_power = snakemake.params.existing_capacities["grouping_years_power"] + grouping_years_heat = snakemake.params.existing_capacities["grouping_years_heat"] add_power_capacities_installed_before_baseyear( n, grouping_years_power, costs, baseyear ) @@ -650,7 +648,7 @@ def add_heating_capacities_installed_before_baseyear( .to_pandas() .reindex(index=n.snapshots) ) - default_lifetime = snakemake.params["costs"]["fill_values"]["lifetime"] + default_lifetime = snakemake.params.costs["fill_values"]["lifetime"] add_heating_capacities_installed_before_baseyear( n, baseyear, diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index ce4e533ee..f034a7f2e 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -67,8 +67,8 @@ logger = logging.getLogger(__name__) -def attach_storageunits(n, costs, ext_carriers, max_hours): - carriers = ext_carriers["StorageUnit"] +def attach_storageunits(n, costs, extendable_carriers, max_hours): + carriers = extendable_carriers["StorageUnit"] _add_missing_carriers_from_costs(n, costs, carriers) @@ -98,8 +98,8 @@ def attach_storageunits(n, costs, ext_carriers, max_hours): ) -def attach_stores(n, costs, ext_carriers): - carriers = ext_carriers["Store"] +def attach_stores(n, costs, extendable_carriers): + carriers = extendable_carriers["Store"] _add_missing_carriers_from_costs(n, costs, carriers) @@ -186,10 +186,10 @@ def attach_stores(n, costs, ext_carriers): ) -def attach_hydrogen_pipelines(n, costs, ext_carriers): - as_stores = ext_carriers.get("Store", []) +def attach_hydrogen_pipelines(n, costs, extendable_carriers): + as_stores = extendable_carriers.get("Store", []) - if "H2 pipeline" not in ext_carriers.get("Link", []): + if "H2 pipeline" not in extendable_carriers.get("Link", []): return assert "H2" in as_stores, ( @@ -233,17 +233,17 @@ def attach_hydrogen_pipelines(n, costs, ext_carriers): configure_logging(snakemake) n = pypsa.Network(snakemake.input.network) - ext_carriers = snakemake.params["ext_carriers"] - max_hours = snakemake.params["max_hours"] + extendable_carriers = snakemake.params.extendable_carriers + max_hours = snakemake.params.max_hours Nyears = n.snapshot_weightings.objective.sum() / 8760.0 costs = load_costs( - snakemake.input.tech_costs, snakemake.params["costs"], max_hours, Nyears + snakemake.input.tech_costs, snakemake.params.costs, max_hours, Nyears ) - attach_storageunits(n, costs, ext_carriers, max_hours) - attach_stores(n, costs, ext_carriers) - attach_hydrogen_pipelines(n, costs, ext_carriers) + attach_storageunits(n, costs, extendable_carriers, max_hours) + attach_stores(n, costs, extendable_carriers) + attach_hydrogen_pipelines(n, costs, extendable_carriers) add_nice_carrier_names(n, snakemake.config) diff --git a/scripts/build_ammonia_production.py b/scripts/build_ammonia_production.py index 6f03324fd..1bcdf9ae5 100644 --- a/scripts/build_ammonia_production.py +++ b/scripts/build_ammonia_production.py @@ -30,7 +30,7 @@ ammonia.index = cc.convert(ammonia.index, to="iso2") years = [str(i) for i in range(2013, 2018)] - countries = ammonia.index.intersection(snakemake.params["countries"]) + countries = ammonia.index.intersection(snakemake.params.countries) ammonia = ammonia.loc[countries, years].astype(float) # convert from ktonN to ktonNH3 diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py index 0b423ad5f..d200a78e4 100644 --- a/scripts/build_biomass_potentials.py +++ b/scripts/build_biomass_potentials.py @@ -210,7 +210,7 @@ def convert_nuts2_to_regions(bio_nuts2, regions): snakemake = mock_snakemake("build_biomass_potentials", simpl="", clusters="5") - params = snakemake.params["biomass"] + params = snakemake.params.biomass year = params["year"] scenario = params["scenario"] diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index e93787921..a6500bb08 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -116,7 +116,7 @@ def voronoi_partition_pts(points, outline): snakemake = mock_snakemake("build_bus_regions") configure_logging(snakemake) - countries = snakemake.params["countries"] + countries = snakemake.params.countries n = pypsa.Network(snakemake.input.base_network) diff --git a/scripts/build_cop_profiles.py b/scripts/build_cop_profiles.py index 983eda2dd..4b1d952ea 100644 --- a/scripts/build_cop_profiles.py +++ b/scripts/build_cop_profiles.py @@ -39,7 +39,7 @@ def coefficient_of_performance(delta_T, source="air"): for source in ["air", "soil"]: source_T = xr.open_dataarray(snakemake.input[f"temp_{source}_{area}"]) - delta_T = snakemake.params["heat_pump_sink_T"] - source_T + delta_T = snakemake.params.heat_pump_sink_T - source_T cop = coefficient_of_performance(delta_T, source) diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index 798588d9d..9a7f9e008 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -106,9 +106,9 @@ snakemake = mock_snakemake("build_cutout", cutout="europe-2013-era5") configure_logging(snakemake) - cutout_params = snakemake.params["cutouts"][snakemake.wildcards.cutout] + cutout_params = snakemake.params.cutouts[snakemake.wildcards.cutout] - snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"]) + snapshots = pd.date_range(freq="h", **snakemake.params.snapshots) time = [snapshots[0], snapshots[-1]] cutout_params["time"] = slice(*cutout_params.get("time", time)) diff --git a/scripts/build_electricity_demand.py b/scripts/build_electricity_demand.py index 4ef56d1df..ba1fb8817 100755 --- a/scripts/build_electricity_demand.py +++ b/scripts/build_electricity_demand.py @@ -279,16 +279,16 @@ def manual_adjustment(load, fn_load, powerstatistics): configure_logging(snakemake) - powerstatistics = snakemake.params["load"]["power_statistics"] - interpolate_limit = snakemake.params["load"]["interpolate_limit"] - countries = snakemake.params["countries"] - snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"]) + powerstatistics = snakemake.params.load["power_statistics"] + interpolate_limit = snakemake.params.load["interpolate_limit"] + countries = snakemake.params.countries + snapshots = pd.date_range(freq="h", **snakemake.params.snapshots) years = slice(snapshots[0], snapshots[-1]) - time_shift = snakemake.params["load"]["time_shift_for_large_gaps"] + time_shift = snakemake.params.load["time_shift_for_large_gaps"] load = load_timeseries(snakemake.input[0], years, countries, powerstatistics) - if snakemake.params["load"]["manual_adjustments"]: + if snakemake.params.load["manual_adjustments"]: load = manual_adjustment(load, snakemake.input[0], powerstatistics) logger.info(f"Linearly interpolate gaps of size {interpolate_limit} and less.") diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 3e3cb4853..891c4e2a9 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -737,16 +737,16 @@ def build_transport_data(countries, population, idees): logging.basicConfig(level=snakemake.config["logging"]["level"]) - params = snakemake.params["energy"] + params = snakemake.params.energy nuts3 = gpd.read_file(snakemake.input.nuts3_shapes).set_index("index") population = nuts3["pop"].groupby(nuts3.country).sum() - countries = snakemake.params["countries"] + countries = snakemake.params.countries idees_countries = pd.Index(countries).intersection(eu28) data_year = params["energy_totals_year"] - report_year = snakemake.params["energy"]["eurostat_report_year"] + report_year = snakemake.params.energy["eurostat_report_year"] input_eurostat = snakemake.input.eurostat eurostat = build_eurostat(input_eurostat, countries, report_year, data_year) swiss = build_swiss(data_year) @@ -756,7 +756,7 @@ def build_transport_data(countries, population, idees): energy.to_csv(snakemake.output.energy_name) base_year_emissions = params["base_emissions_year"] - emissions_scope = snakemake.params["energy"]["emissions"] + emissions_scope = snakemake.params.energy["emissions"] eea_co2 = build_eea_co2(snakemake.input.co2, base_year_emissions, emissions_scope) eurostat_co2 = build_eurostat_co2( input_eurostat, countries, report_year, base_year_emissions diff --git a/scripts/build_heat_demand.py b/scripts/build_heat_demand.py index 655df28f4..734942600 100644 --- a/scripts/build_heat_demand.py +++ b/scripts/build_heat_demand.py @@ -27,7 +27,7 @@ cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) client = Client(cluster, asynchronous=True) - time = pd.date_range(freq="h", **snakemake.params["snapshots"]) + time = pd.date_range(freq="h", **snakemake.params.snapshots) cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) clustered_regions = ( diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index 26bf31c66..bed666f20 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -130,10 +130,10 @@ def get_eia_annual_hydro_generation(fn, countries): snakemake = mock_snakemake("build_hydro_profile") configure_logging(snakemake) - params_hydro = snakemake.params["hydro"] + params_hydro = snakemake.params.hydro cutout = atlite.Cutout(snakemake.input.cutout) - countries = snakemake.params["countries"] + countries = snakemake.params.countries country_shapes = ( gpd.read_file(snakemake.input.country_shapes) .set_index("name")["geometry"] diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index a51ff8b26..e93e43c2d 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -73,7 +73,7 @@ def prepare_hotmaps_database(regions): df[["srid", "coordinates"]] = df.geom.str.split(";", expand=True) - if snakemake.params["hotmaps_locate_missing"]: + if snakemake.params.hotmaps_locate_missing: df = locate_missing_industrial_sites(df) # remove those sites without valid locations @@ -143,7 +143,7 @@ def build_nodal_distribution_key(hotmaps, regions, countries): logging.basicConfig(level=snakemake.config["logging"]["level"]) - countries = snakemake.params["countries"] + countries = snakemake.params.countries regions = gpd.read_file(snakemake.input.regions_onshore).set_index("name") diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index 9f8c47d08..9ca0d003d 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -178,9 +178,9 @@ def industrial_energy_demand(countries, year): snakemake = mock_snakemake("build_industrial_energy_demand_per_country_today") - params = snakemake.params["industry"] + params = snakemake.params.industry year = params.get("reference_year", 2015) - countries = pd.Index(snakemake.params["countries"]) + countries = pd.Index(snakemake.params.countries) demand = industrial_energy_demand(countries.intersection(eu28), year) diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index 889c9ecd9..74cb19497 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -279,11 +279,11 @@ def separate_basic_chemicals(demand, year): logging.basicConfig(level=snakemake.config["logging"]["level"]) - countries = snakemake.params["countries"] + countries = snakemake.params.countries - year = snakemake.params["industry"]["reference_year"] + year = snakemake.params.industry["reference_year"] - params = snakemake.params["industry"] + params = snakemake.params.industry jrc_dir = snakemake.input.jrc eurostat_dir = snakemake.input.eurostat diff --git a/scripts/build_industrial_production_per_country_tomorrow.py b/scripts/build_industrial_production_per_country_tomorrow.py index 609170aa5..ffed51952 100644 --- a/scripts/build_industrial_production_per_country_tomorrow.py +++ b/scripts/build_industrial_production_per_country_tomorrow.py @@ -15,7 +15,7 @@ snakemake = mock_snakemake("build_industrial_production_per_country_tomorrow") - params = snakemake.params["industry"] + params = snakemake.params.industry investment_year = int(snakemake.wildcards.planning_horizons) diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py index fa9e5c188..457050026 100644 --- a/scripts/build_industry_sector_ratios.py +++ b/scripts/build_industry_sector_ratios.py @@ -439,7 +439,7 @@ def chemicals_industry(): sector = "Ammonia" df[sector] = 0.0 - if snakemake.params["sector_amonia"]: + if snakemake.params.ammonia: df.loc["ammonia", sector] = params["MWh_NH3_per_tNH3"] else: df.loc["hydrogen", sector] = params["MWh_H2_per_tNH3_electrolysis"] @@ -1468,7 +1468,7 @@ def other_industrial_sectors(): # TODO make params option year = 2015 - params = snakemake.params["industry"] + params = snakemake.params.industry df = pd.concat( [ diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index 6edd4ac48..7f396e1ed 100755 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -115,7 +115,7 @@ def replace_natural_gas_fueltype(df): configure_logging(snakemake) n = pypsa.Network(snakemake.input.base_network) - countries = snakemake.params["countries"] + countries = snakemake.params.countries ppl = ( pm.powerplants(from_url=True) @@ -134,12 +134,12 @@ def replace_natural_gas_fueltype(df): ppl = ppl.query('not (Country in @available_countries and Fueltype == "Bioenergy")') ppl = pd.concat([ppl, opsd]) - ppl_query = snakemake.params["powerplants_filter"] + ppl_query = snakemake.params.powerplants_filter if isinstance(ppl_query, str): ppl.query(ppl_query, inplace=True) # add carriers from own powerplant files: - custom_ppl_query = snakemake.params["custom_powerplants"] + custom_ppl_query = snakemake.params.custom_powerplants ppl = add_custom_powerplants( ppl, snakemake.input.custom_powerplants, custom_ppl_query ) diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 9e90d42ba..9f1f1a51a 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -204,7 +204,7 @@ nprocesses = int(snakemake.threads) noprogress = snakemake.config["run"].get("disable_progressbar", True) - params = snakemake.params["renewable"][snakemake.wildcards.technology] + params = snakemake.params.renewable[snakemake.wildcards.technology] resource = params["resource"] # pv panel params / wind turbine params correction_factor = params.get("correction_factor", 1.0) capacity_per_sqkm = params["capacity_per_sqkm"] diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py index ac7eb4ae5..c830415ed 100644 --- a/scripts/build_retro_cost.py +++ b/scripts/build_retro_cost.py @@ -305,7 +305,7 @@ def prepare_building_stock_data(): u_values.set_index(["country_code", "subsector", "bage", "type"], inplace=True) # only take in config.yaml specified countries into account - countries = snakemake.params["countries"] + countries = snakemake.params.countries area_tot = area_tot.loc[countries] return u_values, country_iso_dic, countries, area_tot, area @@ -1040,7 +1040,7 @@ def sample_dE_costs_area( # ******** config ********************************************************* - retro_opts = snakemake.params["retrofitting"] + retro_opts = snakemake.params.retrofitting interest_rate = retro_opts["interest_rate"] annualise_cost = retro_opts["annualise_cost"] # annualise the investment costs tax_weighting = retro_opts[ diff --git a/scripts/build_sequestration_potentials.py b/scripts/build_sequestration_potentials.py index 9d26b0b9f..0e59e55bc 100644 --- a/scripts/build_sequestration_potentials.py +++ b/scripts/build_sequestration_potentials.py @@ -41,7 +41,7 @@ def allocate_sequestration_potential( "build_sequestration_potentials", simpl="", clusters="181" ) - cf = snakemake.params["co2seq_potential"] + cf = snakemake.params.co2seq_potential gdf = gpd.read_file(snakemake.input.sequestration_potential[0]) diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 0c8b0a94c..eb8374092 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -255,13 +255,11 @@ def nuts3(country_shapes, nuts3, nuts3pop, nuts3gdp, ch_cantons, ch_popgdp): snakemake = mock_snakemake("build_shapes") configure_logging(snakemake) - country_shapes = countries( - snakemake.input.naturalearth, snakemake.params["countries"] - ) + country_shapes = countries(snakemake.input.naturalearth, snakemake.params.countries) country_shapes.reset_index().to_file(snakemake.output.country_shapes) offshore_shapes = eez( - country_shapes, snakemake.input.eez, snakemake.params["countries"] + country_shapes, snakemake.input.eez, snakemake.params.countries ) offshore_shapes.reset_index().to_file(snakemake.output.offshore_shapes) diff --git a/scripts/build_solar_thermal_profiles.py b/scripts/build_solar_thermal_profiles.py index 180007b74..d285691a4 100644 --- a/scripts/build_solar_thermal_profiles.py +++ b/scripts/build_solar_thermal_profiles.py @@ -27,9 +27,9 @@ cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) client = Client(cluster, asynchronous=True) - config = snakemake.params["solar_thermal"] + config = snakemake.params.solar_thermal - time = pd.date_range(freq="h", **snakemake.params["snapshots"]) + time = pd.date_range(freq="h", **snakemake.params.snapshots) cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) clustered_regions = ( diff --git a/scripts/build_temperature_profiles.py b/scripts/build_temperature_profiles.py index ee06aebb4..9db37c257 100644 --- a/scripts/build_temperature_profiles.py +++ b/scripts/build_temperature_profiles.py @@ -27,7 +27,7 @@ cluster = LocalCluster(n_workers=nprocesses, threads_per_worker=1) client = Client(cluster, asynchronous=True) - time = pd.date_range(freq="h", **snakemake.params["snapshots"]) + time = pd.date_range(freq="h", **snakemake.params.snapshots) cutout = atlite.Cutout(snakemake.input.cutout).sel(time=time) clustered_regions = ( diff --git a/scripts/build_transport_demand.py b/scripts/build_transport_demand.py index dc2b94b9d..c5bf46329 100644 --- a/scripts/build_transport_demand.py +++ b/scripts/build_transport_demand.py @@ -175,9 +175,9 @@ def bev_dsm_profile(snapshots, nodes, options): snakemake.input.pop_weighted_energy_totals, index_col=0 ) - options = snakemake.params["sector"] + options = snakemake.params.sector - snapshots = pd.date_range(freq="h", **snakemake.params["snapshots"], tz="UTC") + snapshots = pd.date_range(freq="h", **snakemake.params.snapshots, tz="UTC") nyears = len(snapshots) / 8760 diff --git a/scripts/make_summary.py b/scripts/make_summary.py index da0712d7a..00fca2fde 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -198,7 +198,7 @@ def calculate_costs(n, label, costs): def calculate_cumulative_cost(): - planning_horizons = snakemake.params["scenario"]["planning_horizons"] + planning_horizons = snakemake.params.scenario["planning_horizons"] cumulative_cost = pd.DataFrame( index=df["costs"].sum().index, @@ -688,19 +688,19 @@ def to_csv(df): (cluster, ll, opt + sector_opt, planning_horizon): "results/" + snakemake.params.RDIR + f"/postnetworks/elec_s{simpl}_{cluster}_l{ll}_{opt}_{sector_opt}_{planning_horizon}.nc" - for simpl in snakemake.params["scenario"]["simpl"] - for cluster in snakemake.params["scenario"]["clusters"] - for opt in snakemake.params["scenario"]["opts"] - for sector_opt in snakemake.params["scenario"]["sector_opts"] - for ll in snakemake.params["scenario"]["ll"] - for planning_horizon in snakemake.params["scenario"]["planning_horizons"] + for simpl in snakemake.params.scenario["simpl"] + for cluster in snakemake.params.scenario["clusters"] + for opt in snakemake.params.scenario["opts"] + for sector_opt in snakemake.params.scenario["sector_opts"] + for ll in snakemake.params.scenario["ll"] + for planning_horizon in snakemake.params.scenario["planning_horizons"] } - Nyears = len(pd.date_range(freq="h", **snakemake.params["snapshots"])) / 8760 + Nyears = len(pd.date_range(freq="h", **snakemake.params.snapshots)) / 8760 costs_db = prepare_costs( snakemake.input.costs, - snakemake.params["costs"], + snakemake.params.costs, Nyears, ) @@ -710,7 +710,7 @@ def to_csv(df): to_csv(df) - if snakemake.params["foresight"] == "myopic": + if snakemake.params.foresight == "myopic": cumulative_cost = calculate_cumulative_cost() cumulative_cost.to_csv( "results/" + snakemake.params.RDIR + "/csvs/cumulative_cost.csv" diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 399f46e87..184d86f0c 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -70,7 +70,7 @@ def plot_map( transmission=False, with_legend=True, ): - tech_colors = snakemake.params["plotting"]["tech_colors"] + tech_colors = snakemake.params.plotting["tech_colors"] n = network.copy() assign_location(n) @@ -116,9 +116,7 @@ def plot_map( costs = costs.stack() # .sort_index() # hack because impossible to drop buses... - eu_location = snakemake.params["plotting"].get( - "eu_node_location", dict(x=-5.5, y=46) - ) + eu_location = snakemake.params.plotting.get("eu_node_location", dict(x=-5.5, y=46)) n.buses.loc["EU gas", "x"] = eu_location["x"] n.buses.loc["EU gas", "y"] = eu_location["y"] @@ -315,7 +313,7 @@ def plot_h2_map(network, regions): h2_new = n.links[n.links.carrier == "H2 pipeline"] h2_retro = n.links[n.links.carrier == "H2 pipeline retrofitted"] - if snakemake.params["foresight"] == "myopic": + if snakemake.params.foresight == "myopic": # sum capacitiy for pipelines from different investment periods h2_new = group_pipes(h2_new) @@ -558,7 +556,7 @@ def plot_ch4_map(network): link_widths_used = max_usage / linewidth_factor link_widths_used[max_usage < line_lower_threshold] = 0.0 - tech_colors = snakemake.params["plotting"]["tech_colors"] + tech_colors = snakemake.params.plotting["tech_colors"] pipe_colors = { "gas pipeline": "#f08080", @@ -700,7 +698,7 @@ def plot_map_without(network): # hack because impossible to drop buses... if "EU gas" in n.buses.index: - eu_location = snakemake.params["plotting"].get( + eu_location = snakemake.params.plotting.get( "eu_node_location", dict(x=-5.5, y=46) ) n.buses.loc["EU gas", "x"] = eu_location["x"] @@ -876,7 +874,7 @@ def plot_series(network, carrier="AC", name="test"): stacked=True, linewidth=0.0, color=[ - snakemake.params["plotting"]["tech_colors"][i.replace(suffix, "")] + snakemake.params.plotting["tech_colors"][i.replace(suffix, "")] for i in new_columns ], ) @@ -937,7 +935,7 @@ def plot_series(network, carrier="AC", name="test"): regions = gpd.read_file(snakemake.input.regions).set_index("name") - map_opts = snakemake.params["plotting"]["map"] + map_opts = snakemake.params.plotting["map"] if map_opts["boundaries"] is None: map_opts["boundaries"] = regions.total_bounds[[0, 2, 1, 3]] + [-1, 1, -1, 1] diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index 36f75207a..e7de5473d 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -142,7 +142,7 @@ def plot_costs(): df = df.groupby(df.index.map(rename_techs)).sum() - to_drop = df.index[df.max(axis=1) < snakemake.params["plotting"]["costs_threshold"]] + to_drop = df.index[df.max(axis=1) < snakemake.params.plotting["costs_threshold"]] logger.info( f"Dropping technology with costs below {snakemake.params['plotting']['costs_threshold']} EUR billion per year" @@ -165,7 +165,7 @@ def plot_costs(): kind="bar", ax=ax, stacked=True, - color=[snakemake.params["plotting"]["tech_colors"][i] for i in new_index], + color=[snakemake.params.plotting["tech_colors"][i] for i in new_index], ) handles, labels = ax.get_legend_handles_labels() @@ -173,7 +173,7 @@ def plot_costs(): handles.reverse() labels.reverse() - ax.set_ylim([0, snakemake.params["plotting"]["costs_max"]]) + ax.set_ylim([0, snakemake.params.plotting["costs_max"]]) ax.set_ylabel("System Cost [EUR billion per year]") @@ -201,7 +201,7 @@ def plot_energy(): df = df.groupby(df.index.map(rename_techs)).sum() to_drop = df.index[ - df.abs().max(axis=1) < snakemake.params["plotting"]["energy_threshold"] + df.abs().max(axis=1) < snakemake.params.plotting["energy_threshold"] ] logger.info( @@ -227,7 +227,7 @@ def plot_energy(): kind="bar", ax=ax, stacked=True, - color=[snakemake.params["plotting"]["tech_colors"][i] for i in new_index], + color=[snakemake.params.plotting["tech_colors"][i] for i in new_index], ) handles, labels = ax.get_legend_handles_labels() @@ -237,8 +237,8 @@ def plot_energy(): ax.set_ylim( [ - snakemake.params["plotting"]["energy_min"], - snakemake.params["plotting"]["energy_max"], + snakemake.params.plotting["energy_min"], + snakemake.params.plotting["energy_max"], ] ) @@ -287,7 +287,7 @@ def plot_balances(): df = df.groupby(df.index.map(rename_techs)).sum() to_drop = df.index[ - df.abs().max(axis=1) < snakemake.params["plotting"]["energy_threshold"] / 10 + df.abs().max(axis=1) < snakemake.params.plotting["energy_threshold"] / 10 ] if v[0] in co2_carriers: @@ -317,7 +317,7 @@ def plot_balances(): kind="bar", ax=ax, stacked=True, - color=[snakemake.params["plotting"]["tech_colors"][i] for i in new_index], + color=[snakemake.params.plotting["tech_colors"][i] for i in new_index], ) handles, labels = ax.get_legend_handles_labels() @@ -455,10 +455,10 @@ def plot_carbon_budget_distribution(input_eurostat): ax1 = plt.subplot(gs1[0, 0]) ax1.set_ylabel("CO$_2$ emissions (Gt per year)", fontsize=22) ax1.set_ylim([0, 5]) - ax1.set_xlim([1990, snakemake.params["planning_horizons"][-1] + 1]) + ax1.set_xlim([1990, snakemake.params.planning_horizons[-1] + 1]) path_cb = "results/" + snakemake.params.RDIR + "/csvs/" - countries = snakemake.params["countries"] + countries = snakemake.params.countries e_1990 = co2_emissions_year(countries, input_eurostat, opts, year=1990) CO2_CAP = pd.read_csv(path_cb + "carbon_budget_distribution.csv", index_col=0) @@ -555,7 +555,7 @@ def plot_carbon_budget_distribution(input_eurostat): plot_balances() - for sector_opts in snakemake.params["sector_opts"]: + for sector_opts in snakemake.params.sector_opts: opts = sector_opts.split("-") for o in opts: if "cb" in o: diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 51777ef56..e48ee1629 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -253,12 +253,12 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): Nyears = n.snapshot_weightings.objective.sum() / 8760.0 costs = load_costs( snakemake.input.tech_costs, - snakemake.params["costs"], - snakemake.params["max_hours"], + snakemake.params.costs, + snakemake.params.max_hours, Nyears, ) - set_line_s_max_pu(n, snakemake.params["lines"]["s_max_pu"]) + set_line_s_max_pu(n, snakemake.params.lines["s_max_pu"]) for o in opts: m = re.match(r"^\d+h$", o, re.IGNORECASE) @@ -269,7 +269,7 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): for o in opts: m = re.match(r"^\d+seg$", o, re.IGNORECASE) if m is not None: - solver_name = snakemake.params["solver_name"] + solver_name = snakemake.params.solver_name n = apply_time_segmentation(n, m.group(0)[:-3], solver_name) break @@ -277,11 +277,11 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): if "Co2L" in o: m = re.findall("[0-9]*\.?[0-9]+$", o) if len(m) > 0: - co2limit = float(m[0]) * snakemake.params["co2base"] + co2limit = float(m[0]) * snakemake.params.co2base add_co2limit(n, co2limit, Nyears) logger.info("Setting CO2 limit according to wildcard value.") else: - add_co2limit(n, snakemake.params["co2limit"], Nyears) + add_co2limit(n, snakemake.params.co2limit, Nyears) logger.info("Setting CO2 limit according to config value.") break @@ -293,7 +293,7 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): add_gaslimit(n, limit, Nyears) logger.info("Setting gas usage limit according to wildcard value.") else: - add_gaslimit(n, snakemake.params["gaslimit"], Nyears) + add_gaslimit(n, snakemake.params.gaslimit, Nyears) logger.info("Setting gas usage limit according to config value.") break @@ -322,7 +322,7 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): add_emission_prices(n, dict(co2=float(m[0]))) else: logger.info("Setting emission prices according to config value.") - add_emission_prices(n, snakemake.params["costs"]["emission_prices"]) + add_emission_prices(n, snakemake.params.costs["emission_prices"]) break ll_type, factor = snakemake.wildcards.ll[0], snakemake.wildcards.ll[1:] @@ -330,8 +330,8 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): set_line_nom_max( n, - s_nom_max_set=snakemake.params["lines"].get("s_nom_max,", np.inf), - p_nom_max_set=snakemake.params["links"].get("p_nom_max,", np.inf), + s_nom_max_set=snakemake.params.lines.get("s_nom_max,", np.inf), + p_nom_max_set=snakemake.params.links.get("p_nom_max,", np.inf), ) if "ATK" in opts: diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 8b8b7d80a..77fab4f8d 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -200,12 +200,12 @@ def co2_emissions_year( """ Calculate CO2 emissions in one specific year (e.g. 1990 or 2018). """ - emissions_scope = snakemake.params["energy"]["emissions"] + emissions_scope = snakemake.params.energy["emissions"] eea_co2 = build_eea_co2(snakemake.input.co2, year, emissions_scope) # TODO: read Eurostat data from year > 2014 # this only affects the estimation of CO2 emissions for BA, RS, AL, ME, MK - report_year = snakemake.params["energy"]["eurostat_report_year"] + report_year = snakemake.params.energy["eurostat_report_year"] if year > 2014: eurostat_co2 = build_eurostat_co2( input_eurostat, countries, report_year, year=2014 @@ -241,7 +241,7 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year): carbon_budget = float(o[o.find("cb") + 2 : o.find("ex")]) r = float(o[o.find("ex") + 2 :]) - countries = snakemake.params["countries"] + countries = snakemake.params.countries e_1990 = co2_emissions_year( countries, input_eurostat, opts, emissions_scope, report_year, year=1990 @@ -252,7 +252,7 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year): countries, input_eurostat, opts, emissions_scope, report_year, year=2018 ) - planning_horizons = snakemake.params["planning_horizons"] + planning_horizons = snakemake.params.planning_horizons t_0 = planning_horizons[0] if "be" in o: @@ -391,7 +391,7 @@ def update_wind_solar_costs(n, costs): with xr.open_dataset(profile) as ds: underwater_fraction = ds["underwater_fraction"].to_pandas() connection_cost = ( - snakemake.params["length_factor"] + snakemake.params.length_factor * ds["average_distance"].to_pandas() * ( underwater_fraction @@ -483,8 +483,8 @@ def remove_elec_base_techs(n): batteries and H2) from base electricity-only network, since they're added here differently using links. """ - for c in n.iterate_components(snakemake.params["pypsa_eur"]): - to_keep = snakemake.params["pypsa_eur"][c.name] + for c in n.iterate_components(snakemake.params.pypsa_eur): + to_keep = snakemake.params.pypsa_eur[c.name] to_remove = pd.Index(c.df.carrier.unique()).symmetric_difference(to_keep) if to_remove.empty: continue @@ -674,7 +674,7 @@ def add_dac(n, costs): def add_co2limit(n, nyears=1.0, limit=0.0): logger.info(f"Adding CO2 budget limit as per unit of 1990 levels of {limit}") - countries = snakemake.params["countries"] + countries = snakemake.params.countries sectors = emission_sectors_from_opts(opts) @@ -787,7 +787,7 @@ def add_ammonia(n, costs): nodes = pop_layout.index - cf_industry = snakemake.params["industry"] + cf_industry = snakemake.params.industry n.add("Carrier", "NH3") @@ -1102,7 +1102,7 @@ def add_storage_and_grids(n, costs): lifetime=costs.at["OCGT", "lifetime"], ) - cavern_types = snakemake.params["sector"]["hydrogen_underground_storage_locations"] + cavern_types = snakemake.params.sector["hydrogen_underground_storage_locations"] h2_caverns = pd.read_csv(snakemake.input.h2_cavern, index_col=0) if ( @@ -3274,7 +3274,7 @@ def set_temporal_aggregation(n, opts, solver_name): update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) - options = snakemake.params["sector"] + options = snakemake.params.sector opts = snakemake.wildcards.sector_opts.split("-") @@ -3289,7 +3289,7 @@ def set_temporal_aggregation(n, opts, solver_name): costs = prepare_costs( snakemake.input.costs, - snakemake.params["costs"], + snakemake.params.costs, nyears, ) @@ -3301,10 +3301,10 @@ def set_temporal_aggregation(n, opts, solver_name): spatial = define_spatial(pop_layout.index, options) - if snakemake.params["foresight"] == "myopic": + if snakemake.params.foresight == "myopic": add_lifetime_wind_solar(n, costs) - conventional = snakemake.params["conventional_carriers"] + conventional = snakemake.params.conventional_carriers for carrier in conventional: add_carrier_buses(n, carrier) @@ -3369,19 +3369,19 @@ def set_temporal_aggregation(n, opts, solver_name): if options["allam_cycle"]: add_allam(n, costs) - solver_name = snakemake.params["solver_name"] + solver_name = snakemake.params.solver_name n = set_temporal_aggregation(n, opts, solver_name) limit_type = "config" - limit = get(snakemake.params["co2_budget"], investment_year) + limit = get(snakemake.params.co2_budget, investment_year) for o in opts: if "cb" not in o: continue limit_type = "carbon budget" fn = "results/" + snakemake.params.RDIR + "/csvs/carbon_budget_distribution.csv" if not os.path.exists(fn): - emissions_scope = snakemake.params["emissions_scope"] - report_year = snakemake.params["report_year"] + emissions_scope = snakemake.params.emissions_scope + report_year = snakemake.params.report_year build_carbon_budget( o, snakemake.input.eurostat, fn, emissions_scope, report_year ) @@ -3416,8 +3416,8 @@ def set_temporal_aggregation(n, opts, solver_name): if options["electricity_grid_connection"]: add_electricity_grid_connection(n, costs) - first_year_myopic = (snakemake.params["foresight"] == "myopic") and ( - snakemake.params["planning_horizons"][0] == investment_year + first_year_myopic = (snakemake.params.foresight == "myopic") and ( + snakemake.params.planning_horizons[0] == investment_year ) if options.get("cluster_heat_buses", False) and not first_year_myopic: diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py index 2204ac367..c7053bed2 100644 --- a/scripts/retrieve_databundle.py +++ b/scripts/retrieve_databundle.py @@ -53,7 +53,7 @@ snakemake ) # TODO Make logging compatible with progressbar (see PR #102) - if snakemake.params["tutorial"]: + if snakemake.params.tutorial: url = "https://zenodo.org/record/3517921/files/pypsa-eur-tutorial-data-bundle.tar.xz" else: url = "https://zenodo.org/record/3517935/files/pypsa-eur-data-bundle.tar.xz" diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 41679a035..d80e38a08 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -44,9 +44,9 @@ from pypsa.descriptors import get_switchable_as_dense as get_as_dense -def add_land_use_constraint(n, planning_horizons_param, config): +def add_land_use_constraint(n, planning_horizons, config): if "m" in snakemake.wildcards.clusters: - _add_land_use_constraint_m(n, planning_horizons_param, config) + _add_land_use_constraint_m(n, planning_horizons, config) else: _add_land_use_constraint(n) @@ -80,7 +80,7 @@ def _add_land_use_constraint(n): n.generators.p_nom_max.clip(lower=0, inplace=True) -def _add_land_use_constraint_m(n, planning_horizons_param, config): +def _add_land_use_constraint_m(n, planning_horizons, config): # if generators clustering is lower than network clustering, land_use accounting is at generators clusters planning_horizons = param["planning_horizons"] @@ -145,9 +145,9 @@ def prepare_network( n, solve_opts=None, config=None, - foresight_param=None, - planning_horizons_param=None, - co2_sequestration_potential_param=None, + foresight=None, + planning_horizons=None, + co2_sequestration_potential=None, ): if "clip_p_max_pu" in solve_opts: for df in ( @@ -198,11 +198,11 @@ def prepare_network( n.set_snapshots(n.snapshots[:nhours]) n.snapshot_weightings[:] = 8760.0 / nhours - if foresight_param == "myopic": - add_land_use_constraint(n, planning_horizons_param, config) + if foresight == "myopic": + add_land_use_constraint(n, planning_horizons, config) if n.stores.carrier.eq("co2 stored").any(): - limit = co2_sequestration_potential_param + limit = co2_sequestration_potential add_co2_sequestration_limit(n, limit=limit) return n @@ -597,13 +597,11 @@ def extra_functionality(n, snapshots): add_pipe_retrofit_constraint(n) -def solve_network(n, config, solving_param, opts="", **kwargs): - set_of_options = solving_param["solver"]["options"] - solver_options = ( - solving_param["solver_options"][set_of_options] if set_of_options else {} - ) - solver_name = solving_param["solver"]["name"] - cf_solving = solving_param["options"] +def solve_network(n, config, solving, opts="", **kwargs): + set_of_options = solving["solver"]["options"] + solver_options = solving["solver_options"][set_of_options] if set_of_options else {} + solver_name = solving["solver"]["name"] + cf_solving = solving["options"] track_iterations = cf_solving.get("track_iterations", False) min_iterations = cf_solving.get("min_iterations", 4) max_iterations = cf_solving.get("max_iterations", 6) @@ -672,7 +670,7 @@ def solve_network(n, config, solving_param, opts="", **kwargs): if "sector_opts" in snakemake.wildcards.keys(): opts += "-" + snakemake.wildcards.sector_opts opts = [o for o in opts.split("-") if o != ""] - solve_opts = snakemake.params["solving"]["options"] + solve_opts = snakemake.params.solving["options"] np.random.seed(solve_opts.get("seed", 123)) @@ -686,17 +684,15 @@ def solve_network(n, config, solving_param, opts="", **kwargs): n, solve_opts, config=snakemake.config, - foresight_param=snakemake.params["foresight"], - planning_horizons_param=snakemake.params["planning_horizons"], - co2_sequestration_potential_param=snakemake.params[ - "co2_sequestration_potential" - ], + foresight=snakemake.params.foresight, + planning_horizons=snakemake.params.planning_horizons, + co2_sequestration_potential=snakemake.params["co2_sequestration_potential"], ) n = solve_network( n, config=snakemake.config, - solving_param=snakemake.params["solving"], + solving=snakemake.params.solving, opts=opts, log_fn=snakemake.log.solver, ) diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index 421bc515f..37e853e57 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -41,7 +41,7 @@ opts = (snakemake.wildcards.opts + "-" + snakemake.wildcards.sector_opts).split("-") opts = [o for o in opts if o != ""] - solve_opts = snakemake.params["options"] + solve_opts = snakemake.params.options np.random.seed(solve_opts.get("seed", 123)) From 27eea273bd20559dcf02d32ed0638c2938fb4acf Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 15 Jun 2023 19:12:30 +0200 Subject: [PATCH 23/24] harmonize params names --- rules/build_electricity.smk | 9 +++------ rules/build_sector.smk | 7 ++++--- scripts/add_electricity.py | 21 ++++++--------------- scripts/build_sequestration_potentials.py | 2 +- scripts/prepare_network.py | 2 +- scripts/prepare_sector_network.py | 4 ++-- 6 files changed, 17 insertions(+), 28 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 9d4315df1..44ebf4042 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -320,11 +320,10 @@ rule simplify_network: aggregation_strategies=config["clustering"].get("aggregation_strategies", {}), focus_weights=config.get("focus_weights", None), renewable_carriers=config["electricity"]["renewable_carriers"], - costs=config["costs"], max_hours=config["electricity"]["max_hours"], length_factor=config["lines"]["length_factor"], p_max_pu=config["links"].get("p_max_pu", 1.0), - solver_name=config["solving"]["solver"]["name"], + costs=config["costs"], input: network=RESOURCES + "networks/elec.nc", tech_costs=COSTS, @@ -357,10 +356,9 @@ rule cluster_network: focus_weights=config.get("focus_weights", None), renewable_carriers=config["electricity"]["renewable_carriers"], conventional_carriers=config["electricity"].get("conventional_carriers", []), - costs=config["costs"], max_hours=config["electricity"]["max_hours"], length_factor=config["lines"]["length_factor"], - solver_name=config["solving"]["solver"]["name"], + costs=config["costs"], input: network=RESOURCES + "networks/elec_s{simpl}.nc", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}.geojson", @@ -393,9 +391,9 @@ rule cluster_network: rule add_extra_components: params: - costs=config["costs"], extendable_carriers=config["electricity"]["extendable_carriers"], max_hours=config["electricity"]["max_hours"], + costs=config["costs"], input: network=RESOURCES + "networks/elec_s{simpl}_{clusters}.nc", tech_costs=COSTS, @@ -418,7 +416,6 @@ rule prepare_network: params: links=config["links"], lines=config["lines"], - solver_name=config["solving"]["solver"]["name"], co2base=config["electricity"]["co2base"], co2limit=config["electricity"]["co2limit"], gaslimit=config["electricity"].get("gaslimit"), diff --git a/rules/build_sector.smk b/rules/build_sector.smk index e15394ffb..1c3507fd6 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -330,7 +330,9 @@ if config["sector"]["regional_co2_sequestration_potential"]["enable"]: rule build_sequestration_potentials: params: - co2seq_potential=config["sector"]["regional_co2_sequestration_potential"], + sequestration_potential=config["sector"][ + "regional_co2_sequestration_potential" + ], input: sequestration_potential=HTTP.remote( "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson", @@ -705,7 +707,6 @@ rule build_transport_demand: rule prepare_sector_network: params: co2_budget=config["co2_budget"], - solver_name=config["solving"]["solver"]["name"], conventional_carriers=config["existing_capacities"]["conventional_carriers"], foresight=config["foresight"], costs=config["costs"], @@ -716,7 +717,7 @@ rule prepare_sector_network: planning_horizons=config["scenario"]["planning_horizons"], countries=config["countries"], emissions_scope=config["energy"]["emissions"], - report_year=config["energy"]["eurostat_report_year"], + eurostat_report_year=config["energy"]["eurostat_report_year"], RDIR=RDIR, input: **build_retro_cost_output, diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index d98dc7671..e2f7fb655 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -137,7 +137,7 @@ def _add_missing_carriers_from_costs(n, costs, carriers): n.import_components_from_dataframe(emissions, "Carrier") -def load_costs(tech_costs, params, max_hours, Nyears=1.0): +def load_costs(tech_costs, costs, max_hours, Nyears=1.0): # set all asset costs and other parameters costs = pd.read_csv(tech_costs, index_col=[0, 1]).sort_index() @@ -145,7 +145,7 @@ def load_costs(tech_costs, params, max_hours, Nyears=1.0): costs.loc[costs.unit.str.contains("/kW"), "value"] *= 1e3 costs.unit = costs.unit.str.replace("/kW", "/MW") - fill_values = params["fill_values"] + fill_values = costs["fill_values"] costs = costs.value.unstack().fillna(fill_values) costs["capital_cost"] = ( @@ -168,8 +168,8 @@ def load_costs(tech_costs, params, max_hours, Nyears=1.0): costs.at["CCGT", "co2_emissions"] = costs.at["gas", "co2_emissions"] costs.at["solar", "capital_cost"] = ( - params["rooftop_share"] * costs.at["solar-rooftop", "capital_cost"] - + (1 - params["rooftop_share"]) * costs.at["solar-utility", "capital_cost"] + costs["rooftop_share"] * costs.at["solar-rooftop", "capital_cost"] + + (1 - costs["rooftop_share"]) * costs.at["solar-utility", "capital_cost"] ) def costs_for_storage(store, link1, link2=None, max_hours=1.0): @@ -193,7 +193,7 @@ def costs_for_storage(store, link1, link2=None, max_hours=1.0): ) for attr in ("marginal_cost", "capital_cost"): - overwrites = params.get(attr) + overwrites = costs.get(attr) if overwrites is not None: overwrites = pd.Series(overwrites) costs.loc[overwrites.index, attr] = overwrites @@ -728,16 +728,6 @@ def add_nice_carrier_names(n, config): ) ppl = load_powerplants(snakemake.input.powerplants) - if "renewable_carriers" in params.electricity: - renewable_carriers = set(params.electricity["renewable_carriers"]) - else: - logger.warning( - "Missing key `renewable_carriers` under config entry `electricity`. " - "In future versions, this will raise an error. " - "Falling back to carriers listed under `renewable`." - ) - renewable_carriers = params.renewable - attach_load( n, snakemake.input.regions, @@ -749,6 +739,7 @@ def add_nice_carrier_names(n, config): update_transmission_costs(n, costs, params.length_factor) + renewable_carriers = set(params.electricity["renewable_carriers"]) extendable_carriers = params.electricity["extendable_carriers"] conventional_carriers = params.electricity["conventional_carriers"] conventional_inputs = { diff --git a/scripts/build_sequestration_potentials.py b/scripts/build_sequestration_potentials.py index 0e59e55bc..e19a96da6 100644 --- a/scripts/build_sequestration_potentials.py +++ b/scripts/build_sequestration_potentials.py @@ -41,7 +41,7 @@ def allocate_sequestration_potential( "build_sequestration_potentials", simpl="", clusters="181" ) - cf = snakemake.params.co2seq_potential + cf = snakemake.params.sequestration_potential gdf = gpd.read_file(snakemake.input.sequestration_potential[0]) diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index e48ee1629..adbfa36cf 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -269,7 +269,7 @@ def set_line_nom_max(n, s_nom_max_set=np.inf, p_nom_max_set=np.inf): for o in opts: m = re.match(r"^\d+seg$", o, re.IGNORECASE) if m is not None: - solver_name = snakemake.params.solver_name + solver_name = snakemake.config["solving"]["solver"]["name"] n = apply_time_segmentation(n, m.group(0)[:-3], solver_name) break diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 77fab4f8d..73adf5224 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3369,7 +3369,7 @@ def set_temporal_aggregation(n, opts, solver_name): if options["allam_cycle"]: add_allam(n, costs) - solver_name = snakemake.params.solver_name + solver_name = snakemake.config["solving"]["solver"]["name"] n = set_temporal_aggregation(n, opts, solver_name) limit_type = "config" @@ -3381,7 +3381,7 @@ def set_temporal_aggregation(n, opts, solver_name): fn = "results/" + snakemake.params.RDIR + "/csvs/carbon_budget_distribution.csv" if not os.path.exists(fn): emissions_scope = snakemake.params.emissions_scope - report_year = snakemake.params.report_year + report_year = snakemake.params.eurostat_report_year build_carbon_budget( o, snakemake.input.eurostat, fn, emissions_scope, report_year ) From a17647a07e3c5e93fcaa2a686e0f72c09ab0dbfc Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 15 Jun 2023 19:35:41 +0200 Subject: [PATCH 24/24] add_electricity: fix cost ref --- scripts/add_electricity.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index e2f7fb655..594988bdb 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -137,7 +137,7 @@ def _add_missing_carriers_from_costs(n, costs, carriers): n.import_components_from_dataframe(emissions, "Carrier") -def load_costs(tech_costs, costs, max_hours, Nyears=1.0): +def load_costs(tech_costs, config, max_hours, Nyears=1.0): # set all asset costs and other parameters costs = pd.read_csv(tech_costs, index_col=[0, 1]).sort_index() @@ -145,7 +145,7 @@ def load_costs(tech_costs, costs, max_hours, Nyears=1.0): costs.loc[costs.unit.str.contains("/kW"), "value"] *= 1e3 costs.unit = costs.unit.str.replace("/kW", "/MW") - fill_values = costs["fill_values"] + fill_values = config["fill_values"] costs = costs.value.unstack().fillna(fill_values) costs["capital_cost"] = ( @@ -168,8 +168,8 @@ def load_costs(tech_costs, costs, max_hours, Nyears=1.0): costs.at["CCGT", "co2_emissions"] = costs.at["gas", "co2_emissions"] costs.at["solar", "capital_cost"] = ( - costs["rooftop_share"] * costs.at["solar-rooftop", "capital_cost"] - + (1 - costs["rooftop_share"]) * costs.at["solar-utility", "capital_cost"] + config["rooftop_share"] * costs.at["solar-rooftop", "capital_cost"] + + (1 - config["rooftop_share"]) * costs.at["solar-utility", "capital_cost"] ) def costs_for_storage(store, link1, link2=None, max_hours=1.0): @@ -193,7 +193,7 @@ def costs_for_storage(store, link1, link2=None, max_hours=1.0): ) for attr in ("marginal_cost", "capital_cost"): - overwrites = costs.get(attr) + overwrites = config.get(attr) if overwrites is not None: overwrites = pd.Series(overwrites) costs.loc[overwrites.index, attr] = overwrites