diff --git a/Snakefile b/Snakefile index ba93a869f..2e2d7f230 100644 --- a/Snakefile +++ b/Snakefile @@ -24,9 +24,11 @@ run = config["run"] scenarios = get_scenarios(run) RDIR = get_rdir(run) -logs = path_provider("logs/", RDIR, run["shared_resources"]) -benchmarks = path_provider("benchmarks/", RDIR, run["shared_resources"]) -resources = path_provider("resources/", RDIR, run["shared_resources"]) +shared_resources = run["shared_resources"]["mode"] +exclude_from_shared = run["shared_resources"]["exclude"] +logs = path_provider("logs/", RDIR, shared_resources, exclude_from_shared) +benchmarks = path_provider("benchmarks/", RDIR, shared_resources, exclude_from_shared) +resources = path_provider("resources/", RDIR, shared_resources, exclude_from_shared) CDIR = "" if run["shared_cutouts"] else RDIR RESULTS = "results/" + RDIR diff --git a/config/config.default.yaml b/config/config.default.yaml index 1dfcf8d94..e8ebee967 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -26,7 +26,9 @@ run: enable: false file: config/scenarios.yaml disable_progressbar: false - shared_resources: false + shared_resources: + policy: false + exclude: [] shared_cutouts: true # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#foresight diff --git a/config/config.entsoe-all.yaml b/config/config.entsoe-all.yaml index 40e3c0a59..3635b01d3 100644 --- a/config/config.entsoe-all.yaml +++ b/config/config.entsoe-all.yaml @@ -5,7 +5,8 @@ run: name: "entsoe-all" disable_progressbar: true - shared_resources: false + shared_resources: + policy: false shared_cutouts: true scenario: diff --git a/config/test/config.electricity.yaml b/config/test/config.electricity.yaml index 579644157..e1396b4c1 100644 --- a/config/test/config.electricity.yaml +++ b/config/test/config.electricity.yaml @@ -8,7 +8,8 @@ tutorial: true run: name: "test-elec" # use this to keep track of runs with different settings disable_progressbar: true - shared_resources: "test" + shared_resources: + policy: "test" shared_cutouts: true scenario: diff --git a/config/test/config.myopic.yaml b/config/test/config.myopic.yaml index 5abae36d0..ac6112f1d 100644 --- a/config/test/config.myopic.yaml +++ b/config/test/config.myopic.yaml @@ -7,7 +7,8 @@ tutorial: true run: name: "test-sector-myopic" disable_progressbar: true - shared_resources: "test" + shared_resources: + policy: "test" shared_cutouts: true foresight: myopic diff --git a/config/test/config.overnight.yaml b/config/test/config.overnight.yaml index 7fb53e427..a29697e69 100644 --- a/config/test/config.overnight.yaml +++ b/config/test/config.overnight.yaml @@ -7,7 +7,8 @@ tutorial: true run: name: "test-sector-overnight" disable_progressbar: true - shared_resources: "test" + shared_resources: + policy: "test" shared_cutouts: true diff --git a/config/test/config.perfect.yaml b/config/test/config.perfect.yaml index 5d77c9c51..63ad25d92 100644 --- a/config/test/config.perfect.yaml +++ b/config/test/config.perfect.yaml @@ -7,7 +7,8 @@ tutorial: true run: name: "test-sector-perfect" disable_progressbar: true - shared_resources: "test" + shared_resources: + policy: "test" shared_cutouts: true foresight: perfect diff --git a/config/test/config.scenarios.yaml b/config/test/config.scenarios.yaml index 8ecbb91bd..06fe9f23c 100644 --- a/config/test/config.scenarios.yaml +++ b/config/test/config.scenarios.yaml @@ -12,7 +12,8 @@ run: enable: true file: "config/test/scenarios.yaml" disable_progressbar: true - shared_resources: base + shared_resources: + policy: base shared_cutouts: true scenario: diff --git a/doc/configtables/run.csv b/doc/configtables/run.csv index 44f061653..f11a8d960 100644 --- a/doc/configtables/run.csv +++ b/doc/configtables/run.csv @@ -5,5 +5,7 @@ scenarios,,, -- enable,bool,"{true, false}","Switch to select whether workflow should generate scenarios based on ``file``." -- file,str,,"Path to the scenario yaml file. The scenario file contains config overrides for each scenario. In order to be taken account, ``run: scenarios`` has to be set to ``true`` and ``run: name`` has to be a subset of top level keys given in the scenario file. In order to automatically create a `scenario.yaml` file based on a combination of settings, alter and use the ``config/create_scenarios.py`` script in the ``config`` directory." disable_progressbar,bool,"{true, false}","Switch to select whether progressbar should be disabled." -shared_resources,bool/str,,"Switch to select whether resources should be shared across runs. If a string is passed, this is used as a subdirectory name for shared resources. If set to 'base', only resources before creating the elec.nc file are shared." +shared_resources,,, +-- policy,bool/str,,"Boolean switch to select whether resources should be shared across runs. If a string is passed, this is used as a subdirectory name for shared resources. If set to 'base', only resources before creating the elec.nc file are shared." +-- exclude,str,"For the case shared_resources=base, specify additional files that should not be shared across runs." shared_cutouts,bool,"{true, false}","Switch to select whether cutouts should be shared across runs." diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 349a710c4..145e3f526 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -9,6 +9,18 @@ Release Notes Upcoming Release ================ + +* Add config ``run: shared_resources: exclude:`` to specify additional files + that should be excluded from shared resources with the setting ``run: + shared_resources: base``. The function ``_helpers/get_run_path()`` now takes + an additional keyword argument ``exclude_from_shared`` with a list of files + that should not be shared. This keyword argument accepts a list of strings + where the string only needs to match the start of a filename (e.g. + ``"transport_data"`` would exclude both ``transport_data.csv`` and + ``transport_data_{simpl}_{clusters}.csv`` from being shared across scenarios. + +* Move switch ``run: shared_resources:`` to ``run: shared_resources: policy:``. + * Add config land_transport_demand_factor to model growth in land transport demand for different time horizons. * Allow dictionary for the config aviation_demand_factor. diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 77abb989a..24496ccd6 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -433,7 +433,7 @@ rule add_electricity: else resources("networks/base.nc") ), tech_costs=lambda w: resources( - f"costs_{config_provider('costs', 'year')(w)}.csv" + f"costs_{config_provider('costs', 'year') (w)}.csv" ), regions=resources("regions_onshore.geojson"), powerplants=resources("powerplants.csv"), @@ -478,7 +478,7 @@ rule simplify_network: input: network=resources("networks/elec.nc"), tech_costs=lambda w: resources( - f"costs_{config_provider('costs', 'year')(w)}.csv" + f"costs_{config_provider('costs', 'year') (w)}.csv" ), regions_onshore=resources("regions_onshore.geojson"), regions_offshore=resources("regions_offshore.geojson"), @@ -527,7 +527,7 @@ rule cluster_network: else [] ), tech_costs=lambda w: resources( - f"costs_{config_provider('costs', 'year')(w)}.csv" + f"costs_{config_provider('costs', 'year') (w)}.csv" ), output: network=resources("networks/elec_s{simpl}_{clusters}.nc"), @@ -556,7 +556,7 @@ rule add_extra_components: input: network=resources("networks/elec_s{simpl}_{clusters}.nc"), tech_costs=lambda w: resources( - f"costs_{config_provider('costs', 'year')(w)}.csv" + f"costs_{config_provider('costs', 'year') (w)}.csv" ), output: resources("networks/elec_s{simpl}_{clusters}_ec.nc"), @@ -591,7 +591,7 @@ rule prepare_network: input: resources("networks/elec_s{simpl}_{clusters}_ec.nc"), tech_costs=lambda w: resources( - f"costs_{config_provider('costs', 'year')(w)}.csv" + f"costs_{config_provider('costs', 'year') (w)}.csv" ), co2_price=lambda w: resources("co2_price.csv") if "Ept" in w.opts else [], output: diff --git a/scripts/_helpers.py b/scripts/_helpers.py index a6dca6df2..83cffd1e1 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -75,7 +75,7 @@ def get_rdir(run): return RDIR -def get_run_path(fn, dir, rdir, shared_resources): +def get_run_path(fn, dir, rdir, shared_resources, exclude_from_shared): """ Dynamically provide paths based on shared resources and filename. @@ -95,6 +95,8 @@ def get_run_path(fn, dir, rdir, shared_resources): - If string is "base", special handling for shared "base" resources (see notes). - If random string other than "base", this folder is used instead of the `rdir` keyword. - If boolean, directly specifies if the resource is shared. + exclude_from_shared: list + List of filenames to exclude from shared resources. Only relevant if shared_resources is "base". Returns ------- @@ -112,10 +114,12 @@ def get_run_path(fn, dir, rdir, shared_resources): existing_wildcards = set(re.findall(pattern, fn)) irrelevant_wildcards = {"technology", "year", "scope", "kind"} no_relevant_wildcards = not existing_wildcards - irrelevant_wildcards - no_elec_rule = not fn.startswith("networks/elec") and not fn.startswith( - "add_electricity" + not_shared_rule = ( + not fn.startswith("networks/elec") + and not fn.startswith("add_electricity") + and not any(fn.startswith(ex) for ex in exclude_from_shared) ) - is_shared = no_relevant_wildcards and no_elec_rule + is_shared = no_relevant_wildcards and not_shared_rule rdir = "" if is_shared else rdir elif isinstance(shared_resources, str): rdir = shared_resources + "/" @@ -129,7 +133,7 @@ def get_run_path(fn, dir, rdir, shared_resources): return f"{dir}{rdir}{fn}" -def path_provider(dir, rdir, shared_resources): +def path_provider(dir, rdir, shared_resources, exclude_from_shared): """ Returns a partial function that dynamically provides paths based on shared resources and the filename. @@ -140,7 +144,13 @@ def path_provider(dir, rdir, shared_resources): A partial function that takes a filename as input and returns the path to the file based on the shared_resources parameter. """ - return partial(get_run_path, dir=dir, rdir=rdir, shared_resources=shared_resources) + return partial( + get_run_path, + dir=dir, + rdir=rdir, + shared_resources=shared_resources, + exclude_from_shared=exclude_from_shared, + ) def get_opt(opts, expr, flags=None):