Merge pull request #1050 from PyPSA/exclude_shared_resources

allow excluding files from shared_resources

Snakefile

@@ -24,9 +24,11 @@ run = config["run"]
 scenarios = get_scenarios(run)
 RDIR = get_rdir(run)
 
-logs = path_provider("logs/", RDIR, run["shared_resources"])
-benchmarks = path_provider("benchmarks/", RDIR, run["shared_resources"])
-resources = path_provider("resources/", RDIR, run["shared_resources"])
+shared_resources = run["shared_resources"]["mode"]
+exclude_from_shared = run["shared_resources"]["exclude"]
+logs = path_provider("logs/", RDIR, shared_resources, exclude_from_shared)
+benchmarks = path_provider("benchmarks/", RDIR, shared_resources, exclude_from_shared)
+resources = path_provider("resources/", RDIR, shared_resources, exclude_from_shared)
 
 CDIR = "" if run["shared_cutouts"] else RDIR
 RESULTS = "results/" + RDIR

config/config.default.yaml

@@ -26,7 +26,9 @@ run:
     enable: false
     file: config/scenarios.yaml
   disable_progressbar: false
-  shared_resources: false
+  shared_resources:
+    policy: false
+    exclude: []
   shared_cutouts: true
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#foresight

config/config.entsoe-all.yaml

@@ -5,7 +5,8 @@
 run:
   name: "entsoe-all"
   disable_progressbar: true
-  shared_resources: false
+  shared_resources:
+    policy: false
   shared_cutouts: true
 
 scenario:

config/test/config.electricity.yaml

@@ -8,7 +8,8 @@ tutorial: true
 run:
   name: "test-elec" # use this to keep track of runs with different settings
   disable_progressbar: true
-  shared_resources: "test"
+  shared_resources:
+    policy: "test"
   shared_cutouts: true
 
 scenario:

config/test/config.myopic.yaml

@@ -7,7 +7,8 @@ tutorial: true
 run:
   name: "test-sector-myopic"
   disable_progressbar: true
-  shared_resources: "test"
+  shared_resources:
+    policy: "test"
   shared_cutouts: true
 
 foresight: myopic

config/test/config.overnight.yaml

@@ -7,7 +7,8 @@ tutorial: true
 run:
   name: "test-sector-overnight"
   disable_progressbar: true
-  shared_resources: "test"
+  shared_resources:
+    policy: "test"
   shared_cutouts: true
 
 

config/test/config.perfect.yaml

@@ -7,7 +7,8 @@ tutorial: true
 run:
   name: "test-sector-perfect"
   disable_progressbar: true
-  shared_resources: "test"
+  shared_resources:
+    policy: "test"
   shared_cutouts: true
 
 foresight: perfect

config/test/config.scenarios.yaml

@@ -12,7 +12,8 @@ run:
     enable: true
     file: "config/test/scenarios.yaml"
   disable_progressbar: true
-  shared_resources: base
+  shared_resources:
+    policy: base
   shared_cutouts: true
 
 scenario:

doc/configtables/run.csv

@@ -5,5 +5,7 @@ scenarios,,,
 -- enable,bool,"{true, false}","Switch to select whether workflow should generate scenarios based on ``file``."
 -- file,str,,"Path to the scenario yaml file. The scenario file contains config overrides for each scenario. In order to be taken account, ``run: scenarios`` has to be set to ``true`` and ``run: name`` has to be a subset of top level keys given in the scenario file. In order to automatically create a `scenario.yaml` file based on a combination of settings, alter and use the ``config/create_scenarios.py`` script in the ``config`` directory."
 disable_progressbar,bool,"{true, false}","Switch to select whether progressbar should be disabled."
-shared_resources,bool/str,,"Switch to select whether resources should be shared across runs. If a string is passed, this is used as a subdirectory name for shared resources. If set to 'base', only resources before creating the elec.nc file are shared."
+shared_resources,,,
+-- policy,bool/str,,"Boolean switch to select whether resources should be shared across runs. If a string is passed, this is used as a subdirectory name for shared resources. If set to 'base', only resources before creating the elec.nc file are shared."
+-- exclude,str,"For the case shared_resources=base, specify additional files that should not be shared across runs."
 shared_cutouts,bool,"{true, false}","Switch to select whether cutouts should be shared across runs."

doc/release_notes.rst

@@ -9,6 +9,18 @@ Release Notes
 
 Upcoming Release
 ================
+
+* Add config ``run: shared_resources: exclude:`` to specify additional files
+  that should be excluded from shared resources with the setting ``run:
+  shared_resources: base``. The function ``_helpers/get_run_path()`` now takes
+  an additional keyword argument ``exclude_from_shared`` with a list of files
+  that should not be shared. This keyword argument accepts a list of strings
+  where the string only needs to match the start of a filename (e.g.
+  ``"transport_data"`` would exclude both ``transport_data.csv`` and
+  ``transport_data_{simpl}_{clusters}.csv`` from being shared across scenarios.
+
+* Move switch ``run: shared_resources:`` to ``run: shared_resources: policy:``.
+
 * Add config land_transport_demand_factor to model growth in land transport demand for different time horizons.
 
 * Allow dictionary for the config aviation_demand_factor.

rules/build_electricity.smk

@@ -433,7 +433,7 @@ rule add_electricity:
             else resources("networks/base.nc")
         ),
         tech_costs=lambda w: resources(
-            f"costs_{config_provider('costs', 'year')(w)}.csv"
+            f"costs_{config_provider('costs', 'year') (w)}.csv"
         ),
         regions=resources("regions_onshore.geojson"),
         powerplants=resources("powerplants.csv"),
@@ -478,7 +478,7 @@ rule simplify_network:
     input:
         network=resources("networks/elec.nc"),
         tech_costs=lambda w: resources(
-            f"costs_{config_provider('costs', 'year')(w)}.csv"
+            f"costs_{config_provider('costs', 'year') (w)}.csv"
         ),
         regions_onshore=resources("regions_onshore.geojson"),
         regions_offshore=resources("regions_offshore.geojson"),
@@ -527,7 +527,7 @@ rule cluster_network:
             else []
         ),
         tech_costs=lambda w: resources(
-            f"costs_{config_provider('costs', 'year')(w)}.csv"
+            f"costs_{config_provider('costs', 'year') (w)}.csv"
         ),
     output:
         network=resources("networks/elec_s{simpl}_{clusters}.nc"),
@@ -556,7 +556,7 @@ rule add_extra_components:
     input:
         network=resources("networks/elec_s{simpl}_{clusters}.nc"),
         tech_costs=lambda w: resources(
-            f"costs_{config_provider('costs', 'year')(w)}.csv"
+            f"costs_{config_provider('costs', 'year') (w)}.csv"
         ),
     output:
         resources("networks/elec_s{simpl}_{clusters}_ec.nc"),
@@ -591,7 +591,7 @@ rule prepare_network:
     input:
         resources("networks/elec_s{simpl}_{clusters}_ec.nc"),
         tech_costs=lambda w: resources(
-            f"costs_{config_provider('costs', 'year')(w)}.csv"
+            f"costs_{config_provider('costs', 'year') (w)}.csv"
         ),
         co2_price=lambda w: resources("co2_price.csv") if "Ept" in w.opts else [],
     output:

scripts/_helpers.py

@@ -75,7 +75,7 @@ def get_rdir(run):
     return RDIR
 
 
-def get_run_path(fn, dir, rdir, shared_resources):
+def get_run_path(fn, dir, rdir, shared_resources, exclude_from_shared):
     """
     Dynamically provide paths based on shared resources and filename.
 
@@ -95,6 +95,8 @@ def get_run_path(fn, dir, rdir, shared_resources):
         - If string is "base", special handling for shared "base" resources (see notes).
         - If random string other than "base", this folder is used instead of the `rdir` keyword.
         - If boolean, directly specifies if the resource is shared.
+    exclude_from_shared: list
+        List of filenames to exclude from shared resources. Only relevant if shared_resources is "base".
 
     Returns
     -------
@@ -112,10 +114,12 @@ def get_run_path(fn, dir, rdir, shared_resources):
         existing_wildcards = set(re.findall(pattern, fn))
         irrelevant_wildcards = {"technology", "year", "scope", "kind"}
         no_relevant_wildcards = not existing_wildcards - irrelevant_wildcards
-        no_elec_rule = not fn.startswith("networks/elec") and not fn.startswith(
-            "add_electricity"
+        not_shared_rule = (
+            not fn.startswith("networks/elec")
+            and not fn.startswith("add_electricity")
+            and not any(fn.startswith(ex) for ex in exclude_from_shared)
         )
-        is_shared = no_relevant_wildcards and no_elec_rule
+        is_shared = no_relevant_wildcards and not_shared_rule
         rdir = "" if is_shared else rdir
     elif isinstance(shared_resources, str):
         rdir = shared_resources + "/"
@@ -129,7 +133,7 @@ def get_run_path(fn, dir, rdir, shared_resources):
     return f"{dir}{rdir}{fn}"
 
 
-def path_provider(dir, rdir, shared_resources):
+def path_provider(dir, rdir, shared_resources, exclude_from_shared):
     """
     Returns a partial function that dynamically provides paths based on shared
     resources and the filename.
@@ -140,7 +144,13 @@ def path_provider(dir, rdir, shared_resources):
         A partial function that takes a filename as input and
         returns the path to the file based on the shared_resources parameter.
     """
-    return partial(get_run_path, dir=dir, rdir=rdir, shared_resources=shared_resources)
+    return partial(
+        get_run_path,
+        dir=dir,
+        rdir=rdir,
+        shared_resources=shared_resources,
+        exclude_from_shared=exclude_from_shared,
+    )
 
 
 def get_opt(opts, expr, flags=None):