patch for statistics plotting from @p-glaum

rules/collect.smk

@@ -96,21 +96,22 @@ rule validate_elec_networks:
         ),
 
 
-# temporarily comment out since conflicting with scenario management
-# rule plot_statistics:
-#    input:
-#        [
-#            expand(
-#                RESULTS
-#                + "statistics/figures/comparison/country_{country}/.statistics_{carrier}_plots",
-#                country=config["plotting"].get("countries", "all"),
-#                carrier=config["plotting"].get("carriers", ["all"]),
-#            ),
-#            expand(
-#                RESULTS
-#                + "statistics/figures/single/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_plots",
-#                **config["scenario"],
-#                country=config["plotting"].get("countries", "all"),
-#                carrier=config["plotting"].get("carriers", ["all"]),
-#            ),
-#        ],
+rule plot_statistics:
+    input:
+        [
+            expand(
+                RESULTS
+                + "statistics/figures/comparison/country_{country}/.statistics_{carrier}_plots",
+                country=config["plotting"].get("countries", "all"),
+                carrier=config["plotting"].get("carriers", ["all"]),
+                run=config["run"]["name"],
+            ),
+            expand(
+                RESULTS
+                + "statistics/figures/single/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_plots",
+                **config["scenario"],
+                country=config["plotting"].get("countries", "all"),
+                carrier=config["plotting"].get("carriers", ["all"]),
+                run=config["run"]["name"],
+            ),
+        ],

rules/postprocess.smk

@@ -133,6 +133,23 @@ if config["foresight"] == "perfect":
             "../scripts/plot_power_network_perfect.py"
 
 
+rule copy_config:
+    params:
+        RDIR=RDIR,
+        config=lambda wildcards: (
+            scenario_config(wildcards.run) if "run" in wildcards else config
+        ),
+    output:
+        RESULTS + "config.yaml",
+    threads: 1
+    resources:
+        mem_mb=1000,
+    conda:
+        "../envs/environment.yaml"
+    script:
+        "../scripts/copy_config.py"
+
+
 rule make_summary:
     params:
         foresight=config_provider("foresight"),
@@ -315,6 +332,9 @@ rule plot_statistics_single:
         },
         barplots_touch=RESULTS
         + "statistics/figures/single/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}/country_{country}/.statistics_{carrier}_plots",
+    log:
+        RESULTS
+        + "logs/plot_statistics_single/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_country-{country}_carrier-{carrier}.log",
     script:
         "../scripts/plot_statistics_single.py"
 
@@ -341,6 +361,9 @@ rule plot_statistics_comparison:
         },
         barplots_touch=RESULTS
         + "statistics/figures/comparison/country_{country}/.statistics_{carrier}_plots",
+    log:
+        RESULTS
+        + "logs/plot_statistics_comparison/country-{country}_carrier-{carrier}.log",
     script:
         "../scripts/plot_statistics_comparison.py"
 

scripts/plot_statistics_comparison.py

@@ -4,6 +4,7 @@
 #
 # SPDX-License-Identifier: MIT
 
+import logging
 import re
 
 import matplotlib.pyplot as plt
@@ -12,6 +13,7 @@
 from _helpers import configure_logging
 from plot_summary import rename_techs
 
+logger = logging.getLogger(__name__)
 sns.set_theme("paper", style="whitegrid")
 STACKED = {
     "capacity_factor": False,
@@ -44,7 +46,13 @@ def plot_static_comparison(df, ax, stacked=False):
     df = df.dropna(axis=0, how="all").fillna(0)
     if df.empty:
         return
-    c = tech_colors[df.index.get_level_values("carrier").map(rename_techs)]
+    df = df.rename(index=rename_techs).groupby(["component", "carrier"]).sum()
+    carriers = df.index.get_level_values("carrier")
+    if not carriers.difference(tech_colors.index).empty:
+        print(
+            f"Missing colors for carrier: {carriers.difference(tech_colors.index).values}\n Dark grey used instead."
+        )
+    c = carriers.map(lambda x: tech_colors.get(x, "#808080"))
     df = df.pipe(rename_index).T
     df = df.div(float(factor)) if factor != "-" else df
     df.plot.bar(color=c.values, ax=ax, stacked=stacked, legend=False, ylabel=unit)

scripts/plot_statistics_single.py

@@ -4,12 +4,15 @@
 #
 # SPDX-License-Identifier: MIT
 
+import logging
+
 import matplotlib.pyplot as plt
 import pandas as pd
 import seaborn as sns
 from _helpers import configure_logging
 from plot_summary import rename_techs
 
+logger = logging.getLogger(__name__)
 sns.set_theme("paper", style="whitegrid")
 
 
@@ -24,7 +27,12 @@ def rename_index(ds):
 def plot_static_single(ds, ax):
     factor, unit = conversion[output]
     ds = ds.dropna()
-    c = tech_colors[ds.index.get_level_values("carrier").map(rename_techs)]
+    carriers = ds.index.get_level_values("carrier").map(rename_techs)
+    if not carriers.difference(tech_colors.index).empty:
+        print(
+            f"Missing colors for carrier: {carriers.difference(tech_colors.index).values}\n Dark grey used instead."
+        )
+    c = carriers.map(lambda x: tech_colors.get(x, "#808080"))
     ds = ds.pipe(rename_index)
     ds = ds.div(float(factor)) if factor != "-" else ds
     ds.T.plot.barh(color=c.values, ax=ax, xlabel=unit)
@@ -47,13 +55,14 @@ def read_csv(input):
 
         snakemake = mock_snakemake(
             "plot_statistics_single",
+            run="240219-test/normal",
             simpl="",
-            ll="v1.5",
-            clusters="5",
+            ll="v1.2",
+            clusters="22",
             opts="",
-            sector_opts="24h-T-H-B-I-A-dist1",
-            planning_horizons="2040",
-            country="all",
+            sector_opts="none",
+            planning_horizons="2020",
+            country="DE",
             carrier="heat",
         )
     configure_logging(snakemake)

scripts/write_statistics.py

@@ -4,21 +4,16 @@
 #
 # SPDX-License-Identifier: MIT
 
+import logging
+
 import matplotlib.pyplot as plt
 import pandas as pd
 import pypsa
 import seaborn as sns
 from _helpers import configure_logging
-from pypsa.statistics import get_carrier
-
+from pypsa.statistics import get_carrier, get_country_and_carrier
 
-# grouperfunctions = hier schreiben und dann in statistics.
-def groupby_country_and_carrier(n, c, nice_names=False):
-    df = n.df(c)
-    bus = "bus1" if "bus" not in n.df(c) else "bus"
-    country = df[bus].map(n.buses.location).map(n.buses.country).rename("country")
-    carrier = get_carrier(n, c, nice_names)
-    return [country, carrier]
+logger = logging.getLogger(__name__)
 
 
 def call_with_handle(func, **kwargs):
@@ -37,16 +32,18 @@ def call_with_handle(func, **kwargs):
 
         snakemake = mock_snakemake(
             "save_statistics_csv",
+            run="240219-test/normal",
             simpl="",
-            ll="v1.5",
-            clusters="5",
+            ll="v1.2",
+            clusters="22",
             opts="",
-            sector_opts="24h-T-H-B-I-A-dist1",
-            planning_horizons="2040",
-            country="all",
-            carrier="electricity",
+            sector_opts="none",
+            planning_horizons="2020",
+            country="DE",
+            carrier="H2",
         )
-    # configure_logging(snakemake)
+
+    configure_logging(snakemake)
     config = snakemake.config
 
     n = pypsa.Network(snakemake.input.network)
@@ -79,7 +76,7 @@ def call_with_handle(func, **kwargs):
     if country == "all" or not country:
         kwargs["groupby"] = get_carrier
     else:
-        kwargs["groupby"] = groupby_country_and_carrier
+        kwargs["groupby"] = get_country_and_carrier
 
     for output in snakemake.output.keys():
         if "touch" in output: