From 67ef38b35eb2e46d8dcc5252216b73ee062465f0 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 12 May 2023 13:59:41 +0200 Subject: [PATCH 1/4] retrieve*: handle extraction path via snakemake.output --- scripts/retrieve_databundle.py | 3 +-- scripts/retrieve_gas_infrastructure_data.py | 2 +- scripts/retrieve_sector_databundle.py | 16 +++++++++------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/scripts/retrieve_databundle.py b/scripts/retrieve_databundle.py index 0c6a7feb5..75d8519e1 100644 --- a/scripts/retrieve_databundle.py +++ b/scripts/retrieve_databundle.py @@ -58,9 +58,8 @@ else: url = "https://zenodo.org/record/3517935/files/pypsa-eur-data-bundle.tar.xz" - # Save locations tarball_fn = Path(f"{rootpath}/bundle.tar.xz") - to_fn = Path(f"{rootpath}/data") + to_fn = Path(rootpath) / Path(snakemake.output[0]).parent.parent logger.info(f"Downloading databundle from '{url}'.") disable_progress = snakemake.config["run"].get("disable_progressbar", False) diff --git a/scripts/retrieve_gas_infrastructure_data.py b/scripts/retrieve_gas_infrastructure_data.py index dda7bd8cf..42b726dbd 100644 --- a/scripts/retrieve_gas_infrastructure_data.py +++ b/scripts/retrieve_gas_infrastructure_data.py @@ -29,7 +29,7 @@ # Save locations zip_fn = Path(f"{rootpath}/IGGIELGN.zip") - to_fn = Path(f"{rootpath}/data/gas_network/scigrid-gas") + to_fn = Path(rootpath) / Path(snakemake.output[0]).parent.parent logger.info(f"Downloading databundle from '{url}'.") disable_progress = snakemake.config["run"].get("disable_progressbar", False) diff --git a/scripts/retrieve_sector_databundle.py b/scripts/retrieve_sector_databundle.py index 97426ab21..0991bbe3f 100644 --- a/scripts/retrieve_sector_databundle.py +++ b/scripts/retrieve_sector_databundle.py @@ -10,23 +10,25 @@ logger = logging.getLogger(__name__) -import os -import sys import tarfile from pathlib import Path -# Add pypsa-eur scripts to path for import of _helpers -sys.path.insert(0, os.getcwd() + "/../pypsa-eur/scripts") - from _helpers import configure_logging, progress_retrieve if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake("retrieve_databundle") + rootpath = ".." + else: + rootpath = "." configure_logging(snakemake) url = "https://zenodo.org/record/5824485/files/pypsa-eur-sec-data-bundle.tar.gz" - tarball_fn = Path("sector-bundle.tar.gz") - to_fn = Path("data") + tarball_fn = Path(f"{rootpath}/sector-bundle.tar.gz") + to_fn = Path(rootpath) / Path(snakemake.output[0]).parent logger.info(f"Downloading databundle from '{url}'.") disable_progress = snakemake.config["run"].get("disable_progressbar", False) From b2216355f10bf11147cc7f03e74b17f6dc4b60e6 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 12 May 2023 14:10:04 +0200 Subject: [PATCH 2/4] retrieve_sector_data: add another .parent --- scripts/retrieve_sector_databundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/retrieve_sector_databundle.py b/scripts/retrieve_sector_databundle.py index 0991bbe3f..0d172c8d1 100644 --- a/scripts/retrieve_sector_databundle.py +++ b/scripts/retrieve_sector_databundle.py @@ -28,7 +28,7 @@ url = "https://zenodo.org/record/5824485/files/pypsa-eur-sec-data-bundle.tar.gz" tarball_fn = Path(f"{rootpath}/sector-bundle.tar.gz") - to_fn = Path(rootpath) / Path(snakemake.output[0]).parent + to_fn = Path(rootpath) / Path(snakemake.output[0]).parent.parent logger.info(f"Downloading databundle from '{url}'.") disable_progress = snakemake.config["run"].get("disable_progressbar", False) From 8f91963e7623d6cf574c0fd267b80b581f9560d8 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 12 May 2023 14:50:58 +0200 Subject: [PATCH 3/4] properly reference p_max_pu files --- scripts/add_electricity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index f910dee44..85ab35e2f 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -418,7 +418,7 @@ def attach_conventional_generators( if f"conventional_{carrier}_{attr}" in conventional_inputs: # Values affecting generators of technology k country-specific # First map generator buses to countries; then map countries to p_max_pu - values = pd.read_csv(values, index_col=0).iloc[:, 0] + values = pd.read_csv(snakemake.input[f"conventional_{carrier}_{attr}"], index_col=0).iloc[:, 0] bus_values = n.buses.country.map(values) n.generators[attr].update( n.generators.loc[idx].bus.map(bus_values).dropna() From 38bae672da42db241eb885eceeb3a6097644eeb5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 12 May 2023 12:51:39 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/add_electricity.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 85ab35e2f..4afa3e22c 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -418,7 +418,9 @@ def attach_conventional_generators( if f"conventional_{carrier}_{attr}" in conventional_inputs: # Values affecting generators of technology k country-specific # First map generator buses to countries; then map countries to p_max_pu - values = pd.read_csv(snakemake.input[f"conventional_{carrier}_{attr}"], index_col=0).iloc[:, 0] + values = pd.read_csv( + snakemake.input[f"conventional_{carrier}_{attr}"], index_col=0 + ).iloc[:, 0] bus_values = n.buses.country.map(values) n.generators[attr].update( n.generators.loc[idx].bus.map(bus_values).dropna()