From 4062779d4339e6e09f93f993aa399743fe06ceca Mon Sep 17 00:00:00 2001 From: Max Parzen Date: Mon, 17 Apr 2023 09:37:00 +0100 Subject: [PATCH 1/5] add zenodo handler for CLI upload --- scripts/zenodo_handler.py | 107 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 scripts/zenodo_handler.py diff --git a/scripts/zenodo_handler.py b/scripts/zenodo_handler.py new file mode 100644 index 000000000..d50c49498 --- /dev/null +++ b/scripts/zenodo_handler.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +# -*- coding: utf-8 -*- +""" +Uploads local files to Zenodo or sandbox.Zenodo. + +Requirements +------------ +- Install zenodopy: pip git+https://github.com/pz-max/zenodopy@patch-4 + +- Setup zenodopy (token): https://github.com/pz-max/zenodopy/tree/patch-4#using-the-package + +- Be aware of Zenodo REST API e.g. for metadata modifications: https://developers.zenodo.org/#introduction + +Relevant Settings +----------------- +""" +import zenodopy +from pathlib import Path + +###################### +# INPUTS AND OPTIONS # +###################### +SANDBOX_BOOL = True # sandbox should be used for testing +NEW_PROJECT = False # if False, use existing project ID +EXISTING_PROJECT_ID = 1183583 +TYPE = 'upload' # 'delete' or 'upload' files +PUBLISH_BOOL = False # publish repos CANNOT be deleted +ROOT = "/home/max/Downloads" # example "/home/max/Downloads" +UPLOAD_PATHS = [ + "tutorial_data_MA.zip", + "cutouts_NGBJ.zip", + ] # list of files from root e.g. "README.md" in /home/max/README.md, each component will receive a download link +DELETE_PATHS = [ + "README.md", + ] # list of files from root e.g. "README.md" in /home/max/README.md, +METADATA = { + 'title': 'PyPSA-Earth (Dataset)', + 'upload_type': 'other', + 'description': 'Used for model https://github.com/pypsa-meets-earth/pypsa-earth. Multiple data licenses apply https://pypsa-earth.readthedocs.io/en/latest/introduction.html#license', + 'creators': [ + { + 'name': 'PyPSA-Earth Authors', + 'affiliation': 'PyPSA meets Earth' + }, + ], + 'access_right': 'open', + 'license': {'id': 'cc-by-4.0'}, + 'keywords': ['Macro Energy Systems', 'Power Systems'], +} # more opton visisble at Zenodo REST API https://developers.zenodo.org/#introduction + + +############# +# EXECUTION # +############# +zeno = zenodopy.Client(sandbox=SANDBOX_BOOL) # test is API key is set +zeno.list_projects + + +if NEW_PROJECT == True: + zeno.create_project(title=METADATA["title"]) + zeno.change_metadata( + dep_id=zeno.deposition_id, + metadata=METADATA, + ) + for path in UPLOAD_PATHS: + path = Path.joinpath(Path(ROOT), path) + zeno.upload_zip(source_dir=str(path)) + + +if NEW_PROJECT == False: + zeno.set_project(dep_id=EXISTING_PROJECT_ID) + zeno.change_metadata( + dep_id=zeno.deposition_id, + metadata=METADATA, + ) + for path in UPLOAD_PATHS: + path = Path.joinpath(Path(ROOT), path) + if path.exists(): + if TYPE == 'upload': + if path.exists(): + try: + if path.is_file(): + zeno.upload_file(str(path)) + elif path.is_dir(): + zeno.upload_zip(str(path)) + except: + zeno.update(path) + continue + + if TYPE == 'delete': + try: + zeno.delete_file(str(path)) + except: + print(f"Cannot delete {path}. Repo needs to be in edit mode.") + else: + raise FileNotFoundError(f"{path} does not exist") + + +if PUBLISH_BOOL == True: + zeno.publish() + + +zeno.list_projects From 09637c0637b58d03a25a165a78c1b1f7665dc1e6 Mon Sep 17 00:00:00 2001 From: Max Parzen Date: Mon, 17 Apr 2023 09:45:16 +0100 Subject: [PATCH 2/5] add zenodo links --- configs/bundle_config.yaml | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/configs/bundle_config.yaml b/configs/bundle_config.yaml index 73f143eea..74b536bbf 100644 --- a/configs/bundle_config.yaml +++ b/configs/bundle_config.yaml @@ -36,6 +36,7 @@ databundles: category: data destination: "data" urls: + zenodo: https://sandbox.zenodo.org/record/1183583/files/tutorial_data_NGBJ.zip?download=1 gdrive: https://drive.google.com/file/d/1BSxmm3AG5TTOxrLG0ki8IdlL2n3hEviq/view output: - data/gebco/GEBCO_2021_TID.nc @@ -48,6 +49,7 @@ databundles: category: data destination: "data" urls: + zenodo: https://sandbox.zenodo.org/record/1183583/files/tutorial_data_BW.zip?download=1 gdrive: https://drive.google.com/file/d/1aXqbCxEpe0aNI6EYuCLtFS6CbdXurTn7/view output: - data/gebco/GEBCO_2021_TID.nc @@ -60,6 +62,7 @@ databundles: category: data destination: "data" urls: + zenodo: https://sandbox.zenodo.org/record/1183583/files/tutorial_data_MA.zip?download=1 gdrive: https://drive.google.com/file/d/1zY3WnKqwBF0derS4ncRIPpdTdNn978xw/view output: - data/gebco/GEBCO_2021_TID.nc @@ -72,6 +75,7 @@ databundles: category: cutouts destination: "cutouts" urls: + zenodo: https://sandbox.zenodo.org/record/1183583/files/cutouts_NGBJ.zip?download=1 gdrive: https://drive.google.com/file/d/1N6wJb-dBTqWZX1Tlf5y979lEPtqcVEfQ/view output: [cutouts/africa-2013-era5-tutorial.nc] disable_by_opt: @@ -84,6 +88,7 @@ databundles: category: cutouts destination: "cutouts" urls: + zenodo: https://sandbox.zenodo.org/record/1183583/files/cutouts_BW.zip?download=1 gdrive: https://drive.google.com/file/d/1vtckYOZeSugDXrE0PGqxFwsVkdHNimhV/view output: [cutouts/africa-2013-era5-tutorial.nc] disable_by_opt: @@ -96,6 +101,7 @@ databundles: category: cutouts destination: "cutouts" urls: + zenodo: https://sandbox.zenodo.org/record/1183583/files/cutouts_MA.zip?download=1 gdrive: https://drive.google.com/file/d/1ouIhGQ4e2M2FLzPUZgcKu5GSpRPZotqw/view output: [cutouts/africa-2013-era5-tutorial.nc] disable_by_opt: @@ -108,6 +114,7 @@ databundles: category: common destination: "data" urls: + zenodo: https://sandbox.zenodo.org/record/1183583/files/tutorial_data_general.zip?download=1 gdrive: https://drive.google.com/file/d/1nRLrs_kP0qVl-IHC4BFLjpoKa3HLk2Py/view output: - data/costs.csv @@ -124,7 +131,7 @@ databundles: category: common destination: "data" urls: - # zenodo: https://sandbox.zenodo.org/record/1016540/files/common_data.zip?download=1 + zenodo: https://sandbox.zenodo.org/record/1183583/files/data_v0.1.zip?download=1 gdrive: https://drive.google.com/file/d/1jN5iV_iaRa5lc1XOaEtletKPUgPNOVuf/view output: - data/costs.csv @@ -142,7 +149,7 @@ databundles: category: natura destination: "data" urls: - # zenodo: https://sandbox.zenodo.org/record/1016878/files/resources_africa.zip?download=1 + zenodo: https://sandbox.zenodo.org/record/1183583/files/natura_global.zip?download=1 gdrive: https://drive.google.com/file/d/1WmAAwY0TAcHg8YZyAGof0bPewFacm7my/view output: - data/natura.tiff @@ -153,7 +160,7 @@ databundles: category: landcover destination: "data/landcover" urls: - zenodo: https://sandbox.zenodo.org/record/1016042/files/landcover_africa.zip?download=1 + zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_africa.zip?download=1 gdrive: https://drive.google.com/file/d/1qnJ0HAWZ4z_9q3PtDEUrFmZWBZYg-WNW/view output: [data/landcover/*] @@ -163,7 +170,7 @@ databundles: category: landcover destination: "data/landcover" urls: - zenodo: https://sandbox.zenodo.org/record/1016042/files/landcover_asiapacific.zip?download=1 + zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_asiapacific.zip?download=1 gdrive: https://drive.google.com/file/d/163Uc9F1DPs2pRgDzkTivnblhXCw4nfd5/view output: [data/landcover/*] @@ -173,7 +180,7 @@ databundles: category: landcover destination: "data/landcover" urls: - zenodo: https://sandbox.zenodo.org/record/1016042/files/landcover_latinamerica_caribbean.zip?download=1 + zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_latinamerica_caribbean.zip?download=1 gdrive: https://drive.google.com/file/d/1d3l9qxPnM9XbXODWkP4kWxK_t5cDcEIZ/view output: [data/landcover/*] @@ -183,7 +190,7 @@ databundles: category: landcover destination: "data/landcover" urls: - zenodo: https://sandbox.zenodo.org/record/1016042/files/landcover_westasia.zip?download=1 + zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_westasia.zip?download=1 gdrive: https://drive.google.com/file/d/1XsZVZDORUFMKyHfAkzqxDVBENiuy4jEM/view output: [data/landcover/*] @@ -193,7 +200,7 @@ databundles: category: landcover destination: "data/landcover" urls: - zenodo: https://sandbox.zenodo.org/record/1018640/files/landcover_europe.zip?download=1 + zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_europe.zip?download=1 gdrive: https://drive.google.com/file/d/1MfT3cwHJbNTtTjRLzW2T19f9qBNAxdhw/view output: [data/landcover/*] @@ -203,7 +210,7 @@ databundles: category: landcover destination: "data/landcover" urls: - zenodo: https://sandbox.zenodo.org/record/1018640/files/landcover_northamerica.zip?download=1 + zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_northamerica.zip?download=1 gdrive: https://drive.google.com/file/d/1Dc11J2BZQQeJWxwCC6GiiPd_GWMCN0uF/view output: [data/landcover/*] @@ -213,7 +220,7 @@ databundles: category: landcover destination: "data/landcover" urls: - zenodo: https://sandbox.zenodo.org/record/1018640/files/landcover_polar.zip?download=1 + zenodo: https://sandbox.zenodo.org/record/1183583/files/landcover_polar.zip?download=1 gdrive: https://drive.google.com/file/d/18uODyONQZJGoL15P-URPl0lJs7sphQhX/view output: [data/landcover/*] @@ -266,7 +273,7 @@ databundles: category: cutouts destination: "cutouts" urls: - zenodo: https://sandbox.zenodo.org/record/1016042/files/cutouts_africa.zip?download=1 + zenodo: https://sandbox.zenodo.org/record/1183583/files/cutouts_africa.zip?download=1 gdrive: https://drive.google.com/file/d/1uCn7S5EHnPuaZHG5sWEVx2lT6Ve2r5dk/view output: [cutouts/africa-2013-era5.nc] disable_by_opt: From b456327e218a1a4987f5d681733e685357826b6d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 17 Apr 2023 09:52:59 +0000 Subject: [PATCH 3/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/zenodo_handler.py | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/scripts/zenodo_handler.py b/scripts/zenodo_handler.py index d50c49498..e382ad431 100644 --- a/scripts/zenodo_handler.py +++ b/scripts/zenodo_handler.py @@ -18,38 +18,36 @@ Relevant Settings ----------------- """ -import zenodopy from pathlib import Path +import zenodopy + ###################### # INPUTS AND OPTIONS # ###################### SANDBOX_BOOL = True # sandbox should be used for testing NEW_PROJECT = False # if False, use existing project ID EXISTING_PROJECT_ID = 1183583 -TYPE = 'upload' # 'delete' or 'upload' files +TYPE = "upload" # 'delete' or 'upload' files PUBLISH_BOOL = False # publish repos CANNOT be deleted ROOT = "/home/max/Downloads" # example "/home/max/Downloads" UPLOAD_PATHS = [ "tutorial_data_MA.zip", "cutouts_NGBJ.zip", - ] # list of files from root e.g. "README.md" in /home/max/README.md, each component will receive a download link +] # list of files from root e.g. "README.md" in /home/max/README.md, each component will receive a download link DELETE_PATHS = [ "README.md", - ] # list of files from root e.g. "README.md" in /home/max/README.md, +] # list of files from root e.g. "README.md" in /home/max/README.md, METADATA = { - 'title': 'PyPSA-Earth (Dataset)', - 'upload_type': 'other', - 'description': 'Used for model https://github.com/pypsa-meets-earth/pypsa-earth. Multiple data licenses apply https://pypsa-earth.readthedocs.io/en/latest/introduction.html#license', - 'creators': [ - { - 'name': 'PyPSA-Earth Authors', - 'affiliation': 'PyPSA meets Earth' - }, + "title": "PyPSA-Earth (Dataset)", + "upload_type": "other", + "description": "Used for model https://github.com/pypsa-meets-earth/pypsa-earth. Multiple data licenses apply https://pypsa-earth.readthedocs.io/en/latest/introduction.html#license", + "creators": [ + {"name": "PyPSA-Earth Authors", "affiliation": "PyPSA meets Earth"}, ], - 'access_right': 'open', - 'license': {'id': 'cc-by-4.0'}, - 'keywords': ['Macro Energy Systems', 'Power Systems'], + "access_right": "open", + "license": {"id": "cc-by-4.0"}, + "keywords": ["Macro Energy Systems", "Power Systems"], } # more opton visisble at Zenodo REST API https://developers.zenodo.org/#introduction @@ -65,8 +63,8 @@ zeno.change_metadata( dep_id=zeno.deposition_id, metadata=METADATA, - ) - for path in UPLOAD_PATHS: + ) + for path in UPLOAD_PATHS: path = Path.joinpath(Path(ROOT), path) zeno.upload_zip(source_dir=str(path)) @@ -76,11 +74,11 @@ zeno.change_metadata( dep_id=zeno.deposition_id, metadata=METADATA, - ) + ) for path in UPLOAD_PATHS: path = Path.joinpath(Path(ROOT), path) if path.exists(): - if TYPE == 'upload': + if TYPE == "upload": if path.exists(): try: if path.is_file(): @@ -91,7 +89,7 @@ zeno.update(path) continue - if TYPE == 'delete': + if TYPE == "delete": try: zeno.delete_file(str(path)) except: From 0a96fa7c4a396c645bbe629534466c9665cad241 Mon Sep 17 00:00:00 2001 From: Max Parzen Date: Mon, 17 Apr 2023 10:55:46 +0100 Subject: [PATCH 4/5] add release notes --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 5b14eebd4..0e8127fb1 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -88,6 +88,8 @@ Upcoming Release * Fix append bug in build_powerplants rule `PR #686 `__ +* Add *zenodo_handler.py* to update and upload files via code `PR #688 `__ + PyPSA-Earth 0.1.0 ================= From 8a5a8a2ac70b942b6b21cf992f1e3d6fa23e059b Mon Sep 17 00:00:00 2001 From: Max Parzen Date: Tue, 18 Apr 2023 07:57:40 +0100 Subject: [PATCH 5/5] create non_workflow scripts --- scripts/non_workflow/__init__.py | 4 ++++ scripts/{ => non_workflow}/zenodo_handler.py | 0 .../{zipfolders_googledrive.py => non_workflow/zip_folder.py} | 0 3 files changed, 4 insertions(+) create mode 100644 scripts/non_workflow/__init__.py rename scripts/{ => non_workflow}/zenodo_handler.py (100%) rename scripts/{zipfolders_googledrive.py => non_workflow/zip_folder.py} (100%) diff --git a/scripts/non_workflow/__init__.py b/scripts/non_workflow/__init__.py new file mode 100644 index 000000000..a635e74eb --- /dev/null +++ b/scripts/non_workflow/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors +# +# SPDX-License-Identifier: GPL-3.0-or-later diff --git a/scripts/zenodo_handler.py b/scripts/non_workflow/zenodo_handler.py similarity index 100% rename from scripts/zenodo_handler.py rename to scripts/non_workflow/zenodo_handler.py diff --git a/scripts/zipfolders_googledrive.py b/scripts/non_workflow/zip_folder.py similarity index 100% rename from scripts/zipfolders_googledrive.py rename to scripts/non_workflow/zip_folder.py