From 575bc96ee8df7a88b90833519ae6052408d9a897 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 15 Mar 2025 12:38:19 +0100 Subject: [PATCH 1/6] Clean up pure python --- .../src/python_workflow_definition/purepython.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/python_workflow_definition/src/python_workflow_definition/purepython.py b/python_workflow_definition/src/python_workflow_definition/purepython.py index 80c996a..624347b 100644 --- a/python_workflow_definition/src/python_workflow_definition/purepython.py +++ b/python_workflow_definition/src/python_workflow_definition/purepython.py @@ -3,7 +3,7 @@ from inspect import isfunction -from python_workflow_definition.shared import get_dict, get_list, get_kwargs +from python_workflow_definition.shared import get_dict, get_list, get_kwargs, get_source_handles def resort_total_lst(total_lst, nodes_dict): @@ -35,19 +35,6 @@ def group_edges(edges_lst): return total_lst -def get_source_handles(edges_lst): - source_handle_dict = {} - for ed in edges_lst: - if ed['source'] not in source_handle_dict.keys(): - source_handle_dict[ed['source']] = [ed['sourceHandle']] - else: - source_handle_dict[ed['source']].append(ed['sourceHandle']) - return { - k: list(range(len(v))) if len(v) > 1 and all([el is None for el in v]) else v - for k, v in source_handle_dict.items() - } - - def get_value(result_dict, nodes_new_dict, link_dict): source, source_handle = link_dict["source"], link_dict["sourceHandle"] if source in result_dict.keys(): From 1b5c0fa8924668f01e06fd880dea9ef1c0dbef6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 15 Mar 2025 12:38:38 +0100 Subject: [PATCH 2/6] Add executorlib interface --- .../python_workflow_definition/executorlib.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 python_workflow_definition/src/python_workflow_definition/executorlib.py diff --git a/python_workflow_definition/src/python_workflow_definition/executorlib.py b/python_workflow_definition/src/python_workflow_definition/executorlib.py new file mode 100644 index 0000000..6e3aade --- /dev/null +++ b/python_workflow_definition/src/python_workflow_definition/executorlib.py @@ -0,0 +1,39 @@ +import json +from importlib import import_module +from inspect import isfunction + + +from python_workflow_definition.shared import get_dict, get_list, get_kwargs, get_source_handles +from python_workflow_definition.purepython import resort_total_lst, group_edges, get_value + + +def load_workflow_json(file_name, exe): + with open(file_name, "r") as f: + content = json.load(f) + + edges_new_lst = content["edges"] + nodes_new_dict = {} + for k, v in content["nodes"].items(): + if isinstance(v, str) and "." in v: + p, m = v.rsplit('.', 1) + mod = import_module(p) + nodes_new_dict[int(k)] = getattr(mod, m) + else: + nodes_new_dict[int(k)] = v + + total_lst = group_edges(edges_new_lst) + total_new_lst = resort_total_lst(total_lst=total_lst, nodes_dict=nodes_new_dict) + + result_dict = {} + last_key = None + for lst in total_new_lst: + node = nodes_new_dict[lst[0]] + if isfunction(node): + kwargs = { + k: get_value(result_dict=result_dict, nodes_new_dict=nodes_new_dict, link_dict=v) + for k, v in lst[1].items() + } + result_dict[lst[0]] = exe.submit(node, **kwargs) + last_key = lst[0] + + return result_dict[last_key] From 6adab90a05dcdd70ca1a5f47f0dc15cfe0b96e1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 15 Mar 2025 17:00:34 +0100 Subject: [PATCH 3/6] Add notebooks --- .github/workflows/executorlib.yml | 29 +++++++ .../python_workflow_definition/executorlib.py | 18 ++++- .../python_workflow_definition/purepython.py | 4 +- universal_qe_to_executorlib.ipynb | 1 + universal_simple_to_executorlib.ipynb | 78 +++++++++++++++++++ 5 files changed, 126 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/executorlib.yml create mode 100644 universal_qe_to_executorlib.ipynb create mode 100644 universal_simple_to_executorlib.ipynb diff --git a/.github/workflows/executorlib.yml b/.github/workflows/executorlib.yml new file mode 100644 index 0000000..b0d06a9 --- /dev/null +++ b/.github/workflows/executorlib.yml @@ -0,0 +1,29 @@ +name: executorlib + +on: + push: + branches: [ main ] + pull_request: + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + python-version: "3.12" + environment-file: environment.yml + auto-activate-base: false + - name: Tests + shell: bash -l {0} + run: | + pip install -e adis_tools + pip install -e python_workflow_definition + conda install -c conda-forge jupyter papermill + export ESPRESSO_PSEUDO=$(pwd)/espresso/pseudo + papermill universal_simple_to_executorlib.ipynb universal_simple_to_executorlib_out.ipynb -k "python3" + papermill universal_qe_to_executorlib.ipynb universal_qe_to_executorlib_out.ipynb -k "python3" diff --git a/python_workflow_definition/src/python_workflow_definition/executorlib.py b/python_workflow_definition/src/python_workflow_definition/executorlib.py index 6e3aade..9f6ec09 100644 --- a/python_workflow_definition/src/python_workflow_definition/executorlib.py +++ b/python_workflow_definition/src/python_workflow_definition/executorlib.py @@ -4,7 +4,21 @@ from python_workflow_definition.shared import get_dict, get_list, get_kwargs, get_source_handles -from python_workflow_definition.purepython import resort_total_lst, group_edges, get_value +from python_workflow_definition.purepython import resort_total_lst, group_edges + + +def _get_value(result_dict, nodes_new_dict, link_dict, exe): + source, source_handle = link_dict["source"], link_dict["sourceHandle"] + if source in result_dict.keys(): + result = result_dict[source] + elif source in nodes_new_dict.keys(): + result = nodes_new_dict[source] + else: + raise KeyError() + if source_handle is None: + return result + else: + return exe.submit(getattr, result, source_handle) def load_workflow_json(file_name, exe): @@ -30,7 +44,7 @@ def load_workflow_json(file_name, exe): node = nodes_new_dict[lst[0]] if isfunction(node): kwargs = { - k: get_value(result_dict=result_dict, nodes_new_dict=nodes_new_dict, link_dict=v) + k: _get_value(result_dict=result_dict, nodes_new_dict=nodes_new_dict, link_dict=v, exe=exe) for k, v in lst[1].items() } result_dict[lst[0]] = exe.submit(node, **kwargs) diff --git a/python_workflow_definition/src/python_workflow_definition/purepython.py b/python_workflow_definition/src/python_workflow_definition/purepython.py index 624347b..8e67055 100644 --- a/python_workflow_definition/src/python_workflow_definition/purepython.py +++ b/python_workflow_definition/src/python_workflow_definition/purepython.py @@ -35,7 +35,7 @@ def group_edges(edges_lst): return total_lst -def get_value(result_dict, nodes_new_dict, link_dict): +def _get_value(result_dict, nodes_new_dict, link_dict): source, source_handle = link_dict["source"], link_dict["sourceHandle"] if source in result_dict.keys(): result = result_dict[source] @@ -72,7 +72,7 @@ def load_workflow_json(file_name): node = nodes_new_dict[lst[0]] if isfunction(node): kwargs = { - k: get_value(result_dict=result_dict, nodes_new_dict=nodes_new_dict, link_dict=v) + k: _get_value(result_dict=result_dict, nodes_new_dict=nodes_new_dict, link_dict=v) for k, v in lst[1].items() } result_dict[lst[0]] = node(**kwargs) diff --git a/universal_qe_to_executorlib.ipynb b/universal_qe_to_executorlib.ipynb new file mode 100644 index 0000000..3edb34b --- /dev/null +++ b/universal_qe_to_executorlib.ipynb @@ -0,0 +1 @@ +{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.8","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"ae0b5775-fb9d-4150-be79-d48dfb0b2cf4","cell_type":"code","source":"from executorlib import SingleNodeExecutor","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"b0f920a2-c646-485b-af0f-48251e061163","cell_type":"code","source":"from python_workflow_definition.executorlib import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"46559120-d58a-492f-b734-db27424b5659","cell_type":"code","source":"with SingleNodeExecutor() as exe:\n result = load_workflow_json(file_name=\"workflow_qe.json\", exe=exe).result()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"b37f476d-b7f8-4400-8070-83f8f2eb8bf9","cell_type":"code","source":"result","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"336dfbea-7e5c-4950-a0b3-c027a2a8e935","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]} \ No newline at end of file diff --git a/universal_simple_to_executorlib.ipynb b/universal_simple_to_executorlib.ipynb new file mode 100644 index 0000000..eea1f7e --- /dev/null +++ b/universal_simple_to_executorlib.ipynb @@ -0,0 +1,78 @@ +{ + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.8", + "mimetype": "text/x-python", + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "pygments_lexer": "ipython3", + "nbconvert_exporter": "python", + "file_extension": ".py" + } + }, + "nbformat_minor": 5, + "nbformat": 4, + "cells": [ + { + "id": "ae0b5775-fb9d-4150-be79-d48dfb0b2cf4", + "cell_type": "code", + "source": "from executorlib import SingleNodeExecutor", + "metadata": { + "trusted": true + }, + "outputs": [], + "execution_count": null + }, + { + "id": "b0f920a2-c646-485b-af0f-48251e061163", + "cell_type": "code", + "source": "from python_workflow_definition.executorlib import load_workflow_json", + "metadata": { + "trusted": true + }, + "outputs": [], + "execution_count": null + }, + { + "id": "46559120-d58a-492f-b734-db27424b5659", + "cell_type": "code", + "source": [ + "with SingleNodeExecutor() as exe:\n", + " result = load_workflow_json(file_name=\"workflow_simple.json\", exe=exe).result()" + ], + "metadata": { + "trusted": true + }, + "outputs": [], + "execution_count": null + }, + { + "id": "b37f476d-b7f8-4400-8070-83f8f2eb8bf9", + "cell_type": "code", + "source": "result", + "metadata": { + "trusted": true + }, + "outputs": [], + "execution_count": null + }, + { + "id": "336dfbea-7e5c-4950-a0b3-c027a2a8e935", + "cell_type": "code", + "source": "", + "metadata": { + "trusted": true + }, + "outputs": [], + "execution_count": null + } + ] +} From fa500fdeb68b95074fe32bcfe2ec35cfaf2e18bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 15 Mar 2025 17:09:36 +0100 Subject: [PATCH 4/6] fix syntax --- .../src/python_workflow_definition/executorlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_workflow_definition/src/python_workflow_definition/executorlib.py b/python_workflow_definition/src/python_workflow_definition/executorlib.py index 9f6ec09..0e914f0 100644 --- a/python_workflow_definition/src/python_workflow_definition/executorlib.py +++ b/python_workflow_definition/src/python_workflow_definition/executorlib.py @@ -18,7 +18,7 @@ def _get_value(result_dict, nodes_new_dict, link_dict, exe): if source_handle is None: return result else: - return exe.submit(getattr, result, source_handle) + return exe.submit(fn=getattr, object=result, name=source_handle) def load_workflow_json(file_name, exe): From ab17eab26f0ad3439734540b69690d2d83a532e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 15 Mar 2025 19:16:56 +0100 Subject: [PATCH 5/6] get_item() --- .../src/python_workflow_definition/executorlib.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python_workflow_definition/src/python_workflow_definition/executorlib.py b/python_workflow_definition/src/python_workflow_definition/executorlib.py index 0e914f0..a8fba77 100644 --- a/python_workflow_definition/src/python_workflow_definition/executorlib.py +++ b/python_workflow_definition/src/python_workflow_definition/executorlib.py @@ -7,6 +7,10 @@ from python_workflow_definition.purepython import resort_total_lst, group_edges +def get_item(obj, key): + return obj[key] + + def _get_value(result_dict, nodes_new_dict, link_dict, exe): source, source_handle = link_dict["source"], link_dict["sourceHandle"] if source in result_dict.keys(): @@ -18,7 +22,7 @@ def _get_value(result_dict, nodes_new_dict, link_dict, exe): if source_handle is None: return result else: - return exe.submit(fn=getattr, object=result, name=source_handle) + return exe.submit(fn=get_item, obj=result, key=source_handle) def load_workflow_json(file_name, exe): From a6d4388a9733ea68ce602eaeffc2b8fa057d246b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 15 Mar 2025 19:49:00 +0100 Subject: [PATCH 6/6] update notebooks --- universal_qe_to_executorlib.ipynb | 2 +- universal_simple_to_executorlib.ipynb | 79 +-------------------------- 2 files changed, 2 insertions(+), 79 deletions(-) diff --git a/universal_qe_to_executorlib.ipynb b/universal_qe_to_executorlib.ipynb index 3edb34b..81ed777 100644 --- a/universal_qe_to_executorlib.ipynb +++ b/universal_qe_to_executorlib.ipynb @@ -1 +1 @@ -{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.8","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"ae0b5775-fb9d-4150-be79-d48dfb0b2cf4","cell_type":"code","source":"from executorlib import SingleNodeExecutor","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"b0f920a2-c646-485b-af0f-48251e061163","cell_type":"code","source":"from python_workflow_definition.executorlib import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"46559120-d58a-492f-b734-db27424b5659","cell_type":"code","source":"with SingleNodeExecutor() as exe:\n result = load_workflow_json(file_name=\"workflow_qe.json\", exe=exe).result()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"b37f476d-b7f8-4400-8070-83f8f2eb8bf9","cell_type":"code","source":"result","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"336dfbea-7e5c-4950-a0b3-c027a2a8e935","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]} \ No newline at end of file +{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.8","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"ae0b5775-fb9d-4150-be79-d48dfb0b2cf4","cell_type":"code","source":"from executorlib import SingleNodeExecutor","metadata":{"trusted":true},"outputs":[],"execution_count":1},{"id":"b0f920a2-c646-485b-af0f-48251e061163","cell_type":"code","source":"from python_workflow_definition.executorlib import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":2},{"id":"46559120-d58a-492f-b734-db27424b5659","cell_type":"code","source":"with SingleNodeExecutor(max_workers=1) as exe:\n result = load_workflow_json(file_name=\"workflow_qe.json\", exe=exe).result()","metadata":{"trusted":true},"outputs":[{"name":"stderr","output_type":"stream","text":"[jupyter-pyiron-dev-pyth-flow-definition-4gvxle22:01655] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n[jupyter-pyiron-dev-pyth-flow-definition-4gvxle22:01891] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n[jupyter-pyiron-dev-pyth-flow-definition-4gvxle22:01962] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n[jupyter-pyiron-dev-pyth-flow-definition-4gvxle22:02029] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n[jupyter-pyiron-dev-pyth-flow-definition-4gvxle22:02093] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n[jupyter-pyiron-dev-pyth-flow-definition-4gvxle22:02159] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n"}],"execution_count":3},{"id":"b37f476d-b7f8-4400-8070-83f8f2eb8bf9","cell_type":"code","source":"result","metadata":{"trusted":true},"outputs":[],"execution_count":4},{"id":"336dfbea-7e5c-4950-a0b3-c027a2a8e935","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]} \ No newline at end of file diff --git a/universal_simple_to_executorlib.ipynb b/universal_simple_to_executorlib.ipynb index eea1f7e..5b0d6ee 100644 --- a/universal_simple_to_executorlib.ipynb +++ b/universal_simple_to_executorlib.ipynb @@ -1,78 +1 @@ -{ - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.12.8", - "mimetype": "text/x-python", - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "pygments_lexer": "ipython3", - "nbconvert_exporter": "python", - "file_extension": ".py" - } - }, - "nbformat_minor": 5, - "nbformat": 4, - "cells": [ - { - "id": "ae0b5775-fb9d-4150-be79-d48dfb0b2cf4", - "cell_type": "code", - "source": "from executorlib import SingleNodeExecutor", - "metadata": { - "trusted": true - }, - "outputs": [], - "execution_count": null - }, - { - "id": "b0f920a2-c646-485b-af0f-48251e061163", - "cell_type": "code", - "source": "from python_workflow_definition.executorlib import load_workflow_json", - "metadata": { - "trusted": true - }, - "outputs": [], - "execution_count": null - }, - { - "id": "46559120-d58a-492f-b734-db27424b5659", - "cell_type": "code", - "source": [ - "with SingleNodeExecutor() as exe:\n", - " result = load_workflow_json(file_name=\"workflow_simple.json\", exe=exe).result()" - ], - "metadata": { - "trusted": true - }, - "outputs": [], - "execution_count": null - }, - { - "id": "b37f476d-b7f8-4400-8070-83f8f2eb8bf9", - "cell_type": "code", - "source": "result", - "metadata": { - "trusted": true - }, - "outputs": [], - "execution_count": null - }, - { - "id": "336dfbea-7e5c-4950-a0b3-c027a2a8e935", - "cell_type": "code", - "source": "", - "metadata": { - "trusted": true - }, - "outputs": [], - "execution_count": null - } - ] -} +{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.8","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"ae0b5775-fb9d-4150-be79-d48dfb0b2cf4","cell_type":"code","source":"from executorlib import SingleNodeExecutor","metadata":{"trusted":true},"outputs":[],"execution_count":1},{"id":"b0f920a2-c646-485b-af0f-48251e061163","cell_type":"code","source":"from python_workflow_definition.executorlib import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":2},{"id":"46559120-d58a-492f-b734-db27424b5659","cell_type":"code","source":"with SingleNodeExecutor(max_workers=1) as exe:\n result = load_workflow_json(file_name=\"workflow_simple.json\", exe=exe).result()","metadata":{"trusted":true},"outputs":[],"execution_count":3},{"id":"b37f476d-b7f8-4400-8070-83f8f2eb8bf9","cell_type":"code","source":"result","metadata":{"trusted":true},"outputs":[{"execution_count":4,"output_type":"execute_result","data":{"text/plain":"6"},"metadata":{}}],"execution_count":4},{"id":"336dfbea-7e5c-4950-a0b3-c027a2a8e935","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]} \ No newline at end of file