diff --git a/.github/workflows/executorlib.yml b/.github/workflows/executorlib.yml new file mode 100644 index 0000000..b0d06a9 --- /dev/null +++ b/.github/workflows/executorlib.yml @@ -0,0 +1,29 @@ +name: executorlib + +on: + push: + branches: [ main ] + pull_request: + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + python-version: "3.12" + environment-file: environment.yml + auto-activate-base: false + - name: Tests + shell: bash -l {0} + run: | + pip install -e adis_tools + pip install -e python_workflow_definition + conda install -c conda-forge jupyter papermill + export ESPRESSO_PSEUDO=$(pwd)/espresso/pseudo + papermill universal_simple_to_executorlib.ipynb universal_simple_to_executorlib_out.ipynb -k "python3" + papermill universal_qe_to_executorlib.ipynb universal_qe_to_executorlib_out.ipynb -k "python3" diff --git a/python_workflow_definition/src/python_workflow_definition/executorlib.py b/python_workflow_definition/src/python_workflow_definition/executorlib.py new file mode 100644 index 0000000..a8fba77 --- /dev/null +++ b/python_workflow_definition/src/python_workflow_definition/executorlib.py @@ -0,0 +1,57 @@ +import json +from importlib import import_module +from inspect import isfunction + + +from python_workflow_definition.shared import get_dict, get_list, get_kwargs, get_source_handles +from python_workflow_definition.purepython import resort_total_lst, group_edges + + +def get_item(obj, key): + return obj[key] + + +def _get_value(result_dict, nodes_new_dict, link_dict, exe): + source, source_handle = link_dict["source"], link_dict["sourceHandle"] + if source in result_dict.keys(): + result = result_dict[source] + elif source in nodes_new_dict.keys(): + result = nodes_new_dict[source] + else: + raise KeyError() + if source_handle is None: + return result + else: + return exe.submit(fn=get_item, obj=result, key=source_handle) + + +def load_workflow_json(file_name, exe): + with open(file_name, "r") as f: + content = json.load(f) + + edges_new_lst = content["edges"] + nodes_new_dict = {} + for k, v in content["nodes"].items(): + if isinstance(v, str) and "." in v: + p, m = v.rsplit('.', 1) + mod = import_module(p) + nodes_new_dict[int(k)] = getattr(mod, m) + else: + nodes_new_dict[int(k)] = v + + total_lst = group_edges(edges_new_lst) + total_new_lst = resort_total_lst(total_lst=total_lst, nodes_dict=nodes_new_dict) + + result_dict = {} + last_key = None + for lst in total_new_lst: + node = nodes_new_dict[lst[0]] + if isfunction(node): + kwargs = { + k: _get_value(result_dict=result_dict, nodes_new_dict=nodes_new_dict, link_dict=v, exe=exe) + for k, v in lst[1].items() + } + result_dict[lst[0]] = exe.submit(node, **kwargs) + last_key = lst[0] + + return result_dict[last_key] diff --git a/python_workflow_definition/src/python_workflow_definition/purepython.py b/python_workflow_definition/src/python_workflow_definition/purepython.py index 80c996a..8e67055 100644 --- a/python_workflow_definition/src/python_workflow_definition/purepython.py +++ b/python_workflow_definition/src/python_workflow_definition/purepython.py @@ -3,7 +3,7 @@ from inspect import isfunction -from python_workflow_definition.shared import get_dict, get_list, get_kwargs +from python_workflow_definition.shared import get_dict, get_list, get_kwargs, get_source_handles def resort_total_lst(total_lst, nodes_dict): @@ -35,20 +35,7 @@ def group_edges(edges_lst): return total_lst -def get_source_handles(edges_lst): - source_handle_dict = {} - for ed in edges_lst: - if ed['source'] not in source_handle_dict.keys(): - source_handle_dict[ed['source']] = [ed['sourceHandle']] - else: - source_handle_dict[ed['source']].append(ed['sourceHandle']) - return { - k: list(range(len(v))) if len(v) > 1 and all([el is None for el in v]) else v - for k, v in source_handle_dict.items() - } - - -def get_value(result_dict, nodes_new_dict, link_dict): +def _get_value(result_dict, nodes_new_dict, link_dict): source, source_handle = link_dict["source"], link_dict["sourceHandle"] if source in result_dict.keys(): result = result_dict[source] @@ -85,7 +72,7 @@ def load_workflow_json(file_name): node = nodes_new_dict[lst[0]] if isfunction(node): kwargs = { - k: get_value(result_dict=result_dict, nodes_new_dict=nodes_new_dict, link_dict=v) + k: _get_value(result_dict=result_dict, nodes_new_dict=nodes_new_dict, link_dict=v) for k, v in lst[1].items() } result_dict[lst[0]] = node(**kwargs) diff --git a/universal_qe_to_executorlib.ipynb b/universal_qe_to_executorlib.ipynb new file mode 100644 index 0000000..81ed777 --- /dev/null +++ b/universal_qe_to_executorlib.ipynb @@ -0,0 +1 @@ +{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.8","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"ae0b5775-fb9d-4150-be79-d48dfb0b2cf4","cell_type":"code","source":"from executorlib import SingleNodeExecutor","metadata":{"trusted":true},"outputs":[],"execution_count":1},{"id":"b0f920a2-c646-485b-af0f-48251e061163","cell_type":"code","source":"from python_workflow_definition.executorlib import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":2},{"id":"46559120-d58a-492f-b734-db27424b5659","cell_type":"code","source":"with SingleNodeExecutor(max_workers=1) as exe:\n result = load_workflow_json(file_name=\"workflow_qe.json\", exe=exe).result()","metadata":{"trusted":true},"outputs":[{"name":"stderr","output_type":"stream","text":"[jupyter-pyiron-dev-pyth-flow-definition-4gvxle22:01655] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n[jupyter-pyiron-dev-pyth-flow-definition-4gvxle22:01891] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n[jupyter-pyiron-dev-pyth-flow-definition-4gvxle22:01962] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n[jupyter-pyiron-dev-pyth-flow-definition-4gvxle22:02029] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n[jupyter-pyiron-dev-pyth-flow-definition-4gvxle22:02093] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n[jupyter-pyiron-dev-pyth-flow-definition-4gvxle22:02159] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n"}],"execution_count":3},{"id":"b37f476d-b7f8-4400-8070-83f8f2eb8bf9","cell_type":"code","source":"result","metadata":{"trusted":true},"outputs":[],"execution_count":4},{"id":"336dfbea-7e5c-4950-a0b3-c027a2a8e935","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]} \ No newline at end of file diff --git a/universal_simple_to_executorlib.ipynb b/universal_simple_to_executorlib.ipynb new file mode 100644 index 0000000..5b0d6ee --- /dev/null +++ b/universal_simple_to_executorlib.ipynb @@ -0,0 +1 @@ +{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.8","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"ae0b5775-fb9d-4150-be79-d48dfb0b2cf4","cell_type":"code","source":"from executorlib import SingleNodeExecutor","metadata":{"trusted":true},"outputs":[],"execution_count":1},{"id":"b0f920a2-c646-485b-af0f-48251e061163","cell_type":"code","source":"from python_workflow_definition.executorlib import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":2},{"id":"46559120-d58a-492f-b734-db27424b5659","cell_type":"code","source":"with SingleNodeExecutor(max_workers=1) as exe:\n result = load_workflow_json(file_name=\"workflow_simple.json\", exe=exe).result()","metadata":{"trusted":true},"outputs":[],"execution_count":3},{"id":"b37f476d-b7f8-4400-8070-83f8f2eb8bf9","cell_type":"code","source":"result","metadata":{"trusted":true},"outputs":[{"execution_count":4,"output_type":"execute_result","data":{"text/plain":"6"},"metadata":{}}],"execution_count":4},{"id":"336dfbea-7e5c-4950-a0b3-c027a2a8e935","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]} \ No newline at end of file