From 50a72c98111f940f0e03c26a9b6478a0e00b809b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Tue, 13 May 2025 21:36:56 +0200 Subject: [PATCH 01/23] Implement export to cwl --- .github/workflows/pipeline.yml | 1 + binder/environment.yml | 1 + example_workflows/arithmetic/cwl.ipynb | 1 + .../cwl/__init__.py | 0 .../cwl/__main__.py | 42 +++++ .../python_workflow_definition/cwl/export.py | 174 ++++++++++++++++++ 6 files changed, 219 insertions(+) create mode 100644 example_workflows/arithmetic/cwl.ipynb create mode 100644 python_workflow_definition/src/python_workflow_definition/cwl/__init__.py create mode 100644 python_workflow_definition/src/python_workflow_definition/cwl/__main__.py create mode 100644 python_workflow_definition/src/python_workflow_definition/cwl/export.py diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 9750b07..3a67689 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -128,6 +128,7 @@ jobs: run: | cd example_workflows/arithmetic papermill aiida.ipynb aiida_out.ipynb -k "python3" + papermill cwl.ipynb cwl_out.ipynb -k "python3" papermill jobflow.ipynb jobflow_out.ipynb -k "python3" papermill pyiron_base.ipynb pyiron_base_out.ipynb -k "python3" papermill universal_workflow.ipynb universal_workflow_out.ipynb -k "python3" diff --git a/binder/environment.yml b/binder/environment.yml index 67e2a37..00be8de 100644 --- a/binder/environment.yml +++ b/binder/environment.yml @@ -12,3 +12,4 @@ dependencies: - aiida-workgraph =0.5.2 - conda_subprocess =0.0.6 - networkx =3.4.2 +- cwltool =3.1.20250110105449 diff --git a/example_workflows/arithmetic/cwl.ipynb b/example_workflows/arithmetic/cwl.ipynb new file mode 100644 index 0000000..6ee23f2 --- /dev/null +++ b/example_workflows/arithmetic/cwl.ipynb @@ -0,0 +1 @@ +{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.10"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"377fef56-484d-491c-b19e-1be6931e44eb","cell_type":"code","source":"import pickle","metadata":{"trusted":true},"outputs":[],"execution_count":4},{"id":"92e3921b-2bb8-4333-8cfe-4bd27f785d24","cell_type":"code","source":"from python_workflow_definition.cwl.export import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":17},{"id":"5303c059-8ae4-4557-858e-b4bd64eac711","cell_type":"code","source":"load_workflow_json(file_name=\"workflow.json\")","metadata":{"trusted":true},"outputs":[],"execution_count":18},{"id":"df302bd2-e9b6-4595-979c-67c46414d986","cell_type":"code","source":"! cwltool workflow.cwl workflow.yml","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"/srv/conda/envs/notebook/bin/cwltool:11: DeprecationWarning: Nesting argument groups is deprecated.\n sys.exit(run())\n\u001b[1;30mINFO\u001b[0m /srv/conda/envs/notebook/bin/cwltool 3.1.20250110105449\n\u001b[1;30mINFO\u001b[0m Resolved 'workflow.cwl' to 'file:///home/jovyan/workflow.cwl'\n\u001b[1;30mINFO\u001b[0m [workflow ] start\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_prod_and_div\n\u001b[1;30mINFO\u001b[0m [step get_prod_and_div] start\n\u001b[1;30mINFO\u001b[0m [job get_prod_and_div] /tmp/abngt5eq$ python \\\n /tmp/7pxhgcet/stg6ba69a25-7dc0-46be-9bbe-5c08e6ce1925/wrapper.py \\\n --function=workflow.get_prod_and_div \\\n --arg_x=/tmp/7pxhgcet/stg007c65c6-d1cd-44cd-b761-e181c0cc37d8/x.pickle \\\n --arg_y=/tmp/7pxhgcet/stg23ac56f7-9c1d-43d9-a90c-86036b3cbafc/y.pickle\n\u001b[1;30mINFO\u001b[0m [job get_prod_and_div] completed success\n\u001b[1;30mINFO\u001b[0m [step get_prod_and_div] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_sum\n\u001b[1;30mINFO\u001b[0m [step get_sum] start\n\u001b[1;30mINFO\u001b[0m [job get_sum] /tmp/wzuzdwec$ python \\\n /tmp/lz9a89rn/stg44b61930-48fe-423d-a68a-593e237e0477/wrapper.py \\\n --function=workflow.get_sum \\\n --arg_x=/tmp/lz9a89rn/stgb7917545-93cd-45b4-833a-01559a22e620/prod.pickle \\\n --arg_y=/tmp/lz9a89rn/stg0c04cf0f-bc08-494d-ae4b-b12bde14f87a/div.pickle\n\u001b[1;30mINFO\u001b[0m [job get_sum] completed success\n\u001b[1;30mINFO\u001b[0m [step get_sum] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] completed success\n{\n \"result_file\": {\n \"location\": \"file:///home/jovyan/result.pickle\",\n \"basename\": \"result.pickle\",\n \"class\": \"File\",\n \"checksum\": \"sha1$3dfd802cefb317cc7138af1e3a299f565c74ddec\",\n \"size\": 21,\n \"path\": \"/home/jovyan/result.pickle\"\n }\n}\u001b[1;30mINFO\u001b[0m Final process status is success\n"}],"execution_count":19},{"id":"2942dbba-ea0a-4d20-be5c-ed9992d09ff8","cell_type":"code","source":"with open(\"result.pickle\", \"rb\") as f:\n print(pickle.load(f))","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"2.5\n"}],"execution_count":20},{"id":"60e909ee-d0d0-4bd1-81c8-dd5274ae5834","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]} \ No newline at end of file diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/__init__.py b/python_workflow_definition/src/python_workflow_definition/cwl/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py b/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py new file mode 100644 index 0000000..803f9c3 --- /dev/null +++ b/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py @@ -0,0 +1,42 @@ +import sys +import pickle +from ast import literal_eval +from importlib import import_module + + +def load_function(funct): + p, m = funct.rsplit('.', 1) + return getattr(import_module(p), m) + + +def convert_argument(arg): + if ".pickle" in arg: + with open(arg, 'rb') as f: + return pickle.load(f) + else: + return literal_eval(arg) + + +if __name__ == "__main__": + # load input + argument_lst = sys.argv[1:] + funct = [ + load_function(funct=arg.split("=")[-1]) + for arg in argument_lst if "--function=" in arg + ][0] + kwargs = { + arg.split("=")[0][6:]: convert_argument(arg=arg.split("=")[-1]) + for arg in argument_lst if "--arg_" in arg + } + + # evaluate function + result = funct(**kwargs) + + # store output + if isinstance(result, dict): + for k, v in result.items(): + with open(k + ".pickle", 'wb') as f: + pickle.dump(v, f) + else: + with open("result.pickle", 'wb') as f: + pickle.dump(result, f) \ No newline at end of file diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/export.py b/python_workflow_definition/src/python_workflow_definition/cwl/export.py new file mode 100644 index 0000000..30a5e1f --- /dev/null +++ b/python_workflow_definition/src/python_workflow_definition/cwl/export.py @@ -0,0 +1,174 @@ +import json +import pickle +from yaml import CDumper as Dumper, dump + + +from python_workflow_definition.purepython import ( + group_edges, + resort_total_lst, +) +from python_workflow_definition.shared import ( + convert_nodes_list_to_dict, + remove_result, + EDGES_LABEL, + NODES_LABEL, + TARGET_LABEL, + TARGET_PORT_LABEL, + SOURCE_LABEL, + SOURCE_PORT_LABEL, +) + + +def get_function_argument(argument: str, position: int = 3) -> dict: + return { + argument + '_file': { + 'type': 'File', + 'inputBinding': {'prefix': '--arg_' + argument + '=', 'separate': False, 'position': position}, + }, + } + + +def get_function_template(function_name: str) -> dict: + return { + 'function': { + 'default': function_name, + 'inputBinding': {'position': 2, 'prefix': '--function=', 'separate': False}, + 'type': 'string', + }, + } + + +def get_output_name(output_name: str) -> dict: + return { + output_name + '_file': { + 'type': 'File', + 'outputBinding': { + 'glob': output_name + '.pickle' + } + } + } + + +def get_function(workflow): + function_nodes_dict = { + n['id']: n['value'] + for n in workflow[NODES_LABEL] if n["type"] == "function" + } + funct_dict = {} + for funct_id in function_nodes_dict.keys(): + target_ports = list(set([e[TARGET_PORT_LABEL] for e in workflow[EDGES_LABEL] if e["target"] == funct_id])) + source_ports = list(set([e[SOURCE_PORT_LABEL] for e in workflow[EDGES_LABEL] if e["source"] == funct_id])) + funct_dict[funct_id] = {"targetPorts": target_ports, "sourcePorts": source_ports} + return function_nodes_dict, funct_dict + + +def write_function_cwl(workflow): + function_nodes_dict, funct_dict = get_function(workflow) + file_lst = [] + + for i in range(len(function_nodes_dict)): + template = { + 'cwlVersion': 'v1.2', + 'class': 'CommandLineTool', + 'baseCommand': 'python', + 'inputs': { + 'wrapper': { + 'type': 'File', + 'inputBinding': {'position': 1}, + 'default': {'class': 'File', 'location': 'wrapper.py'} + }, + }, + 'outputs': { + } + } + file_name = function_nodes_dict[i].split(".")[-1] + ".cwl" + if file_name not in file_lst: + file_lst.append(file_name) + template["inputs"].update(get_function_template(function_name=function_nodes_dict[i])) + for j, arg in enumerate(funct_dict[i]['targetPorts']): + template["inputs"].update(get_function_argument(argument=arg, position=3+j)) + for out in funct_dict[i]['sourcePorts']: + if out is None: + template["outputs"].update(get_output_name(output_name="result")) + else: + template["outputs"].update(get_output_name(output_name=out)) + with open(file_name, "w") as f: + dump(template, f, Dumper=Dumper) + + +def write_workflow_config(workflow): + input_dict = { + n["name"]: n["value"] + for n in workflow[NODES_LABEL] if n["type"] == "input" + } + with open("workflow.yml", "w") as f: + dump( + { + k + "_file": {"class": "File", "path": k + ".pickle"} + for k in input_dict.keys() + }, + f, + Dumper=Dumper, + ) + for k, v in input_dict.items(): + with open(k + ".pickle", "wb") as f: + pickle.dump(v, f) + + +def write_workflow(workflow): + workflow_template = { + 'cwlVersion': 'v1.2', + 'class': 'Workflow', + 'inputs': {}, + 'steps': {}, + 'outputs': {}, + } + input_dict = { + n["name"]: n["value"] + for n in workflow[NODES_LABEL] if n["type"] == "input" + } + function_nodes_dict, funct_dict = get_function(workflow) + result_id = [n["id"] for n in workflow[NODES_LABEL] if n["type"] == "output"][0] + last_compute_id = [e[SOURCE_LABEL] for e in workflow[EDGES_LABEL] if e[TARGET_LABEL] == result_id][0] + workflow_template["inputs"].update({k + "_file": "File" for k in input_dict.keys()}) + if funct_dict[last_compute_id]["sourcePorts"] == [None]: + workflow_template["outputs"] = { + "result_file": { + "type": "File", + "outputSource": function_nodes_dict[last_compute_id].split(".")[-1] + "/result_file" + }, + } + else: + raise ValueError() + + content = remove_result(workflow_dict=workflow) + edges_new_lst = content[EDGES_LABEL] + total_lst = group_edges(edges_new_lst) + nodes_new_dict = { + int(k): v for k, v in convert_nodes_list_to_dict(nodes_list=content[NODES_LABEL]).items() + } + total_new_lst = resort_total_lst(total_lst=total_lst, nodes_dict=nodes_new_dict) + step_name_lst = {t[0]: function_nodes_dict[t[0]].split(".")[-1] for t in total_new_lst} + input_id_dict = {n["id"]: n["name"] for n in workflow[NODES_LABEL] if n["type"] == "input"} + for t in total_new_lst: + ind = t[0] + node_script = step_name_lst[ind] + ".cwl" + output = [o + "_file" if o is not None else "result_file" for o in funct_dict[ind]['sourcePorts']] + in_dict = {} + for k, v in t[1].items(): + if v[SOURCE_LABEL] in input_id_dict: + in_dict[k + "_file"] = input_id_dict[v[SOURCE_LABEL]] + "_file" + else: + in_dict[k + "_file"] = step_name_lst[v[SOURCE_LABEL]] + "/" + v[SOURCE_PORT_LABEL] + "_file" + workflow_template["steps"].update({step_name_lst[ind]: {"run": node_script, "in": in_dict, "out": output}}) + with open("workflow.cwl", "w") as f: + dump(workflow_template, f, Dumper=Dumper) + + +def load_workflow_json(file_name: str): + with open(file_name, "r") as f: + workflow = json.load(f) + + write_function_cwl(workflow=workflow) + write_workflow_config(workflow=workflow) + write_workflow(workflow=workflow) From 2d68d5aead8beee6b827af2e504b6d23d3e352c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Tue, 13 May 2025 21:43:37 +0200 Subject: [PATCH 02/23] fixes --- .../cwl/__main__.py | 16 +- .../python_workflow_definition/cwl/export.py | 164 +++++++++++------- 2 files changed, 110 insertions(+), 70 deletions(-) diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py b/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py index 803f9c3..aefb6a9 100644 --- a/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py +++ b/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py @@ -5,13 +5,13 @@ def load_function(funct): - p, m = funct.rsplit('.', 1) + p, m = funct.rsplit(".", 1) return getattr(import_module(p), m) def convert_argument(arg): if ".pickle" in arg: - with open(arg, 'rb') as f: + with open(arg, "rb") as f: return pickle.load(f) else: return literal_eval(arg) @@ -22,11 +22,13 @@ def convert_argument(arg): argument_lst = sys.argv[1:] funct = [ load_function(funct=arg.split("=")[-1]) - for arg in argument_lst if "--function=" in arg + for arg in argument_lst + if "--function=" in arg ][0] kwargs = { arg.split("=")[0][6:]: convert_argument(arg=arg.split("=")[-1]) - for arg in argument_lst if "--arg_" in arg + for arg in argument_lst + if "--arg_" in arg } # evaluate function @@ -35,8 +37,8 @@ def convert_argument(arg): # store output if isinstance(result, dict): for k, v in result.items(): - with open(k + ".pickle", 'wb') as f: + with open(k + ".pickle", "wb") as f: pickle.dump(v, f) else: - with open("result.pickle", 'wb') as f: - pickle.dump(result, f) \ No newline at end of file + with open("result.pickle", "wb") as f: + pickle.dump(result, f) diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/export.py b/python_workflow_definition/src/python_workflow_definition/cwl/export.py index 30a5e1f..2e25d09 100644 --- a/python_workflow_definition/src/python_workflow_definition/cwl/export.py +++ b/python_workflow_definition/src/python_workflow_definition/cwl/export.py @@ -19,87 +19,108 @@ ) -def get_function_argument(argument: str, position: int = 3) -> dict: +def _get_function_argument(argument: str, position: int = 3) -> dict: return { - argument + '_file': { - 'type': 'File', - 'inputBinding': {'prefix': '--arg_' + argument + '=', 'separate': False, 'position': position}, + argument + + "_file": { + "type": "File", + "inputBinding": { + "prefix": "--arg_" + argument + "=", + "separate": False, + "position": position, + }, }, } -def get_function_template(function_name: str) -> dict: +def _get_function_template(function_name: str) -> dict: return { - 'function': { - 'default': function_name, - 'inputBinding': {'position': 2, 'prefix': '--function=', 'separate': False}, - 'type': 'string', + "function": { + "default": function_name, + "inputBinding": {"position": 2, "prefix": "--function=", "separate": False}, + "type": "string", }, } -def get_output_name(output_name: str) -> dict: +def _get_output_name(output_name: str) -> dict: return { - output_name + '_file': { - 'type': 'File', - 'outputBinding': { - 'glob': output_name + '.pickle' - } - } + output_name + + "_file": {"type": "File", "outputBinding": {"glob": output_name + ".pickle"}} } -def get_function(workflow): +def _get_function(workflow): function_nodes_dict = { - n['id']: n['value'] - for n in workflow[NODES_LABEL] if n["type"] == "function" + n["id"]: n["value"] for n in workflow[NODES_LABEL] if n["type"] == "function" } funct_dict = {} for funct_id in function_nodes_dict.keys(): - target_ports = list(set([e[TARGET_PORT_LABEL] for e in workflow[EDGES_LABEL] if e["target"] == funct_id])) - source_ports = list(set([e[SOURCE_PORT_LABEL] for e in workflow[EDGES_LABEL] if e["source"] == funct_id])) - funct_dict[funct_id] = {"targetPorts": target_ports, "sourcePorts": source_ports} + target_ports = list( + set( + [ + e[TARGET_PORT_LABEL] + for e in workflow[EDGES_LABEL] + if e["target"] == funct_id + ] + ) + ) + source_ports = list( + set( + [ + e[SOURCE_PORT_LABEL] + for e in workflow[EDGES_LABEL] + if e["source"] == funct_id + ] + ) + ) + funct_dict[funct_id] = { + "targetPorts": target_ports, + "sourcePorts": source_ports, + } return function_nodes_dict, funct_dict -def write_function_cwl(workflow): - function_nodes_dict, funct_dict = get_function(workflow) +def _write_function_cwl(workflow): + function_nodes_dict, funct_dict = _get_function(workflow) file_lst = [] for i in range(len(function_nodes_dict)): template = { - 'cwlVersion': 'v1.2', - 'class': 'CommandLineTool', - 'baseCommand': 'python', - 'inputs': { - 'wrapper': { - 'type': 'File', - 'inputBinding': {'position': 1}, - 'default': {'class': 'File', 'location': 'wrapper.py'} + "cwlVersion": "v1.2", + "class": "CommandLineTool", + "baseCommand": "python", + "inputs": { + "wrapper": { + "type": "string", + "inputBinding": {"position": 1, "prefix": "-m"}, + "default": "python_workflow_definition.cwl", }, }, - 'outputs': { - } + "outputs": {}, } file_name = function_nodes_dict[i].split(".")[-1] + ".cwl" if file_name not in file_lst: file_lst.append(file_name) - template["inputs"].update(get_function_template(function_name=function_nodes_dict[i])) - for j, arg in enumerate(funct_dict[i]['targetPorts']): - template["inputs"].update(get_function_argument(argument=arg, position=3+j)) - for out in funct_dict[i]['sourcePorts']: + template["inputs"].update( + _get_function_template(function_name=function_nodes_dict[i]) + ) + for j, arg in enumerate(funct_dict[i]["targetPorts"]): + template["inputs"].update( + _get_function_argument(argument=arg, position=3 + j) + ) + for out in funct_dict[i]["sourcePorts"]: if out is None: - template["outputs"].update(get_output_name(output_name="result")) + template["outputs"].update(_get_output_name(output_name="result")) else: - template["outputs"].update(get_output_name(output_name=out)) + template["outputs"].update(_get_output_name(output_name=out)) with open(file_name, "w") as f: dump(template, f, Dumper=Dumper) -def write_workflow_config(workflow): +def _write_workflow_config(workflow): input_dict = { - n["name"]: n["value"] - for n in workflow[NODES_LABEL] if n["type"] == "input" + n["name"]: n["value"] for n in workflow[NODES_LABEL] if n["type"] == "input" } with open("workflow.yml", "w") as f: dump( @@ -115,27 +136,29 @@ def write_workflow_config(workflow): pickle.dump(v, f) -def write_workflow(workflow): +def _write_workflow(workflow): workflow_template = { - 'cwlVersion': 'v1.2', - 'class': 'Workflow', - 'inputs': {}, - 'steps': {}, - 'outputs': {}, + "cwlVersion": "v1.2", + "class": "Workflow", + "inputs": {}, + "steps": {}, + "outputs": {}, } input_dict = { - n["name"]: n["value"] - for n in workflow[NODES_LABEL] if n["type"] == "input" + n["name"]: n["value"] for n in workflow[NODES_LABEL] if n["type"] == "input" } - function_nodes_dict, funct_dict = get_function(workflow) + function_nodes_dict, funct_dict = _get_function(workflow) result_id = [n["id"] for n in workflow[NODES_LABEL] if n["type"] == "output"][0] - last_compute_id = [e[SOURCE_LABEL] for e in workflow[EDGES_LABEL] if e[TARGET_LABEL] == result_id][0] + last_compute_id = [ + e[SOURCE_LABEL] for e in workflow[EDGES_LABEL] if e[TARGET_LABEL] == result_id + ][0] workflow_template["inputs"].update({k + "_file": "File" for k in input_dict.keys()}) if funct_dict[last_compute_id]["sourcePorts"] == [None]: workflow_template["outputs"] = { "result_file": { "type": "File", - "outputSource": function_nodes_dict[last_compute_id].split(".")[-1] + "/result_file" + "outputSource": function_nodes_dict[last_compute_id].split(".")[-1] + + "/result_file", }, } else: @@ -145,22 +168,37 @@ def write_workflow(workflow): edges_new_lst = content[EDGES_LABEL] total_lst = group_edges(edges_new_lst) nodes_new_dict = { - int(k): v for k, v in convert_nodes_list_to_dict(nodes_list=content[NODES_LABEL]).items() + int(k): v + for k, v in convert_nodes_list_to_dict(nodes_list=content[NODES_LABEL]).items() } total_new_lst = resort_total_lst(total_lst=total_lst, nodes_dict=nodes_new_dict) - step_name_lst = {t[0]: function_nodes_dict[t[0]].split(".")[-1] for t in total_new_lst} - input_id_dict = {n["id"]: n["name"] for n in workflow[NODES_LABEL] if n["type"] == "input"} + step_name_lst = { + t[0]: function_nodes_dict[t[0]].split(".")[-1] for t in total_new_lst + } + input_id_dict = { + n["id"]: n["name"] for n in workflow[NODES_LABEL] if n["type"] == "input" + } for t in total_new_lst: ind = t[0] node_script = step_name_lst[ind] + ".cwl" - output = [o + "_file" if o is not None else "result_file" for o in funct_dict[ind]['sourcePorts']] + output = [ + o + "_file" if o is not None else "result_file" + for o in funct_dict[ind]["sourcePorts"] + ] in_dict = {} for k, v in t[1].items(): if v[SOURCE_LABEL] in input_id_dict: in_dict[k + "_file"] = input_id_dict[v[SOURCE_LABEL]] + "_file" else: - in_dict[k + "_file"] = step_name_lst[v[SOURCE_LABEL]] + "/" + v[SOURCE_PORT_LABEL] + "_file" - workflow_template["steps"].update({step_name_lst[ind]: {"run": node_script, "in": in_dict, "out": output}}) + in_dict[k + "_file"] = ( + step_name_lst[v[SOURCE_LABEL]] + + "/" + + v[SOURCE_PORT_LABEL] + + "_file" + ) + workflow_template["steps"].update( + {step_name_lst[ind]: {"run": node_script, "in": in_dict, "out": output}} + ) with open("workflow.cwl", "w") as f: dump(workflow_template, f, Dumper=Dumper) @@ -169,6 +207,6 @@ def load_workflow_json(file_name: str): with open(file_name, "r") as f: workflow = json.load(f) - write_function_cwl(workflow=workflow) - write_workflow_config(workflow=workflow) - write_workflow(workflow=workflow) + _write_function_cwl(workflow=workflow) + _write_workflow_config(workflow=workflow) + _write_workflow(workflow=workflow) From a8d0045f775058d9e26a351238728fda18d8627e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Tue, 13 May 2025 22:52:15 +0200 Subject: [PATCH 03/23] copy workflow file --- .../python_workflow_definition/cwl/__main__.py | 2 ++ .../src/python_workflow_definition/cwl/export.py | 15 +++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py b/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py index aefb6a9..8af8c84 100644 --- a/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py +++ b/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py @@ -1,4 +1,6 @@ import sys +import os +import shutil import pickle from ast import literal_eval from importlib import import_module diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/export.py b/python_workflow_definition/src/python_workflow_definition/cwl/export.py index 2e25d09..4cf6360 100644 --- a/python_workflow_definition/src/python_workflow_definition/cwl/export.py +++ b/python_workflow_definition/src/python_workflow_definition/cwl/export.py @@ -37,7 +37,7 @@ def _get_function_template(function_name: str) -> dict: return { "function": { "default": function_name, - "inputBinding": {"position": 2, "prefix": "--function=", "separate": False}, + "inputBinding": {"position": 3, "prefix": "--function=", "separate": False}, "type": "string", }, } @@ -96,9 +96,20 @@ def _write_function_cwl(workflow): "inputBinding": {"position": 1, "prefix": "-m"}, "default": "python_workflow_definition.cwl", }, + "workflowfile": { + "type": "string", + "inputBinding": { + "position": 2, + "separate": False, + }, + "default": "workflow.py", + }, }, "outputs": {}, } + template["inputs"]["workflowfile"]["default"] = ( + function_nodes_dict[i].split(".")[0] + ".py" + ) file_name = function_nodes_dict[i].split(".")[-1] + ".cwl" if file_name not in file_lst: file_lst.append(file_name) @@ -107,7 +118,7 @@ def _write_function_cwl(workflow): ) for j, arg in enumerate(funct_dict[i]["targetPorts"]): template["inputs"].update( - _get_function_argument(argument=arg, position=3 + j) + _get_function_argument(argument=arg, position=4 + j) ) for out in funct_dict[i]["sourcePorts"]: if out is None: From a7e63ce0e5aba8c6ee8461ae3ae097407c0553c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Tue, 13 May 2025 22:59:02 +0200 Subject: [PATCH 04/23] fix --- .../src/python_workflow_definition/cwl/export.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/export.py b/python_workflow_definition/src/python_workflow_definition/cwl/export.py index 4cf6360..a0b7ed9 100644 --- a/python_workflow_definition/src/python_workflow_definition/cwl/export.py +++ b/python_workflow_definition/src/python_workflow_definition/cwl/export.py @@ -97,12 +97,9 @@ def _write_function_cwl(workflow): "default": "python_workflow_definition.cwl", }, "workflowfile": { - "type": "string", - "inputBinding": { - "position": 2, - "separate": False, - }, - "default": "workflow.py", + "type": "File", + "inputBinding": {"position": 2}, + "default": {"class": "File", "location": "workflow.py"}, }, }, "outputs": {}, From c4a9a044900967fc571f509ae4e8dc2baa934224 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Tue, 13 May 2025 23:03:55 +0200 Subject: [PATCH 05/23] fix path --- .../src/python_workflow_definition/cwl/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/export.py b/python_workflow_definition/src/python_workflow_definition/cwl/export.py index a0b7ed9..25b227b 100644 --- a/python_workflow_definition/src/python_workflow_definition/cwl/export.py +++ b/python_workflow_definition/src/python_workflow_definition/cwl/export.py @@ -104,7 +104,7 @@ def _write_function_cwl(workflow): }, "outputs": {}, } - template["inputs"]["workflowfile"]["default"] = ( + template["inputs"]["workflowfile"]["default"]["location"] = ( function_nodes_dict[i].split(".")[0] + ".py" ) file_name = function_nodes_dict[i].split(".")[-1] + ".cwl" From 30e5eb5a9b110cc17cac6308c84905e66ca76406 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Tue, 13 May 2025 23:42:48 +0200 Subject: [PATCH 06/23] copy file --- .../src/python_workflow_definition/cwl/__main__.py | 6 ++++++ .../src/python_workflow_definition/cwl/export.py | 8 ++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py b/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py index 8af8c84..476c19f 100644 --- a/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py +++ b/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py @@ -27,6 +27,12 @@ def convert_argument(arg): for arg in argument_lst if "--function=" in arg ][0] + file = [ + load_function(funct=arg.split("=")[-1]) + for arg in argument_lst + if "--workflowfile=" in arg + ][0] + shutil.copyfile(file, os.curdir) kwargs = { arg.split("=")[0][6:]: convert_argument(arg=arg.split("=")[-1]) for arg in argument_lst diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/export.py b/python_workflow_definition/src/python_workflow_definition/cwl/export.py index 25b227b..c505fb6 100644 --- a/python_workflow_definition/src/python_workflow_definition/cwl/export.py +++ b/python_workflow_definition/src/python_workflow_definition/cwl/export.py @@ -98,18 +98,18 @@ def _write_function_cwl(workflow): }, "workflowfile": { "type": "File", - "inputBinding": {"position": 2}, + "inputBinding": {"position": 2, "prefix": "--workflowfile=", "separate": False}, "default": {"class": "File", "location": "workflow.py"}, }, }, "outputs": {}, } - template["inputs"]["workflowfile"]["default"]["location"] = ( - function_nodes_dict[i].split(".")[0] + ".py" - ) file_name = function_nodes_dict[i].split(".")[-1] + ".cwl" if file_name not in file_lst: file_lst.append(file_name) + template["inputs"]["workflowfile"]["default"]["location"] = ( + function_nodes_dict[i].split(".")[0] + ".py" + ) template["inputs"].update( _get_function_template(function_name=function_nodes_dict[i]) ) From cfb042585e5b5f50cfa317c38e2c93b778bb0511 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Tue, 13 May 2025 23:52:12 +0200 Subject: [PATCH 07/23] last fix for tonight --- .../python_workflow_definition/cwl/__main__.py | 16 ++++++---------- .../src/python_workflow_definition/cwl/export.py | 6 +++++- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py b/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py index 476c19f..b2f5a35 100644 --- a/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py +++ b/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py @@ -1,14 +1,16 @@ import sys -import os -import shutil import pickle from ast import literal_eval -from importlib import import_module +import importlib.util def load_function(funct): p, m = funct.rsplit(".", 1) - return getattr(import_module(p), m) + spec = importlib.util.spec_from_file_location(p, p + ".py") + module = importlib.util.module_from_spec(spec) + sys.modules[p] = module + spec.loader.exec_module(module) + return getattr(module, m) def convert_argument(arg): @@ -27,12 +29,6 @@ def convert_argument(arg): for arg in argument_lst if "--function=" in arg ][0] - file = [ - load_function(funct=arg.split("=")[-1]) - for arg in argument_lst - if "--workflowfile=" in arg - ][0] - shutil.copyfile(file, os.curdir) kwargs = { arg.split("=")[0][6:]: convert_argument(arg=arg.split("=")[-1]) for arg in argument_lst diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/export.py b/python_workflow_definition/src/python_workflow_definition/cwl/export.py index c505fb6..9650ab0 100644 --- a/python_workflow_definition/src/python_workflow_definition/cwl/export.py +++ b/python_workflow_definition/src/python_workflow_definition/cwl/export.py @@ -98,7 +98,11 @@ def _write_function_cwl(workflow): }, "workflowfile": { "type": "File", - "inputBinding": {"position": 2, "prefix": "--workflowfile=", "separate": False}, + "inputBinding": { + "position": 2, + "prefix": "--workflowfile=", + "separate": False, + }, "default": {"class": "File", "location": "workflow.py"}, }, }, From 57c5c4d9580a4265796508114c60c3117ea12b8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Wed, 14 May 2025 00:00:58 +0200 Subject: [PATCH 08/23] another try --- .../cwl/__main__.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py b/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py index b2f5a35..92b50b2 100644 --- a/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py +++ b/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py @@ -4,13 +4,12 @@ import importlib.util -def load_function(funct): - p, m = funct.rsplit(".", 1) - spec = importlib.util.spec_from_file_location(p, p + ".py") +def load_function(file_name, funct): + spec = importlib.util.spec_from_file_location("workflow", file_name) module = importlib.util.module_from_spec(spec) - sys.modules[p] = module + sys.modules["workflow"] = module spec.loader.exec_module(module) - return getattr(module, m) + return getattr(module, funct) def convert_argument(arg): @@ -24,11 +23,9 @@ def convert_argument(arg): if __name__ == "__main__": # load input argument_lst = sys.argv[1:] - funct = [ - load_function(funct=arg.split("=")[-1]) - for arg in argument_lst - if "--function=" in arg - ][0] + funct = [arg.split("=")[-1] for arg in argument_lst if "--function=" in arg][0] + file = [arg.split("=")[-1] for arg in argument_lst if "--workflowfile=" in arg][0] + workflow_function = load_function(file_name=file, funct=funct) kwargs = { arg.split("=")[0][6:]: convert_argument(arg=arg.split("=")[-1]) for arg in argument_lst @@ -36,7 +33,7 @@ def convert_argument(arg): } # evaluate function - result = funct(**kwargs) + result = workflow_function(**kwargs) # store output if isinstance(result, dict): From 157ce593c1372b6f58e2946c7413ae7ceaf83662 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Wed, 14 May 2025 00:11:32 +0200 Subject: [PATCH 09/23] fix function name --- .../src/python_workflow_definition/cwl/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py b/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py index 92b50b2..a70a3ff 100644 --- a/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py +++ b/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py @@ -9,7 +9,7 @@ def load_function(file_name, funct): module = importlib.util.module_from_spec(spec) sys.modules["workflow"] = module spec.loader.exec_module(module) - return getattr(module, funct) + return getattr(module, funct.split(".")[-1]) def convert_argument(arg): From 92ee192d3883ea99527dbfebcb3643203fcc42fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Wed, 14 May 2025 00:15:21 +0200 Subject: [PATCH 10/23] number all steps --- .../src/python_workflow_definition/cwl/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/export.py b/python_workflow_definition/src/python_workflow_definition/cwl/export.py index 9650ab0..6c9e1bb 100644 --- a/python_workflow_definition/src/python_workflow_definition/cwl/export.py +++ b/python_workflow_definition/src/python_workflow_definition/cwl/export.py @@ -209,7 +209,7 @@ def _write_workflow(workflow): + "_file" ) workflow_template["steps"].update( - {step_name_lst[ind]: {"run": node_script, "in": in_dict, "out": output}} + {step_name_lst[ind] + "_" + str(ind): {"run": node_script, "in": in_dict, "out": output}} ) with open("workflow.cwl", "w") as f: dump(workflow_template, f, Dumper=Dumper) From 25874883a4c44ede4c25756d7e31e4b8a901e1f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Wed, 14 May 2025 00:19:00 +0200 Subject: [PATCH 11/23] fix links --- .../src/python_workflow_definition/cwl/export.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/export.py b/python_workflow_definition/src/python_workflow_definition/cwl/export.py index 6c9e1bb..55b471c 100644 --- a/python_workflow_definition/src/python_workflow_definition/cwl/export.py +++ b/python_workflow_definition/src/python_workflow_definition/cwl/export.py @@ -170,6 +170,8 @@ def _write_workflow(workflow): "result_file": { "type": "File", "outputSource": function_nodes_dict[last_compute_id].split(".")[-1] + + "_" + + str(last_compute_id) + "/result_file", }, } @@ -204,12 +206,18 @@ def _write_workflow(workflow): else: in_dict[k + "_file"] = ( step_name_lst[v[SOURCE_LABEL]] + + "_" + + str(v[SOURCE_LABEL]) + "/" + v[SOURCE_PORT_LABEL] + "_file" ) workflow_template["steps"].update( - {step_name_lst[ind] + "_" + str(ind): {"run": node_script, "in": in_dict, "out": output}} + { + step_name_lst[ind] + + "_" + + str(ind): {"run": node_script, "in": in_dict, "out": output} + } ) with open("workflow.cwl", "w") as f: dump(workflow_template, f, Dumper=Dumper) From 1dbaf9cc10381c72f3c480a23f7fa03315b5dc8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Wed, 14 May 2025 00:21:00 +0200 Subject: [PATCH 12/23] add more CWL workflows --- .github/workflows/pipeline.yml | 2 ++ example_workflows/nfdi/cwl.ipynb | 1 + example_workflows/quantum_espresso/cwl.ipynb | 1 + 3 files changed, 4 insertions(+) create mode 100644 example_workflows/nfdi/cwl.ipynb create mode 100644 example_workflows/quantum_espresso/cwl.ipynb diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 3a67689..24afdc7 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -68,6 +68,7 @@ jobs: run: | cd example_workflows/nfdi/ papermill aiida.ipynb aiida_out.ipynb -k "python3" + papermill cwl.ipynb cwl_out.ipynb -k "python3" papermill jobflow.ipynb jobflow_out.ipynb -k "python3" papermill pyiron_base.ipynb pyiron_base_out.ipynb -k "python3" papermill universal_workflow.ipynb universal_workflow_out.ipynb -k "python3" @@ -101,6 +102,7 @@ jobs: cp -r example_workflows/quantum_espresso/espresso . cd example_workflows/quantum_espresso papermill aiida.ipynb aiida_out.ipynb -k "python3" + papermill cwl.ipynb cwl_out.ipynb -k "python3" papermill jobflow.ipynb jobflow_out.ipynb -k "python3" papermill pyiron_base.ipynb pyiron_base_out.ipynb -k "python3" papermill universal_workflow.ipynb universal_workflow_out.ipynb -k "python3" diff --git a/example_workflows/nfdi/cwl.ipynb b/example_workflows/nfdi/cwl.ipynb new file mode 100644 index 0000000..6ee23f2 --- /dev/null +++ b/example_workflows/nfdi/cwl.ipynb @@ -0,0 +1 @@ +{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.10"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"377fef56-484d-491c-b19e-1be6931e44eb","cell_type":"code","source":"import pickle","metadata":{"trusted":true},"outputs":[],"execution_count":4},{"id":"92e3921b-2bb8-4333-8cfe-4bd27f785d24","cell_type":"code","source":"from python_workflow_definition.cwl.export import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":17},{"id":"5303c059-8ae4-4557-858e-b4bd64eac711","cell_type":"code","source":"load_workflow_json(file_name=\"workflow.json\")","metadata":{"trusted":true},"outputs":[],"execution_count":18},{"id":"df302bd2-e9b6-4595-979c-67c46414d986","cell_type":"code","source":"! cwltool workflow.cwl workflow.yml","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"/srv/conda/envs/notebook/bin/cwltool:11: DeprecationWarning: Nesting argument groups is deprecated.\n sys.exit(run())\n\u001b[1;30mINFO\u001b[0m /srv/conda/envs/notebook/bin/cwltool 3.1.20250110105449\n\u001b[1;30mINFO\u001b[0m Resolved 'workflow.cwl' to 'file:///home/jovyan/workflow.cwl'\n\u001b[1;30mINFO\u001b[0m [workflow ] start\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_prod_and_div\n\u001b[1;30mINFO\u001b[0m [step get_prod_and_div] start\n\u001b[1;30mINFO\u001b[0m [job get_prod_and_div] /tmp/abngt5eq$ python \\\n /tmp/7pxhgcet/stg6ba69a25-7dc0-46be-9bbe-5c08e6ce1925/wrapper.py \\\n --function=workflow.get_prod_and_div \\\n --arg_x=/tmp/7pxhgcet/stg007c65c6-d1cd-44cd-b761-e181c0cc37d8/x.pickle \\\n --arg_y=/tmp/7pxhgcet/stg23ac56f7-9c1d-43d9-a90c-86036b3cbafc/y.pickle\n\u001b[1;30mINFO\u001b[0m [job get_prod_and_div] completed success\n\u001b[1;30mINFO\u001b[0m [step get_prod_and_div] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_sum\n\u001b[1;30mINFO\u001b[0m [step get_sum] start\n\u001b[1;30mINFO\u001b[0m [job get_sum] /tmp/wzuzdwec$ python \\\n /tmp/lz9a89rn/stg44b61930-48fe-423d-a68a-593e237e0477/wrapper.py \\\n --function=workflow.get_sum \\\n --arg_x=/tmp/lz9a89rn/stgb7917545-93cd-45b4-833a-01559a22e620/prod.pickle \\\n --arg_y=/tmp/lz9a89rn/stg0c04cf0f-bc08-494d-ae4b-b12bde14f87a/div.pickle\n\u001b[1;30mINFO\u001b[0m [job get_sum] completed success\n\u001b[1;30mINFO\u001b[0m [step get_sum] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] completed success\n{\n \"result_file\": {\n \"location\": \"file:///home/jovyan/result.pickle\",\n \"basename\": \"result.pickle\",\n \"class\": \"File\",\n \"checksum\": \"sha1$3dfd802cefb317cc7138af1e3a299f565c74ddec\",\n \"size\": 21,\n \"path\": \"/home/jovyan/result.pickle\"\n }\n}\u001b[1;30mINFO\u001b[0m Final process status is success\n"}],"execution_count":19},{"id":"2942dbba-ea0a-4d20-be5c-ed9992d09ff8","cell_type":"code","source":"with open(\"result.pickle\", \"rb\") as f:\n print(pickle.load(f))","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"2.5\n"}],"execution_count":20},{"id":"60e909ee-d0d0-4bd1-81c8-dd5274ae5834","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]} \ No newline at end of file diff --git a/example_workflows/quantum_espresso/cwl.ipynb b/example_workflows/quantum_espresso/cwl.ipynb new file mode 100644 index 0000000..6ee23f2 --- /dev/null +++ b/example_workflows/quantum_espresso/cwl.ipynb @@ -0,0 +1 @@ +{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.10"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"377fef56-484d-491c-b19e-1be6931e44eb","cell_type":"code","source":"import pickle","metadata":{"trusted":true},"outputs":[],"execution_count":4},{"id":"92e3921b-2bb8-4333-8cfe-4bd27f785d24","cell_type":"code","source":"from python_workflow_definition.cwl.export import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":17},{"id":"5303c059-8ae4-4557-858e-b4bd64eac711","cell_type":"code","source":"load_workflow_json(file_name=\"workflow.json\")","metadata":{"trusted":true},"outputs":[],"execution_count":18},{"id":"df302bd2-e9b6-4595-979c-67c46414d986","cell_type":"code","source":"! cwltool workflow.cwl workflow.yml","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"/srv/conda/envs/notebook/bin/cwltool:11: DeprecationWarning: Nesting argument groups is deprecated.\n sys.exit(run())\n\u001b[1;30mINFO\u001b[0m /srv/conda/envs/notebook/bin/cwltool 3.1.20250110105449\n\u001b[1;30mINFO\u001b[0m Resolved 'workflow.cwl' to 'file:///home/jovyan/workflow.cwl'\n\u001b[1;30mINFO\u001b[0m [workflow ] start\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_prod_and_div\n\u001b[1;30mINFO\u001b[0m [step get_prod_and_div] start\n\u001b[1;30mINFO\u001b[0m [job get_prod_and_div] /tmp/abngt5eq$ python \\\n /tmp/7pxhgcet/stg6ba69a25-7dc0-46be-9bbe-5c08e6ce1925/wrapper.py \\\n --function=workflow.get_prod_and_div \\\n --arg_x=/tmp/7pxhgcet/stg007c65c6-d1cd-44cd-b761-e181c0cc37d8/x.pickle \\\n --arg_y=/tmp/7pxhgcet/stg23ac56f7-9c1d-43d9-a90c-86036b3cbafc/y.pickle\n\u001b[1;30mINFO\u001b[0m [job get_prod_and_div] completed success\n\u001b[1;30mINFO\u001b[0m [step get_prod_and_div] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_sum\n\u001b[1;30mINFO\u001b[0m [step get_sum] start\n\u001b[1;30mINFO\u001b[0m [job get_sum] /tmp/wzuzdwec$ python \\\n /tmp/lz9a89rn/stg44b61930-48fe-423d-a68a-593e237e0477/wrapper.py \\\n --function=workflow.get_sum \\\n --arg_x=/tmp/lz9a89rn/stgb7917545-93cd-45b4-833a-01559a22e620/prod.pickle \\\n --arg_y=/tmp/lz9a89rn/stg0c04cf0f-bc08-494d-ae4b-b12bde14f87a/div.pickle\n\u001b[1;30mINFO\u001b[0m [job get_sum] completed success\n\u001b[1;30mINFO\u001b[0m [step get_sum] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] completed success\n{\n \"result_file\": {\n \"location\": \"file:///home/jovyan/result.pickle\",\n \"basename\": \"result.pickle\",\n \"class\": \"File\",\n \"checksum\": \"sha1$3dfd802cefb317cc7138af1e3a299f565c74ddec\",\n \"size\": 21,\n \"path\": \"/home/jovyan/result.pickle\"\n }\n}\u001b[1;30mINFO\u001b[0m Final process status is success\n"}],"execution_count":19},{"id":"2942dbba-ea0a-4d20-be5c-ed9992d09ff8","cell_type":"code","source":"with open(\"result.pickle\", \"rb\") as f:\n print(pickle.load(f))","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"2.5\n"}],"execution_count":20},{"id":"60e909ee-d0d0-4bd1-81c8-dd5274ae5834","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]} \ No newline at end of file From 502de0f7203d07e3999aae25970724a277c30a52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Wed, 14 May 2025 06:27:17 +0200 Subject: [PATCH 13/23] more fixes --- example_workflows/arithmetic/cwl.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example_workflows/arithmetic/cwl.ipynb b/example_workflows/arithmetic/cwl.ipynb index 6ee23f2..770f612 100644 --- a/example_workflows/arithmetic/cwl.ipynb +++ b/example_workflows/arithmetic/cwl.ipynb @@ -1 +1 @@ -{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.10"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"377fef56-484d-491c-b19e-1be6931e44eb","cell_type":"code","source":"import pickle","metadata":{"trusted":true},"outputs":[],"execution_count":4},{"id":"92e3921b-2bb8-4333-8cfe-4bd27f785d24","cell_type":"code","source":"from python_workflow_definition.cwl.export import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":17},{"id":"5303c059-8ae4-4557-858e-b4bd64eac711","cell_type":"code","source":"load_workflow_json(file_name=\"workflow.json\")","metadata":{"trusted":true},"outputs":[],"execution_count":18},{"id":"df302bd2-e9b6-4595-979c-67c46414d986","cell_type":"code","source":"! cwltool workflow.cwl workflow.yml","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"/srv/conda/envs/notebook/bin/cwltool:11: DeprecationWarning: Nesting argument groups is deprecated.\n sys.exit(run())\n\u001b[1;30mINFO\u001b[0m /srv/conda/envs/notebook/bin/cwltool 3.1.20250110105449\n\u001b[1;30mINFO\u001b[0m Resolved 'workflow.cwl' to 'file:///home/jovyan/workflow.cwl'\n\u001b[1;30mINFO\u001b[0m [workflow ] start\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_prod_and_div\n\u001b[1;30mINFO\u001b[0m [step get_prod_and_div] start\n\u001b[1;30mINFO\u001b[0m [job get_prod_and_div] /tmp/abngt5eq$ python \\\n /tmp/7pxhgcet/stg6ba69a25-7dc0-46be-9bbe-5c08e6ce1925/wrapper.py \\\n --function=workflow.get_prod_and_div \\\n --arg_x=/tmp/7pxhgcet/stg007c65c6-d1cd-44cd-b761-e181c0cc37d8/x.pickle \\\n --arg_y=/tmp/7pxhgcet/stg23ac56f7-9c1d-43d9-a90c-86036b3cbafc/y.pickle\n\u001b[1;30mINFO\u001b[0m [job get_prod_and_div] completed success\n\u001b[1;30mINFO\u001b[0m [step get_prod_and_div] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_sum\n\u001b[1;30mINFO\u001b[0m [step get_sum] start\n\u001b[1;30mINFO\u001b[0m [job get_sum] /tmp/wzuzdwec$ python \\\n /tmp/lz9a89rn/stg44b61930-48fe-423d-a68a-593e237e0477/wrapper.py \\\n --function=workflow.get_sum \\\n --arg_x=/tmp/lz9a89rn/stgb7917545-93cd-45b4-833a-01559a22e620/prod.pickle \\\n --arg_y=/tmp/lz9a89rn/stg0c04cf0f-bc08-494d-ae4b-b12bde14f87a/div.pickle\n\u001b[1;30mINFO\u001b[0m [job get_sum] completed success\n\u001b[1;30mINFO\u001b[0m [step get_sum] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] completed success\n{\n \"result_file\": {\n \"location\": \"file:///home/jovyan/result.pickle\",\n \"basename\": \"result.pickle\",\n \"class\": \"File\",\n \"checksum\": \"sha1$3dfd802cefb317cc7138af1e3a299f565c74ddec\",\n \"size\": 21,\n \"path\": \"/home/jovyan/result.pickle\"\n }\n}\u001b[1;30mINFO\u001b[0m Final process status is success\n"}],"execution_count":19},{"id":"2942dbba-ea0a-4d20-be5c-ed9992d09ff8","cell_type":"code","source":"with open(\"result.pickle\", \"rb\") as f:\n print(pickle.load(f))","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"2.5\n"}],"execution_count":20},{"id":"60e909ee-d0d0-4bd1-81c8-dd5274ae5834","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]} \ No newline at end of file +{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.8","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"377fef56-484d-491c-b19e-1be6931e44eb","cell_type":"code","source":"import pickle","metadata":{"trusted":true},"outputs":[],"execution_count":1},{"id":"92e3921b-2bb8-4333-8cfe-4bd27f785d24","cell_type":"code","source":"from python_workflow_definition.cwl.export import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":2},{"id":"5303c059-8ae4-4557-858e-b4bd64eac711","cell_type":"code","source":"load_workflow_json(file_name=\"workflow.json\")","metadata":{"trusted":true},"outputs":[],"execution_count":3},{"id":"df302bd2-e9b6-4595-979c-67c46414d986","cell_type":"code","source":"! cwltool workflow.cwl workflow.yml","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"/srv/conda/envs/notebook/bin/cwltool:11: DeprecationWarning: Nesting argument groups is deprecated.\n sys.exit(run())\n\u001b[1;30mINFO\u001b[0m /srv/conda/envs/notebook/bin/cwltool 3.1.20250110105449\n\u001b[1;30mINFO\u001b[0m Resolved 'workflow.cwl' to 'file:///home/jovyan/example_workflows/arithmetic/workflow.cwl'\n\u001b[1;30mINFO\u001b[0m [workflow ] start\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_prod_and_div_0\n\u001b[1;30mINFO\u001b[0m [step get_prod_and_div_0] start\n\u001b[1;30mINFO\u001b[0m [job get_prod_and_div_0] /tmp/hig0p5dd$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/0xeme5n4/stgba321752-0a06-41dd-95c0-f5890f1dfaae/workflow.py \\\n --function=workflow.get_prod_and_div \\\n --arg_y=/tmp/0xeme5n4/stg9f21530f-2f98-44e6-80b6-afa2aac5a083/y.pickle \\\n --arg_x=/tmp/0xeme5n4/stg2e35a78a-a776-48de-b82f-8075d3284a3c/x.pickle\n\u001b[1;30mINFO\u001b[0m [job get_prod_and_div_0] completed success\n\u001b[1;30mINFO\u001b[0m [step get_prod_and_div_0] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_sum_1\n\u001b[1;30mINFO\u001b[0m [step get_sum_1] start\n\u001b[1;30mINFO\u001b[0m [job get_sum_1] /tmp/u07nctrs$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/sa53wz5h/stg686b1324-0030-4bfd-bdab-64fc9ee4d491/workflow.py \\\n --function=workflow.get_sum \\\n --arg_y=/tmp/sa53wz5h/stg1679a702-ac89-4c84-8cb5-7a57ae9e1834/div.pickle \\\n --arg_x=/tmp/sa53wz5h/stg6ab588fe-2723-416e-a7b8-648b1a63406e/prod.pickle\n\u001b[1;30mINFO\u001b[0m [job get_sum_1] completed success\n\u001b[1;30mINFO\u001b[0m [step get_sum_1] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] completed success\n{\n \"result_file\": {\n \"location\": \"file:///home/jovyan/example_workflows/arithmetic/result.pickle\",\n \"basename\": \"result.pickle\",\n \"class\": \"File\",\n \"checksum\": \"sha1$3dfd802cefb317cc7138af1e3a299f565c74ddec\",\n \"size\": 21,\n \"path\": \"/home/jovyan/example_workflows/arithmetic/result.pickle\"\n }\n}\u001b[1;30mINFO\u001b[0m Final process status is success\n"}],"execution_count":4},{"id":"2942dbba-ea0a-4d20-be5c-ed9992d09ff8","cell_type":"code","source":"with open(\"result.pickle\", \"rb\") as f:\n print(pickle.load(f))","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"2.5\n"}],"execution_count":5},{"id":"60e909ee-d0d0-4bd1-81c8-dd5274ae5834","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]} \ No newline at end of file From 6e67fd6b4a27241f5ae30515e2daedc660c07386 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Wed, 14 May 2025 06:27:36 +0200 Subject: [PATCH 14/23] more fixes --- .../python_workflow_definition/cwl/export.py | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/export.py b/python_workflow_definition/src/python_workflow_definition/cwl/export.py index 55b471c..de0bea0 100644 --- a/python_workflow_definition/src/python_workflow_definition/cwl/export.py +++ b/python_workflow_definition/src/python_workflow_definition/cwl/export.py @@ -204,14 +204,22 @@ def _write_workflow(workflow): if v[SOURCE_LABEL] in input_id_dict: in_dict[k + "_file"] = input_id_dict[v[SOURCE_LABEL]] + "_file" else: - in_dict[k + "_file"] = ( - step_name_lst[v[SOURCE_LABEL]] - + "_" - + str(v[SOURCE_LABEL]) - + "/" - + v[SOURCE_PORT_LABEL] - + "_file" - ) + if v['sourcePort'] is None: + in_dict[k + "_file"] = ( + step_name_lst[v[SOURCE_LABEL]] + + "_" + + str(v[SOURCE_LABEL]) + + "/result_file" + ) + else: + in_dict[k + "_file"] = ( + step_name_lst[v[SOURCE_LABEL]] + + "_" + + str(v[SOURCE_LABEL]) + + "/" + + v[SOURCE_PORT_LABEL] + + "_file" + ) workflow_template["steps"].update( { step_name_lst[ind] From 29274eeb374b8768ef003fcaa0e2dd55bcec0694 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Wed, 14 May 2025 06:39:15 +0200 Subject: [PATCH 15/23] black formatting --- .../src/python_workflow_definition/cwl/export.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/export.py b/python_workflow_definition/src/python_workflow_definition/cwl/export.py index de0bea0..ebb3f8d 100644 --- a/python_workflow_definition/src/python_workflow_definition/cwl/export.py +++ b/python_workflow_definition/src/python_workflow_definition/cwl/export.py @@ -204,12 +204,12 @@ def _write_workflow(workflow): if v[SOURCE_LABEL] in input_id_dict: in_dict[k + "_file"] = input_id_dict[v[SOURCE_LABEL]] + "_file" else: - if v['sourcePort'] is None: + if v["sourcePort"] is None: in_dict[k + "_file"] = ( - step_name_lst[v[SOURCE_LABEL]] - + "_" - + str(v[SOURCE_LABEL]) - + "/result_file" + step_name_lst[v[SOURCE_LABEL]] + + "_" + + str(v[SOURCE_LABEL]) + + "/result_file" ) else: in_dict[k + "_file"] = ( From 6653fdfc22ea751bfa3c052b3b8fea6309c3b162 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Wed, 14 May 2025 07:32:57 +0200 Subject: [PATCH 16/23] iterate over keys --- .../src/python_workflow_definition/cwl/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/export.py b/python_workflow_definition/src/python_workflow_definition/cwl/export.py index ebb3f8d..84749d5 100644 --- a/python_workflow_definition/src/python_workflow_definition/cwl/export.py +++ b/python_workflow_definition/src/python_workflow_definition/cwl/export.py @@ -85,7 +85,7 @@ def _write_function_cwl(workflow): function_nodes_dict, funct_dict = _get_function(workflow) file_lst = [] - for i in range(len(function_nodes_dict)): + for i in function_nodes_dict.keys(): template = { "cwlVersion": "v1.2", "class": "CommandLineTool", From b9554c9fd4861db2d36a34b938e88d0fdd2fb012 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 14 May 2025 09:03:22 +0200 Subject: [PATCH 17/23] create a cwl file for every node --- .../python_workflow_definition/cwl/export.py | 37 +++++++++---------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/export.py b/python_workflow_definition/src/python_workflow_definition/cwl/export.py index 84749d5..dfb1c0f 100644 --- a/python_workflow_definition/src/python_workflow_definition/cwl/export.py +++ b/python_workflow_definition/src/python_workflow_definition/cwl/export.py @@ -83,7 +83,6 @@ def _get_function(workflow): def _write_function_cwl(workflow): function_nodes_dict, funct_dict = _get_function(workflow) - file_lst = [] for i in function_nodes_dict.keys(): template = { @@ -108,26 +107,24 @@ def _write_function_cwl(workflow): }, "outputs": {}, } - file_name = function_nodes_dict[i].split(".")[-1] + ".cwl" - if file_name not in file_lst: - file_lst.append(file_name) - template["inputs"]["workflowfile"]["default"]["location"] = ( - function_nodes_dict[i].split(".")[0] + ".py" - ) + file_name = function_nodes_dict[i].split(".")[-1] + "_" + str(i) + ".cwl" + template["inputs"]["workflowfile"]["default"]["location"] = ( + function_nodes_dict[i].split(".")[0] + ".py" + ) + template["inputs"].update( + _get_function_template(function_name=function_nodes_dict[i]) + ) + for j, arg in enumerate(funct_dict[i]["targetPorts"]): template["inputs"].update( - _get_function_template(function_name=function_nodes_dict[i]) + _get_function_argument(argument=arg, position=4 + j) ) - for j, arg in enumerate(funct_dict[i]["targetPorts"]): - template["inputs"].update( - _get_function_argument(argument=arg, position=4 + j) - ) - for out in funct_dict[i]["sourcePorts"]: - if out is None: - template["outputs"].update(_get_output_name(output_name="result")) - else: - template["outputs"].update(_get_output_name(output_name=out)) - with open(file_name, "w") as f: - dump(template, f, Dumper=Dumper) + for out in funct_dict[i]["sourcePorts"]: + if out is None: + template["outputs"].update(_get_output_name(output_name="result")) + else: + template["outputs"].update(_get_output_name(output_name=out)) + with open(file_name, "w") as f: + dump(template, f, Dumper=Dumper) def _write_workflow_config(workflow): @@ -194,7 +191,7 @@ def _write_workflow(workflow): } for t in total_new_lst: ind = t[0] - node_script = step_name_lst[ind] + ".cwl" + node_script = step_name_lst[ind] + "_" + str(ind) + ".cwl" output = [ o + "_file" if o is not None else "result_file" for o in funct_dict[ind]["sourcePorts"] From ad57e17882426f87807fe2df5fc838bbeacbf260 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 14 May 2025 09:45:44 +0200 Subject: [PATCH 18/23] fix import of internal functions --- .../src/python_workflow_definition/cwl/__main__.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py b/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py index a70a3ff..d1fddad 100644 --- a/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py +++ b/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py @@ -5,10 +5,13 @@ def load_function(file_name, funct): - spec = importlib.util.spec_from_file_location("workflow", file_name) - module = importlib.util.module_from_spec(spec) - sys.modules["workflow"] = module - spec.loader.exec_module(module) + if file_name.split("/")[-1] != "python_workflow_definition.py": + spec = importlib.util.spec_from_file_location("workflow", file_name) + module = importlib.util.module_from_spec(spec) + sys.modules["workflow"] = module + spec.loader.exec_module(module) + else: + module = importlib.import_module("python_workflow_definition.shared") return getattr(module, funct.split(".")[-1]) From f3a34a79ddd9f413f68b357b1c0c4642986c9d00 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 14 May 2025 10:33:33 +0200 Subject: [PATCH 19/23] fix internal functions --- .../cwl/__main__.py | 21 ++++++++++--------- .../python_workflow_definition/cwl/export.py | 9 +++++--- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py b/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py index d1fddad..84a7cd7 100644 --- a/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py +++ b/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py @@ -5,13 +5,10 @@ def load_function(file_name, funct): - if file_name.split("/")[-1] != "python_workflow_definition.py": - spec = importlib.util.spec_from_file_location("workflow", file_name) - module = importlib.util.module_from_spec(spec) - sys.modules["workflow"] = module - spec.loader.exec_module(module) - else: - module = importlib.import_module("python_workflow_definition.shared") + spec = importlib.util.spec_from_file_location("workflow", file_name) + module = importlib.util.module_from_spec(spec) + sys.modules["workflow"] = module + spec.loader.exec_module(module) return getattr(module, funct.split(".")[-1]) @@ -26,9 +23,13 @@ def convert_argument(arg): if __name__ == "__main__": # load input argument_lst = sys.argv[1:] - funct = [arg.split("=")[-1] for arg in argument_lst if "--function=" in arg][0] - file = [arg.split("=")[-1] for arg in argument_lst if "--workflowfile=" in arg][0] - workflow_function = load_function(file_name=file, funct=funct) + funct_lst = [arg.split("=")[-1] for arg in argument_lst if "--function=" in arg] + file_lst = [arg.split("=")[-1] for arg in argument_lst if "--workflowfile=" in arg] + if len(file_lst) > 0: + workflow_function = load_function(file_name=file_lst[0], funct=funct_lst[0]) + else: + m, p = funct_lst[0].rsplit(".", 1) + workflow_function = importlib.import_module(m) kwargs = { arg.split("=")[0][6:]: convert_argument(arg=arg.split("=")[-1]) for arg in argument_lst diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/export.py b/python_workflow_definition/src/python_workflow_definition/cwl/export.py index dfb1c0f..951c111 100644 --- a/python_workflow_definition/src/python_workflow_definition/cwl/export.py +++ b/python_workflow_definition/src/python_workflow_definition/cwl/export.py @@ -108,9 +108,12 @@ def _write_function_cwl(workflow): "outputs": {}, } file_name = function_nodes_dict[i].split(".")[-1] + "_" + str(i) + ".cwl" - template["inputs"]["workflowfile"]["default"]["location"] = ( - function_nodes_dict[i].split(".")[0] + ".py" - ) + if function_nodes_dict[i].split(".")[0] != "python_workflow_defintion": + template["inputs"]["workflowfile"]["default"]["location"] = ( + function_nodes_dict[i].split(".")[0] + ".py" + ) + else: + del template["inputs"]["workflowfile"] template["inputs"].update( _get_function_template(function_name=function_nodes_dict[i]) ) From 7cdd67ca27887692471d7decd78f91177f2697f7 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 14 May 2025 10:40:41 +0200 Subject: [PATCH 20/23] fix spelling --- .../src/python_workflow_definition/cwl/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/export.py b/python_workflow_definition/src/python_workflow_definition/cwl/export.py index 951c111..ee87a7c 100644 --- a/python_workflow_definition/src/python_workflow_definition/cwl/export.py +++ b/python_workflow_definition/src/python_workflow_definition/cwl/export.py @@ -108,7 +108,7 @@ def _write_function_cwl(workflow): "outputs": {}, } file_name = function_nodes_dict[i].split(".")[-1] + "_" + str(i) + ".cwl" - if function_nodes_dict[i].split(".")[0] != "python_workflow_defintion": + if function_nodes_dict[i].split(".")[0] != "python_workflow_definition": template["inputs"]["workflowfile"]["default"]["location"] = ( function_nodes_dict[i].split(".")[0] + ".py" ) From 36921a1093df6ff9ef0ed9bf1bcc2301b9cc5283 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 14 May 2025 11:19:28 +0200 Subject: [PATCH 21/23] fix function import --- .../src/python_workflow_definition/cwl/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py b/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py index 84a7cd7..d1118f3 100644 --- a/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py +++ b/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py @@ -29,7 +29,7 @@ def convert_argument(arg): workflow_function = load_function(file_name=file_lst[0], funct=funct_lst[0]) else: m, p = funct_lst[0].rsplit(".", 1) - workflow_function = importlib.import_module(m) + workflow_function = getattr(importlib.import_module(m), p) kwargs = { arg.split("=")[0][6:]: convert_argument(arg=arg.split("=")[-1]) for arg in argument_lst From f5019068673840bf98d776348e825360664211bf Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 14 May 2025 11:41:10 +0200 Subject: [PATCH 22/23] fix quantum espresso --- example_workflows/quantum_espresso/cwl.ipynb | 2 +- .../src/python_workflow_definition/cwl/__main__.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/example_workflows/quantum_espresso/cwl.ipynb b/example_workflows/quantum_espresso/cwl.ipynb index 6ee23f2..da2fcbc 100644 --- a/example_workflows/quantum_espresso/cwl.ipynb +++ b/example_workflows/quantum_espresso/cwl.ipynb @@ -1 +1 @@ -{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.10"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"377fef56-484d-491c-b19e-1be6931e44eb","cell_type":"code","source":"import pickle","metadata":{"trusted":true},"outputs":[],"execution_count":4},{"id":"92e3921b-2bb8-4333-8cfe-4bd27f785d24","cell_type":"code","source":"from python_workflow_definition.cwl.export import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":17},{"id":"5303c059-8ae4-4557-858e-b4bd64eac711","cell_type":"code","source":"load_workflow_json(file_name=\"workflow.json\")","metadata":{"trusted":true},"outputs":[],"execution_count":18},{"id":"df302bd2-e9b6-4595-979c-67c46414d986","cell_type":"code","source":"! cwltool workflow.cwl workflow.yml","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"/srv/conda/envs/notebook/bin/cwltool:11: DeprecationWarning: Nesting argument groups is deprecated.\n sys.exit(run())\n\u001b[1;30mINFO\u001b[0m /srv/conda/envs/notebook/bin/cwltool 3.1.20250110105449\n\u001b[1;30mINFO\u001b[0m Resolved 'workflow.cwl' to 'file:///home/jovyan/workflow.cwl'\n\u001b[1;30mINFO\u001b[0m [workflow ] start\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_prod_and_div\n\u001b[1;30mINFO\u001b[0m [step get_prod_and_div] start\n\u001b[1;30mINFO\u001b[0m [job get_prod_and_div] /tmp/abngt5eq$ python \\\n /tmp/7pxhgcet/stg6ba69a25-7dc0-46be-9bbe-5c08e6ce1925/wrapper.py \\\n --function=workflow.get_prod_and_div \\\n --arg_x=/tmp/7pxhgcet/stg007c65c6-d1cd-44cd-b761-e181c0cc37d8/x.pickle \\\n --arg_y=/tmp/7pxhgcet/stg23ac56f7-9c1d-43d9-a90c-86036b3cbafc/y.pickle\n\u001b[1;30mINFO\u001b[0m [job get_prod_and_div] completed success\n\u001b[1;30mINFO\u001b[0m [step get_prod_and_div] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_sum\n\u001b[1;30mINFO\u001b[0m [step get_sum] start\n\u001b[1;30mINFO\u001b[0m [job get_sum] /tmp/wzuzdwec$ python \\\n /tmp/lz9a89rn/stg44b61930-48fe-423d-a68a-593e237e0477/wrapper.py \\\n --function=workflow.get_sum \\\n --arg_x=/tmp/lz9a89rn/stgb7917545-93cd-45b4-833a-01559a22e620/prod.pickle \\\n --arg_y=/tmp/lz9a89rn/stg0c04cf0f-bc08-494d-ae4b-b12bde14f87a/div.pickle\n\u001b[1;30mINFO\u001b[0m [job get_sum] completed success\n\u001b[1;30mINFO\u001b[0m [step get_sum] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] completed success\n{\n \"result_file\": {\n \"location\": \"file:///home/jovyan/result.pickle\",\n \"basename\": \"result.pickle\",\n \"class\": \"File\",\n \"checksum\": \"sha1$3dfd802cefb317cc7138af1e3a299f565c74ddec\",\n \"size\": 21,\n \"path\": \"/home/jovyan/result.pickle\"\n }\n}\u001b[1;30mINFO\u001b[0m Final process status is success\n"}],"execution_count":19},{"id":"2942dbba-ea0a-4d20-be5c-ed9992d09ff8","cell_type":"code","source":"with open(\"result.pickle\", \"rb\") as f:\n print(pickle.load(f))","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"2.5\n"}],"execution_count":20},{"id":"60e909ee-d0d0-4bd1-81c8-dd5274ae5834","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]} \ No newline at end of file +{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.8","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"4eca79ef-1053-4f69-89ad-2bee8411068e","cell_type":"code","source":"import os","metadata":{"trusted":true},"outputs":[],"execution_count":1},{"id":"377fef56-484d-491c-b19e-1be6931e44eb","cell_type":"code","source":"import pickle","metadata":{"trusted":true},"outputs":[],"execution_count":2},{"id":"92e3921b-2bb8-4333-8cfe-4bd27f785d24","cell_type":"code","source":"from python_workflow_definition.cwl.export import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":3},{"id":"5303c059-8ae4-4557-858e-b4bd64eac711","cell_type":"code","source":"load_workflow_json(file_name=\"workflow.json\")","metadata":{"trusted":true},"outputs":[],"execution_count":4},{"id":"0192ca74-3971-464b-9435-c156e0b6e623","cell_type":"code","source":"os.environ[\"ESPRESSO_PSEUDO\"] = os.path.abspath(os.path.join(\"../../espresso/pseudo\"))","metadata":{"trusted":true},"outputs":[],"execution_count":5},{"id":"df302bd2-e9b6-4595-979c-67c46414d986","cell_type":"code","source":"! cwltool --preserve-environment=ESPRESSO_PSEUDO workflow.cwl workflow.yml","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"/srv/conda/envs/notebook/bin/cwltool:11: DeprecationWarning: Nesting argument groups is deprecated.\n sys.exit(run())\n\u001b[1;30mINFO\u001b[0m /srv/conda/envs/notebook/bin/cwltool 3.1.20250110105449\n\u001b[1;30mINFO\u001b[0m Resolved 'workflow.cwl' to 'file:///home/jovyan/example_workflows/quantum_espresso/workflow.cwl'\n\u001b[1;30mINFO\u001b[0m [workflow ] start\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_bulk_structure_0\n\u001b[1;30mINFO\u001b[0m [step get_bulk_structure_0] start\n\u001b[1;30mINFO\u001b[0m [job get_bulk_structure_0] /tmp/o5wvbmxj$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/3uzk268a/stgc8824dc9-f08d-44b9-9d9c-518dc0415f4d/workflow.py \\\n --function=workflow.get_bulk_structure \\\n --arg_a=/tmp/3uzk268a/stgbcb5cab3-e9ef-46e5-8f20-1a73f5b242e9/a.pickle \\\n --arg_element=/tmp/3uzk268a/stgb2678406-1178-4bf6-ba96-2b623c74f408/element.pickle \\\n --arg_cubic=/tmp/3uzk268a/stg7d836c8f-b499-4ea6-a341-97ff1a366b8f/cubic.pickle\n\u001b[1;30mINFO\u001b[0m [job get_bulk_structure_0] Max memory used: 100MiB\n\u001b[1;30mINFO\u001b[0m [job get_bulk_structure_0] completed success\n\u001b[1;30mINFO\u001b[0m [step get_bulk_structure_0] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_dict_13\n\u001b[1;30mINFO\u001b[0m [step get_dict_13] start\n\u001b[1;30mINFO\u001b[0m [job get_dict_13] /tmp/que74c90$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_dict \\\n --arg_kpts=/tmp/k_sm4nc9/stg029d45cb-9d3e-4d26-a4f6-95cb0fb162d5/kpts.pickle \\\n --arg_structure=/tmp/k_sm4nc9/stgbde5b9c0-6926-4c08-886c-a7425fdf5571/result.pickle \\\n --arg_pseudopotentials=/tmp/k_sm4nc9/stg6717dca8-3880-4d1b-bed2-4078dca65fdb/pseudopotentials.pickle \\\n --arg_calculation=/tmp/k_sm4nc9/stg0e219c86-340f-4f1a-bc25-e366286137ea/calculation_0.pickle \\\n --arg_smearing=/tmp/k_sm4nc9/stg299bfaba-cb42-4aa5-b372-3101efa15660/smearing.pickle\n\u001b[1;30mINFO\u001b[0m [job get_dict_13] completed success\n\u001b[1;30mINFO\u001b[0m [step get_dict_13] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step calculate_qe_1\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_1] start\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_1] /tmp/9ew4474v$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/3hzmn68a/stgf58c4a14-7b0f-4efc-bedd-3561b9fdb227/workflow.py \\\n --function=workflow.calculate_qe \\\n --arg_input_dict=/tmp/3hzmn68a/stg41392280-cfd6-4e67-ab11-915bfcbd0304/result.pickle \\\n --arg_working_directory=/tmp/3hzmn68a/stg6aa2aff2-eca4-41b7-bef5-a917188e0853/working_directory_0.pickle\n[jupyter-pythonworkflow-fl--x---ff79c5c6:01664] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_1] Max memory used: 250MiB\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_1] completed success\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_1] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step generate_structures_2\n\u001b[1;30mINFO\u001b[0m [step generate_structures_2] start\n\u001b[1;30mINFO\u001b[0m [job generate_structures_2] /tmp/l8wkgfu9$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/aibj9rr4/stgb715e49b-e705-4371-8c63-5bc9f8addb97/workflow.py \\\n --function=workflow.generate_structures \\\n --arg_structure=/tmp/aibj9rr4/stg88fe1a91-5cb6-4f9c-9fff-a63e477a0ccb/structure.pickle \\\n --arg_strain_lst=/tmp/aibj9rr4/stg3489de05-cdc1-4f7f-af63-cb8612cefe10/strain_lst.pickle\n\u001b[1;30mINFO\u001b[0m [job generate_structures_2] Max memory used: 108MiB\n\u001b[1;30mINFO\u001b[0m [job generate_structures_2] completed success\n\u001b[1;30mINFO\u001b[0m [step generate_structures_2] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_dict_20\n\u001b[1;30mINFO\u001b[0m [step get_dict_20] start\n\u001b[1;30mINFO\u001b[0m [job get_dict_20] /tmp/0ls08hbo$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_dict \\\n --arg_kpts=/tmp/yb9nqwe7/stg475a5983-0e88-40ee-b3a4-0c9b916f08f1/kpts.pickle \\\n --arg_structure=/tmp/yb9nqwe7/stg4f27a44f-d76c-4278-855d-ab654ebe6e32/s_0.pickle \\\n --arg_pseudopotentials=/tmp/yb9nqwe7/stgbfcb2c02-776a-4101-b906-06a8ef758140/pseudopotentials.pickle \\\n --arg_calculation=/tmp/yb9nqwe7/stgd91d742a-82f8-459e-939b-2cd0876f41ee/calculation_1.pickle \\\n --arg_smearing=/tmp/yb9nqwe7/stga65643a5-b1bb-416e-bc6c-ee7b52c28bd8/smearing.pickle\n\u001b[1;30mINFO\u001b[0m [job get_dict_20] completed success\n\u001b[1;30mINFO\u001b[0m [step get_dict_20] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step calculate_qe_3\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_3] start\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_3] /tmp/nzsw1xr6$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/gjwsgfmj/stg3069d789-517f-41bf-8d4b-07e51f9e924d/workflow.py \\\n --function=workflow.calculate_qe \\\n --arg_input_dict=/tmp/gjwsgfmj/stg9575b9dd-3336-4e26-a106-7c52c1e41f91/result.pickle \\\n --arg_working_directory=/tmp/gjwsgfmj/stg28ff68ea-caa6-47bf-be5e-67394ab72f64/working_directory_1.pickle\n[jupyter-pythonworkflow-fl--x---ff79c5c6:01846] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_3] Max memory used: 239MiB\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_3] completed success\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_3] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_dict_25\n\u001b[1;30mINFO\u001b[0m [step get_dict_25] start\n\u001b[1;30mINFO\u001b[0m [job get_dict_25] /tmp/zrs68r3d$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_dict \\\n --arg_kpts=/tmp/jj2_r82j/stg13ba3696-061f-46f7-b012-bb7ab8a40796/kpts.pickle \\\n --arg_structure=/tmp/jj2_r82j/stg5ca9e71a-98fe-44df-8ca5-550908525a4e/s_2.pickle \\\n --arg_pseudopotentials=/tmp/jj2_r82j/stg9e1fe3f0-1192-4cde-9772-6be8dfa314fe/pseudopotentials.pickle \\\n --arg_calculation=/tmp/jj2_r82j/stgde664722-b13c-4ae9-9562-aec2063d548a/calculation_1.pickle \\\n --arg_smearing=/tmp/jj2_r82j/stg799e1f26-f08e-40bf-a7a9-c58e907674c8/smearing.pickle\n\u001b[1;30mINFO\u001b[0m [job get_dict_25] completed success\n\u001b[1;30mINFO\u001b[0m [step get_dict_25] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_dict_23\n\u001b[1;30mINFO\u001b[0m [step get_dict_23] start\n\u001b[1;30mINFO\u001b[0m [job get_dict_23] /tmp/7qgifyu_$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_dict \\\n --arg_kpts=/tmp/ojouuynn/stg7b99ab8b-9235-4007-b6b7-dd4170bd1760/kpts.pickle \\\n --arg_structure=/tmp/ojouuynn/stg8bfd87ff-e83d-4918-9a0d-71b958c46154/s_1.pickle \\\n --arg_pseudopotentials=/tmp/ojouuynn/stg0bcc3451-1700-46e0-9a5e-fca9968c8521/pseudopotentials.pickle \\\n --arg_calculation=/tmp/ojouuynn/stgb9d8de26-7d88-4119-ad13-8374ff498f72/calculation_1.pickle \\\n --arg_smearing=/tmp/ojouuynn/stg01fe91f8-b588-4adc-b207-9d9d4e0956e3/smearing.pickle\n\u001b[1;30mINFO\u001b[0m [job get_dict_23] completed success\n\u001b[1;30mINFO\u001b[0m [step get_dict_23] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_dict_27\n\u001b[1;30mINFO\u001b[0m [step get_dict_27] start\n\u001b[1;30mINFO\u001b[0m [job get_dict_27] /tmp/v83aah99$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_dict \\\n --arg_kpts=/tmp/_wg2md43/stgee67f0e5-87a9-4768-82ed-cb6c6cf74b6d/kpts.pickle \\\n --arg_structure=/tmp/_wg2md43/stgb39aa4d9-0909-4ffb-bfe8-c448a3d77669/s_3.pickle \\\n --arg_pseudopotentials=/tmp/_wg2md43/stgc7b618ee-6e90-4b93-a079-785fab087237/pseudopotentials.pickle \\\n --arg_calculation=/tmp/_wg2md43/stg738933fd-9a31-4c0e-ac99-984d1002fd0a/calculation_1.pickle \\\n --arg_smearing=/tmp/_wg2md43/stg174b61ec-1446-4c63-a433-49a940e48f39/smearing.pickle\n\u001b[1;30mINFO\u001b[0m [job get_dict_27] completed success\n\u001b[1;30mINFO\u001b[0m [step get_dict_27] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step calculate_qe_4\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_4] start\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_4] /tmp/abio0491$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/pltanpun/stg312b10e4-f5f2-4770-b776-12c659453bef/workflow.py \\\n --function=workflow.calculate_qe \\\n --arg_input_dict=/tmp/pltanpun/stgceefbf38-ae57-4369-872d-4ea0895a6976/result.pickle \\\n --arg_working_directory=/tmp/pltanpun/stg6abc0b8b-ed0d-4692-b100-b510fe7de457/working_directory_2.pickle\n[jupyter-pythonworkflow-fl--x---ff79c5c6:01928] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_4] Max memory used: 244MiB\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_4] completed success\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_4] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_dict_29\n\u001b[1;30mINFO\u001b[0m [step get_dict_29] start\n\u001b[1;30mINFO\u001b[0m [job get_dict_29] /tmp/li__ygke$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_dict \\\n --arg_kpts=/tmp/20wxb6mt/stg4f9c6f04-e0da-4f1d-9b91-17867111fcec/kpts.pickle \\\n --arg_structure=/tmp/20wxb6mt/stg27aeb042-5dba-406d-ad3f-c1a15414d462/s_4.pickle \\\n --arg_pseudopotentials=/tmp/20wxb6mt/stg00a521b7-0d6f-44fa-849f-611e82b816d2/pseudopotentials.pickle \\\n --arg_calculation=/tmp/20wxb6mt/stg51a4e465-9a6d-465e-a329-ec65c9631dca/calculation_1.pickle \\\n --arg_smearing=/tmp/20wxb6mt/stg72f4b40d-8647-441e-94ce-b4d60eb84837/smearing.pickle\n\u001b[1;30mINFO\u001b[0m [job get_dict_29] completed success\n\u001b[1;30mINFO\u001b[0m [step get_dict_29] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step calculate_qe_7\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_7] start\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_7] /tmp/26ontyw1$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/3e7bw8e2/stg86169764-ce4e-40f6-9b63-ef5e11208258/workflow.py \\\n --function=workflow.calculate_qe \\\n --arg_input_dict=/tmp/3e7bw8e2/stge349561c-3f24-48da-94fa-2d16e4df4779/result.pickle \\\n --arg_working_directory=/tmp/3e7bw8e2/stg30622d6d-1c64-4440-9d76-7a70e081ff5e/working_directory_5.pickle\n[jupyter-pythonworkflow-fl--x---ff79c5c6:02006] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_7] Max memory used: 243MiB\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_7] completed success\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_7] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step calculate_qe_5\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_5] start\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_5] /tmp/nwlrltkk$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/q_jjx0ja/stgb8b85eae-791f-4e34-9696-ceb277f72155/workflow.py \\\n --function=workflow.calculate_qe \\\n --arg_input_dict=/tmp/q_jjx0ja/stg2efa8335-1138-47c3-b9e9-d8ea2d8754c0/result.pickle \\\n --arg_working_directory=/tmp/q_jjx0ja/stg4f726c25-7adb-4ce0-970e-bd9ddc5191cc/working_directory_3.pickle\n[jupyter-pythonworkflow-fl--x---ff79c5c6:02086] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_5] Max memory used: 246MiB\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_5] completed success\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_5] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step calculate_qe_6\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_6] start\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_6] /tmp/9yp55i6u$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/1c_enl4x/stg2aeb7a80-3364-48a0-810d-955e246aa33b/workflow.py \\\n --function=workflow.calculate_qe \\\n --arg_input_dict=/tmp/1c_enl4x/stg19bec723-020c-4d0e-bca0-8124f6962c7e/result.pickle \\\n --arg_working_directory=/tmp/1c_enl4x/stgc36f93b3-f6f6-4fec-8eb2-0de1ca22f3cd/working_directory_4.pickle\n[jupyter-pythonworkflow-fl--x---ff79c5c6:02164] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_6] Max memory used: 243MiB\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_6] completed success\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_6] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_list_31\n\u001b[1;30mINFO\u001b[0m [step get_list_31] start\n\u001b[1;30mINFO\u001b[0m [job get_list_31] /tmp/xgme48c0$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_list \\\n --arg_2=/tmp/u2adengu/stg036d73c4-50fd-4b84-9054-ef00e66e0ea8/energy.pickle \\\n --arg_4=/tmp/u2adengu/stg8736d42d-b110-43b3-8e1d-da22b9883ff5/energy.pickle \\\n --arg_1=/tmp/u2adengu/stg8cac7eb7-6132-4085-865d-673027e26baa/energy.pickle \\\n --arg_3=/tmp/u2adengu/stg55b370b4-c852-4493-bb7b-4acaa0588aa8/energy.pickle \\\n --arg_0=/tmp/u2adengu/stg5d319d67-b4fc-4c4d-886a-529b9bfd2458/energy.pickle\n\u001b[1;30mINFO\u001b[0m [job get_list_31] completed success\n\u001b[1;30mINFO\u001b[0m [step get_list_31] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_list_30\n\u001b[1;30mINFO\u001b[0m [step get_list_30] start\n\u001b[1;30mINFO\u001b[0m [job get_list_30] /tmp/1xbu0rii$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_list \\\n --arg_2=/tmp/0ads3xzd/stg24fc8770-2739-45d1-b535-89924a6234ed/volume.pickle \\\n --arg_4=/tmp/0ads3xzd/stgfb79ea5f-6109-4f2e-a1d6-0817648a5350/volume.pickle \\\n --arg_1=/tmp/0ads3xzd/stg0b1e9637-dba2-4e42-9dbe-c605e7469714/volume.pickle \\\n --arg_3=/tmp/0ads3xzd/stgdf4991ae-4266-4ab6-82be-8f87d46fa7f6/volume.pickle \\\n --arg_0=/tmp/0ads3xzd/stg623a06b5-d2a0-44fd-9473-833452ad5762/volume.pickle\n\u001b[1;30mINFO\u001b[0m [job get_list_30] completed success\n\u001b[1;30mINFO\u001b[0m [step get_list_30] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step plot_energy_volume_curve_8\n\u001b[1;30mINFO\u001b[0m [step plot_energy_volume_curve_8] start\n\u001b[1;30mINFO\u001b[0m [job plot_energy_volume_curve_8] /tmp/vcffw_nr$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/6s32h5zf/stg79d5980d-139d-4481-9277-3f58bb27d49f/workflow.py \\\n --function=workflow.plot_energy_volume_curve \\\n --arg_volume_lst=/tmp/6s32h5zf/stg67eb8ed0-f498-4181-a5a5-c7e74ecd3276/result.pickle \\\n --arg_energy_lst=/tmp/6s32h5zf/stg3ebdb224-bd9a-4ae7-ad4e-03489dcfb661/result.pickle\n\u001b[1;30mINFO\u001b[0m [job plot_energy_volume_curve_8] Max memory used: 110MiB\n\u001b[1;30mINFO\u001b[0m [job plot_energy_volume_curve_8] completed success\n\u001b[1;30mINFO\u001b[0m [step plot_energy_volume_curve_8] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] completed success\n{\n \"result_file\": {\n \"location\": \"file:///home/jovyan/example_workflows/quantum_espresso/result.pickle\",\n \"basename\": \"result.pickle\",\n \"class\": \"File\",\n \"checksum\": \"sha1$dbc1aaddc8b7343d6d33b34edcf608b8f8801918\",\n \"size\": 4,\n \"path\": \"/home/jovyan/example_workflows/quantum_espresso/result.pickle\"\n }\n}\u001b[1;30mINFO\u001b[0m Final process status is success\n"}],"execution_count":6},{"id":"2942dbba-ea0a-4d20-be5c-ed9992d09ff8","cell_type":"code","source":"with open(\"result.pickle\", \"rb\") as f:\n print(pickle.load(f))","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"None\n"}],"execution_count":7},{"id":"60e909ee-d0d0-4bd1-81c8-dd5274ae5834","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]} \ No newline at end of file diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py b/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py index d1118f3..ec1c6c4 100644 --- a/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py +++ b/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py @@ -27,9 +27,11 @@ def convert_argument(arg): file_lst = [arg.split("=")[-1] for arg in argument_lst if "--workflowfile=" in arg] if len(file_lst) > 0: workflow_function = load_function(file_name=file_lst[0], funct=funct_lst[0]) + internal_function = False else: m, p = funct_lst[0].rsplit(".", 1) workflow_function = getattr(importlib.import_module(m), p) + internal_function = True kwargs = { arg.split("=")[0][6:]: convert_argument(arg=arg.split("=")[-1]) for arg in argument_lst @@ -40,7 +42,7 @@ def convert_argument(arg): result = workflow_function(**kwargs) # store output - if isinstance(result, dict): + if isinstance(result, dict) and not internal_function: for k, v in result.items(): with open(k + ".pickle", "wb") as f: pickle.dump(v, f) From 007c90e8a81f354181c4b195693d689c2b0e6a7f Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 14 May 2025 11:50:14 +0200 Subject: [PATCH 23/23] fix nfdi --- example_workflows/nfdi/cwl.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example_workflows/nfdi/cwl.ipynb b/example_workflows/nfdi/cwl.ipynb index 6ee23f2..f3d625a 100644 --- a/example_workflows/nfdi/cwl.ipynb +++ b/example_workflows/nfdi/cwl.ipynb @@ -1 +1 @@ -{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.10"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"377fef56-484d-491c-b19e-1be6931e44eb","cell_type":"code","source":"import pickle","metadata":{"trusted":true},"outputs":[],"execution_count":4},{"id":"92e3921b-2bb8-4333-8cfe-4bd27f785d24","cell_type":"code","source":"from python_workflow_definition.cwl.export import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":17},{"id":"5303c059-8ae4-4557-858e-b4bd64eac711","cell_type":"code","source":"load_workflow_json(file_name=\"workflow.json\")","metadata":{"trusted":true},"outputs":[],"execution_count":18},{"id":"df302bd2-e9b6-4595-979c-67c46414d986","cell_type":"code","source":"! cwltool workflow.cwl workflow.yml","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"/srv/conda/envs/notebook/bin/cwltool:11: DeprecationWarning: Nesting argument groups is deprecated.\n sys.exit(run())\n\u001b[1;30mINFO\u001b[0m /srv/conda/envs/notebook/bin/cwltool 3.1.20250110105449\n\u001b[1;30mINFO\u001b[0m Resolved 'workflow.cwl' to 'file:///home/jovyan/workflow.cwl'\n\u001b[1;30mINFO\u001b[0m [workflow ] start\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_prod_and_div\n\u001b[1;30mINFO\u001b[0m [step get_prod_and_div] start\n\u001b[1;30mINFO\u001b[0m [job get_prod_and_div] /tmp/abngt5eq$ python \\\n /tmp/7pxhgcet/stg6ba69a25-7dc0-46be-9bbe-5c08e6ce1925/wrapper.py \\\n --function=workflow.get_prod_and_div \\\n --arg_x=/tmp/7pxhgcet/stg007c65c6-d1cd-44cd-b761-e181c0cc37d8/x.pickle \\\n --arg_y=/tmp/7pxhgcet/stg23ac56f7-9c1d-43d9-a90c-86036b3cbafc/y.pickle\n\u001b[1;30mINFO\u001b[0m [job get_prod_and_div] completed success\n\u001b[1;30mINFO\u001b[0m [step get_prod_and_div] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_sum\n\u001b[1;30mINFO\u001b[0m [step get_sum] start\n\u001b[1;30mINFO\u001b[0m [job get_sum] /tmp/wzuzdwec$ python \\\n /tmp/lz9a89rn/stg44b61930-48fe-423d-a68a-593e237e0477/wrapper.py \\\n --function=workflow.get_sum \\\n --arg_x=/tmp/lz9a89rn/stgb7917545-93cd-45b4-833a-01559a22e620/prod.pickle \\\n --arg_y=/tmp/lz9a89rn/stg0c04cf0f-bc08-494d-ae4b-b12bde14f87a/div.pickle\n\u001b[1;30mINFO\u001b[0m [job get_sum] completed success\n\u001b[1;30mINFO\u001b[0m [step get_sum] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] completed success\n{\n \"result_file\": {\n \"location\": \"file:///home/jovyan/result.pickle\",\n \"basename\": \"result.pickle\",\n \"class\": \"File\",\n \"checksum\": \"sha1$3dfd802cefb317cc7138af1e3a299f565c74ddec\",\n \"size\": 21,\n \"path\": \"/home/jovyan/result.pickle\"\n }\n}\u001b[1;30mINFO\u001b[0m Final process status is success\n"}],"execution_count":19},{"id":"2942dbba-ea0a-4d20-be5c-ed9992d09ff8","cell_type":"code","source":"with open(\"result.pickle\", \"rb\") as f:\n print(pickle.load(f))","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"2.5\n"}],"execution_count":20},{"id":"60e909ee-d0d0-4bd1-81c8-dd5274ae5834","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]} \ No newline at end of file +{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.8","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"377fef56-484d-491c-b19e-1be6931e44eb","cell_type":"code","source":"import pickle","metadata":{"trusted":true},"outputs":[],"execution_count":1},{"id":"2033dda1-dc7a-4f96-b1bd-90505b0ec555","cell_type":"code","source":"import json","metadata":{"trusted":true},"outputs":[],"execution_count":2},{"id":"60cb8ace-acb0-47b4-b0bc-bb54d00d19dd","cell_type":"code","source":"import os","metadata":{"trusted":true},"outputs":[],"execution_count":3},{"id":"92e3921b-2bb8-4333-8cfe-4bd27f785d24","cell_type":"code","source":"from python_workflow_definition.cwl.export import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":4},{"id":"b0cf73b9-ea21-4437-8d2a-c51b65bbfa86","cell_type":"markdown","source":"# Overwrite source directory with absolute path","metadata":{}},{"id":"bca61d32-89dd-4df7-92da-fee1a157df5a","cell_type":"code","source":"with open(\"workflow.json\") as f:\n content = json.load(f)","metadata":{"trusted":true},"outputs":[],"execution_count":5},{"id":"8392fa04-4fb3-496e-9387-0106c872fb98","cell_type":"code","source":"node_lst = []\nfor n in content[\"nodes\"]:\n if 'name' in n and n['name'] == 'source_directory':\n n[\"value\"] = os.path.abspath(n[\"value\"])\n node_lst.append(n)\n\ncontent[\"nodes\"] = node_lst","metadata":{"trusted":true},"outputs":[],"execution_count":6},{"id":"e53c7769-28bb-4d8e-b3a4-02298818a001","cell_type":"code","source":"with open(\"workflow.json\", \"w\") as f:\n json.dump(content, f)","metadata":{"trusted":true},"outputs":[],"execution_count":7},{"id":"a9540ba7-f15a-4d04-86aa-0cf2ad4ac185","cell_type":"markdown","source":"# Execute workflow","metadata":{}},{"id":"5303c059-8ae4-4557-858e-b4bd64eac711","cell_type":"code","source":"load_workflow_json(file_name=\"workflow.json\")","metadata":{"trusted":true},"outputs":[],"execution_count":8},{"id":"df302bd2-e9b6-4595-979c-67c46414d986","cell_type":"code","source":"! cwltool --preserve-environment=CONDA_EXE workflow.cwl workflow.yml","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"/srv/conda/envs/notebook/bin/cwltool:11: DeprecationWarning: Nesting argument groups is deprecated.\n sys.exit(run())\n\u001b[1;30mINFO\u001b[0m /srv/conda/envs/notebook/bin/cwltool 3.1.20250110105449\n\u001b[1;30mINFO\u001b[0m Resolved 'workflow.cwl' to 'file:///home/jovyan/example_workflows/nfdi/workflow.cwl'\n\u001b[1;30mINFO\u001b[0m [workflow ] start\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step generate_mesh_0\n\u001b[1;30mINFO\u001b[0m [step generate_mesh_0] start\n\u001b[1;30mINFO\u001b[0m [job generate_mesh_0] /tmp/2zy1qyud$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/3kq1m6dv/stgfdd0e5fa-ab2c-4ee5-a621-421e0f130c2d/workflow.py \\\n --function=workflow.generate_mesh \\\n --arg_source_directory=/tmp/3kq1m6dv/stgb3cb8d7a-2fd4-4bed-bddd-a53347db76d4/source_directory.pickle \\\n --arg_domain_size=/tmp/3kq1m6dv/stgf88b878d-cda3-46dc-b95b-1a5866fc79df/domain_size.pickle\n\u001b[1;30mINFO\u001b[0m [job generate_mesh_0] Max memory used: 51MiB\n\u001b[1;30mINFO\u001b[0m [job generate_mesh_0] completed success\n\u001b[1;30mINFO\u001b[0m [step generate_mesh_0] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step convert_to_xdmf_1\n\u001b[1;30mINFO\u001b[0m [step convert_to_xdmf_1] start\n\u001b[1;30mINFO\u001b[0m [job convert_to_xdmf_1] /tmp/wtfrae9y$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/mnez84gw/stg3b3a2c97-dc53-4fa4-8b26-1c0ef725455a/workflow.py \\\n --function=workflow.convert_to_xdmf \\\n --arg_gmsh_output_file=/tmp/mnez84gw/stg7cb20c0e-e3a4-40c7-b347-90b3861ac948/result.pickle\n\u001b[1;30mINFO\u001b[0m [job convert_to_xdmf_1] Max memory used: 62MiB\n\u001b[1;30mINFO\u001b[0m [job convert_to_xdmf_1] completed success\n\u001b[1;30mINFO\u001b[0m [step convert_to_xdmf_1] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step poisson_2\n\u001b[1;30mINFO\u001b[0m [step poisson_2] start\n\u001b[1;30mINFO\u001b[0m [job poisson_2] /tmp/c25cseik$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/mdjrrwsb/stg9fc40fe1-ad74-4530-9b90-3f25cda3228b/workflow.py \\\n --function=workflow.poisson \\\n --arg_source_directory=/tmp/mdjrrwsb/stg95c5865b-b3b7-41a3-9f91-8c0d4e64e2cc/source_directory.pickle \\\n --arg_meshio_output_xdmf=/tmp/mdjrrwsb/stg3f852952-6283-4843-8a5a-799885c3a16f/xdmf_file.pickle \\\n --arg_meshio_output_h5=/tmp/mdjrrwsb/stg4c12ec47-0d6d-4749-afb0-b44bcfdae15c/h5_file.pickle\n\u001b[1;30mINFO\u001b[0m [job poisson_2] Max memory used: 71MiB\n\u001b[1;30mINFO\u001b[0m [job poisson_2] completed success\n\u001b[1;30mINFO\u001b[0m [step poisson_2] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step plot_over_line_3\n\u001b[1;30mINFO\u001b[0m [step plot_over_line_3] start\n\u001b[1;30mINFO\u001b[0m [job plot_over_line_3] /tmp/6taq1l9y$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/fr8grips/stgcc344580-aae9-480d-915b-b011888e11a9/workflow.py \\\n --function=workflow.plot_over_line \\\n --arg_poisson_output_pvd_file=/tmp/fr8grips/stg7c2c93df-069a-4d33-8ebc-402be33d67d0/pvd_file.pickle \\\n --arg_poisson_output_vtu_file=/tmp/fr8grips/stg481f6c15-8e7c-4198-9f1b-d61aa097b829/vtu_file.pickle \\\n --arg_source_directory=/tmp/fr8grips/stge25af463-7689-43d8-b783-165f68b907bd/source_directory.pickle\n\u001b[1;30mINFO\u001b[0m [job plot_over_line_3] Max memory used: 71MiB\n\u001b[1;30mINFO\u001b[0m [job plot_over_line_3] completed success\n\u001b[1;30mINFO\u001b[0m [step plot_over_line_3] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step substitute_macros_4\n\u001b[1;30mINFO\u001b[0m [step substitute_macros_4] start\n\u001b[1;30mINFO\u001b[0m [job substitute_macros_4] /tmp/ry44k1po$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/_yac4p7u/stge908c7b1-0d4d-44a4-a2d4-62c97b75d25f/workflow.py \\\n --function=workflow.substitute_macros \\\n --arg_ndofs=/tmp/_yac4p7u/stgd19106a7-26ec-4159-87e5-c55df4e05b3e/numdofs.pickle \\\n --arg_source_directory=/tmp/_yac4p7u/stgea87bc3d-5042-4044-b2bb-58a83646f704/source_directory.pickle \\\n --arg_pvbatch_output_file=/tmp/_yac4p7u/stga426ce18-a7c0-4941-9b4c-988615fb6dd6/result.pickle \\\n --arg_domain_size=/tmp/_yac4p7u/stg75e4cce3-e411-4ffe-b1df-e5d40daa8d61/domain_size.pickle\n\u001b[1;30mINFO\u001b[0m [job substitute_macros_4] Max memory used: 92MiB\n\u001b[1;30mINFO\u001b[0m [job substitute_macros_4] completed success\n\u001b[1;30mINFO\u001b[0m [step substitute_macros_4] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step compile_paper_5\n\u001b[1;30mINFO\u001b[0m [step compile_paper_5] start\n\u001b[1;30mINFO\u001b[0m [job compile_paper_5] /tmp/u4tewx74$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/c_jbp86a/stge99c916c-dabe-474c-b5f1-947214d3f9ed/workflow.py \\\n --function=workflow.compile_paper \\\n --arg_source_directory=/tmp/c_jbp86a/stg09ed111f-9743-4670-a6e6-182b1b32360d/source_directory.pickle \\\n --arg_plot_file=/tmp/c_jbp86a/stg5649e17e-4a2b-416c-8107-31df66b6e769/result.pickle \\\n --arg_macros_tex=/tmp/c_jbp86a/stg26421152-a968-4c63-97a0-8ed741b07f1a/result.pickle\n\u001b[1;30mINFO\u001b[0m [job compile_paper_5] Max memory used: 260MiB\n\u001b[1;30mINFO\u001b[0m [job compile_paper_5] completed success\n\u001b[1;30mINFO\u001b[0m [step compile_paper_5] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] completed success\n{\n \"result_file\": {\n \"location\": \"file:///home/jovyan/example_workflows/nfdi/result.pickle\",\n \"basename\": \"result.pickle\",\n \"class\": \"File\",\n \"checksum\": \"sha1$1be622d292821508f2a41ec4bc8a04d06dfbdbaf\",\n \"size\": 53,\n \"path\": \"/home/jovyan/example_workflows/nfdi/result.pickle\"\n }\n}\u001b[1;30mINFO\u001b[0m Final process status is success\n"}],"execution_count":9},{"id":"2942dbba-ea0a-4d20-be5c-ed9992d09ff8","cell_type":"code","source":"with open(\"result.pickle\", \"rb\") as f:\n print(pickle.load(f))","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"/tmp/u4tewx74/postprocessing/paper.pdf\n"}],"execution_count":10},{"id":"60e909ee-d0d0-4bd1-81c8-dd5274ae5834","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]} \ No newline at end of file