diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 9750b07..24afdc7 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -68,6 +68,7 @@ jobs: run: | cd example_workflows/nfdi/ papermill aiida.ipynb aiida_out.ipynb -k "python3" + papermill cwl.ipynb cwl_out.ipynb -k "python3" papermill jobflow.ipynb jobflow_out.ipynb -k "python3" papermill pyiron_base.ipynb pyiron_base_out.ipynb -k "python3" papermill universal_workflow.ipynb universal_workflow_out.ipynb -k "python3" @@ -101,6 +102,7 @@ jobs: cp -r example_workflows/quantum_espresso/espresso . cd example_workflows/quantum_espresso papermill aiida.ipynb aiida_out.ipynb -k "python3" + papermill cwl.ipynb cwl_out.ipynb -k "python3" papermill jobflow.ipynb jobflow_out.ipynb -k "python3" papermill pyiron_base.ipynb pyiron_base_out.ipynb -k "python3" papermill universal_workflow.ipynb universal_workflow_out.ipynb -k "python3" @@ -128,6 +130,7 @@ jobs: run: | cd example_workflows/arithmetic papermill aiida.ipynb aiida_out.ipynb -k "python3" + papermill cwl.ipynb cwl_out.ipynb -k "python3" papermill jobflow.ipynb jobflow_out.ipynb -k "python3" papermill pyiron_base.ipynb pyiron_base_out.ipynb -k "python3" papermill universal_workflow.ipynb universal_workflow_out.ipynb -k "python3" diff --git a/binder/environment.yml b/binder/environment.yml index 67e2a37..00be8de 100644 --- a/binder/environment.yml +++ b/binder/environment.yml @@ -12,3 +12,4 @@ dependencies: - aiida-workgraph =0.5.2 - conda_subprocess =0.0.6 - networkx =3.4.2 +- cwltool =3.1.20250110105449 diff --git a/example_workflows/arithmetic/cwl.ipynb b/example_workflows/arithmetic/cwl.ipynb new file mode 100644 index 0000000..770f612 --- /dev/null +++ b/example_workflows/arithmetic/cwl.ipynb @@ -0,0 +1 @@ +{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.8","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"377fef56-484d-491c-b19e-1be6931e44eb","cell_type":"code","source":"import pickle","metadata":{"trusted":true},"outputs":[],"execution_count":1},{"id":"92e3921b-2bb8-4333-8cfe-4bd27f785d24","cell_type":"code","source":"from python_workflow_definition.cwl.export import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":2},{"id":"5303c059-8ae4-4557-858e-b4bd64eac711","cell_type":"code","source":"load_workflow_json(file_name=\"workflow.json\")","metadata":{"trusted":true},"outputs":[],"execution_count":3},{"id":"df302bd2-e9b6-4595-979c-67c46414d986","cell_type":"code","source":"! cwltool workflow.cwl workflow.yml","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"/srv/conda/envs/notebook/bin/cwltool:11: DeprecationWarning: Nesting argument groups is deprecated.\n sys.exit(run())\n\u001b[1;30mINFO\u001b[0m /srv/conda/envs/notebook/bin/cwltool 3.1.20250110105449\n\u001b[1;30mINFO\u001b[0m Resolved 'workflow.cwl' to 'file:///home/jovyan/example_workflows/arithmetic/workflow.cwl'\n\u001b[1;30mINFO\u001b[0m [workflow ] start\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_prod_and_div_0\n\u001b[1;30mINFO\u001b[0m [step get_prod_and_div_0] start\n\u001b[1;30mINFO\u001b[0m [job get_prod_and_div_0] /tmp/hig0p5dd$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/0xeme5n4/stgba321752-0a06-41dd-95c0-f5890f1dfaae/workflow.py \\\n --function=workflow.get_prod_and_div \\\n --arg_y=/tmp/0xeme5n4/stg9f21530f-2f98-44e6-80b6-afa2aac5a083/y.pickle \\\n --arg_x=/tmp/0xeme5n4/stg2e35a78a-a776-48de-b82f-8075d3284a3c/x.pickle\n\u001b[1;30mINFO\u001b[0m [job get_prod_and_div_0] completed success\n\u001b[1;30mINFO\u001b[0m [step get_prod_and_div_0] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_sum_1\n\u001b[1;30mINFO\u001b[0m [step get_sum_1] start\n\u001b[1;30mINFO\u001b[0m [job get_sum_1] /tmp/u07nctrs$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/sa53wz5h/stg686b1324-0030-4bfd-bdab-64fc9ee4d491/workflow.py \\\n --function=workflow.get_sum \\\n --arg_y=/tmp/sa53wz5h/stg1679a702-ac89-4c84-8cb5-7a57ae9e1834/div.pickle \\\n --arg_x=/tmp/sa53wz5h/stg6ab588fe-2723-416e-a7b8-648b1a63406e/prod.pickle\n\u001b[1;30mINFO\u001b[0m [job get_sum_1] completed success\n\u001b[1;30mINFO\u001b[0m [step get_sum_1] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] completed success\n{\n \"result_file\": {\n \"location\": \"file:///home/jovyan/example_workflows/arithmetic/result.pickle\",\n \"basename\": \"result.pickle\",\n \"class\": \"File\",\n \"checksum\": \"sha1$3dfd802cefb317cc7138af1e3a299f565c74ddec\",\n \"size\": 21,\n \"path\": \"/home/jovyan/example_workflows/arithmetic/result.pickle\"\n }\n}\u001b[1;30mINFO\u001b[0m Final process status is success\n"}],"execution_count":4},{"id":"2942dbba-ea0a-4d20-be5c-ed9992d09ff8","cell_type":"code","source":"with open(\"result.pickle\", \"rb\") as f:\n print(pickle.load(f))","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"2.5\n"}],"execution_count":5},{"id":"60e909ee-d0d0-4bd1-81c8-dd5274ae5834","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]} \ No newline at end of file diff --git a/example_workflows/nfdi/cwl.ipynb b/example_workflows/nfdi/cwl.ipynb new file mode 100644 index 0000000..f3d625a --- /dev/null +++ b/example_workflows/nfdi/cwl.ipynb @@ -0,0 +1 @@ +{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.8","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"377fef56-484d-491c-b19e-1be6931e44eb","cell_type":"code","source":"import pickle","metadata":{"trusted":true},"outputs":[],"execution_count":1},{"id":"2033dda1-dc7a-4f96-b1bd-90505b0ec555","cell_type":"code","source":"import json","metadata":{"trusted":true},"outputs":[],"execution_count":2},{"id":"60cb8ace-acb0-47b4-b0bc-bb54d00d19dd","cell_type":"code","source":"import os","metadata":{"trusted":true},"outputs":[],"execution_count":3},{"id":"92e3921b-2bb8-4333-8cfe-4bd27f785d24","cell_type":"code","source":"from python_workflow_definition.cwl.export import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":4},{"id":"b0cf73b9-ea21-4437-8d2a-c51b65bbfa86","cell_type":"markdown","source":"# Overwrite source directory with absolute path","metadata":{}},{"id":"bca61d32-89dd-4df7-92da-fee1a157df5a","cell_type":"code","source":"with open(\"workflow.json\") as f:\n content = json.load(f)","metadata":{"trusted":true},"outputs":[],"execution_count":5},{"id":"8392fa04-4fb3-496e-9387-0106c872fb98","cell_type":"code","source":"node_lst = []\nfor n in content[\"nodes\"]:\n if 'name' in n and n['name'] == 'source_directory':\n n[\"value\"] = os.path.abspath(n[\"value\"])\n node_lst.append(n)\n\ncontent[\"nodes\"] = node_lst","metadata":{"trusted":true},"outputs":[],"execution_count":6},{"id":"e53c7769-28bb-4d8e-b3a4-02298818a001","cell_type":"code","source":"with open(\"workflow.json\", \"w\") as f:\n json.dump(content, f)","metadata":{"trusted":true},"outputs":[],"execution_count":7},{"id":"a9540ba7-f15a-4d04-86aa-0cf2ad4ac185","cell_type":"markdown","source":"# Execute workflow","metadata":{}},{"id":"5303c059-8ae4-4557-858e-b4bd64eac711","cell_type":"code","source":"load_workflow_json(file_name=\"workflow.json\")","metadata":{"trusted":true},"outputs":[],"execution_count":8},{"id":"df302bd2-e9b6-4595-979c-67c46414d986","cell_type":"code","source":"! cwltool --preserve-environment=CONDA_EXE workflow.cwl workflow.yml","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"/srv/conda/envs/notebook/bin/cwltool:11: DeprecationWarning: Nesting argument groups is deprecated.\n sys.exit(run())\n\u001b[1;30mINFO\u001b[0m /srv/conda/envs/notebook/bin/cwltool 3.1.20250110105449\n\u001b[1;30mINFO\u001b[0m Resolved 'workflow.cwl' to 'file:///home/jovyan/example_workflows/nfdi/workflow.cwl'\n\u001b[1;30mINFO\u001b[0m [workflow ] start\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step generate_mesh_0\n\u001b[1;30mINFO\u001b[0m [step generate_mesh_0] start\n\u001b[1;30mINFO\u001b[0m [job generate_mesh_0] /tmp/2zy1qyud$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/3kq1m6dv/stgfdd0e5fa-ab2c-4ee5-a621-421e0f130c2d/workflow.py \\\n --function=workflow.generate_mesh \\\n --arg_source_directory=/tmp/3kq1m6dv/stgb3cb8d7a-2fd4-4bed-bddd-a53347db76d4/source_directory.pickle \\\n --arg_domain_size=/tmp/3kq1m6dv/stgf88b878d-cda3-46dc-b95b-1a5866fc79df/domain_size.pickle\n\u001b[1;30mINFO\u001b[0m [job generate_mesh_0] Max memory used: 51MiB\n\u001b[1;30mINFO\u001b[0m [job generate_mesh_0] completed success\n\u001b[1;30mINFO\u001b[0m [step generate_mesh_0] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step convert_to_xdmf_1\n\u001b[1;30mINFO\u001b[0m [step convert_to_xdmf_1] start\n\u001b[1;30mINFO\u001b[0m [job convert_to_xdmf_1] /tmp/wtfrae9y$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/mnez84gw/stg3b3a2c97-dc53-4fa4-8b26-1c0ef725455a/workflow.py \\\n --function=workflow.convert_to_xdmf \\\n --arg_gmsh_output_file=/tmp/mnez84gw/stg7cb20c0e-e3a4-40c7-b347-90b3861ac948/result.pickle\n\u001b[1;30mINFO\u001b[0m [job convert_to_xdmf_1] Max memory used: 62MiB\n\u001b[1;30mINFO\u001b[0m [job convert_to_xdmf_1] completed success\n\u001b[1;30mINFO\u001b[0m [step convert_to_xdmf_1] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step poisson_2\n\u001b[1;30mINFO\u001b[0m [step poisson_2] start\n\u001b[1;30mINFO\u001b[0m [job poisson_2] /tmp/c25cseik$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/mdjrrwsb/stg9fc40fe1-ad74-4530-9b90-3f25cda3228b/workflow.py \\\n --function=workflow.poisson \\\n --arg_source_directory=/tmp/mdjrrwsb/stg95c5865b-b3b7-41a3-9f91-8c0d4e64e2cc/source_directory.pickle \\\n --arg_meshio_output_xdmf=/tmp/mdjrrwsb/stg3f852952-6283-4843-8a5a-799885c3a16f/xdmf_file.pickle \\\n --arg_meshio_output_h5=/tmp/mdjrrwsb/stg4c12ec47-0d6d-4749-afb0-b44bcfdae15c/h5_file.pickle\n\u001b[1;30mINFO\u001b[0m [job poisson_2] Max memory used: 71MiB\n\u001b[1;30mINFO\u001b[0m [job poisson_2] completed success\n\u001b[1;30mINFO\u001b[0m [step poisson_2] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step plot_over_line_3\n\u001b[1;30mINFO\u001b[0m [step plot_over_line_3] start\n\u001b[1;30mINFO\u001b[0m [job plot_over_line_3] /tmp/6taq1l9y$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/fr8grips/stgcc344580-aae9-480d-915b-b011888e11a9/workflow.py \\\n --function=workflow.plot_over_line \\\n --arg_poisson_output_pvd_file=/tmp/fr8grips/stg7c2c93df-069a-4d33-8ebc-402be33d67d0/pvd_file.pickle \\\n --arg_poisson_output_vtu_file=/tmp/fr8grips/stg481f6c15-8e7c-4198-9f1b-d61aa097b829/vtu_file.pickle \\\n --arg_source_directory=/tmp/fr8grips/stge25af463-7689-43d8-b783-165f68b907bd/source_directory.pickle\n\u001b[1;30mINFO\u001b[0m [job plot_over_line_3] Max memory used: 71MiB\n\u001b[1;30mINFO\u001b[0m [job plot_over_line_3] completed success\n\u001b[1;30mINFO\u001b[0m [step plot_over_line_3] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step substitute_macros_4\n\u001b[1;30mINFO\u001b[0m [step substitute_macros_4] start\n\u001b[1;30mINFO\u001b[0m [job substitute_macros_4] /tmp/ry44k1po$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/_yac4p7u/stge908c7b1-0d4d-44a4-a2d4-62c97b75d25f/workflow.py \\\n --function=workflow.substitute_macros \\\n --arg_ndofs=/tmp/_yac4p7u/stgd19106a7-26ec-4159-87e5-c55df4e05b3e/numdofs.pickle \\\n --arg_source_directory=/tmp/_yac4p7u/stgea87bc3d-5042-4044-b2bb-58a83646f704/source_directory.pickle \\\n --arg_pvbatch_output_file=/tmp/_yac4p7u/stga426ce18-a7c0-4941-9b4c-988615fb6dd6/result.pickle \\\n --arg_domain_size=/tmp/_yac4p7u/stg75e4cce3-e411-4ffe-b1df-e5d40daa8d61/domain_size.pickle\n\u001b[1;30mINFO\u001b[0m [job substitute_macros_4] Max memory used: 92MiB\n\u001b[1;30mINFO\u001b[0m [job substitute_macros_4] completed success\n\u001b[1;30mINFO\u001b[0m [step substitute_macros_4] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step compile_paper_5\n\u001b[1;30mINFO\u001b[0m [step compile_paper_5] start\n\u001b[1;30mINFO\u001b[0m [job compile_paper_5] /tmp/u4tewx74$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/c_jbp86a/stge99c916c-dabe-474c-b5f1-947214d3f9ed/workflow.py \\\n --function=workflow.compile_paper \\\n --arg_source_directory=/tmp/c_jbp86a/stg09ed111f-9743-4670-a6e6-182b1b32360d/source_directory.pickle \\\n --arg_plot_file=/tmp/c_jbp86a/stg5649e17e-4a2b-416c-8107-31df66b6e769/result.pickle \\\n --arg_macros_tex=/tmp/c_jbp86a/stg26421152-a968-4c63-97a0-8ed741b07f1a/result.pickle\n\u001b[1;30mINFO\u001b[0m [job compile_paper_5] Max memory used: 260MiB\n\u001b[1;30mINFO\u001b[0m [job compile_paper_5] completed success\n\u001b[1;30mINFO\u001b[0m [step compile_paper_5] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] completed success\n{\n \"result_file\": {\n \"location\": \"file:///home/jovyan/example_workflows/nfdi/result.pickle\",\n \"basename\": \"result.pickle\",\n \"class\": \"File\",\n \"checksum\": \"sha1$1be622d292821508f2a41ec4bc8a04d06dfbdbaf\",\n \"size\": 53,\n \"path\": \"/home/jovyan/example_workflows/nfdi/result.pickle\"\n }\n}\u001b[1;30mINFO\u001b[0m Final process status is success\n"}],"execution_count":9},{"id":"2942dbba-ea0a-4d20-be5c-ed9992d09ff8","cell_type":"code","source":"with open(\"result.pickle\", \"rb\") as f:\n print(pickle.load(f))","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"/tmp/u4tewx74/postprocessing/paper.pdf\n"}],"execution_count":10},{"id":"60e909ee-d0d0-4bd1-81c8-dd5274ae5834","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]} \ No newline at end of file diff --git a/example_workflows/quantum_espresso/cwl.ipynb b/example_workflows/quantum_espresso/cwl.ipynb new file mode 100644 index 0000000..da2fcbc --- /dev/null +++ b/example_workflows/quantum_espresso/cwl.ipynb @@ -0,0 +1 @@ +{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.8","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"4eca79ef-1053-4f69-89ad-2bee8411068e","cell_type":"code","source":"import os","metadata":{"trusted":true},"outputs":[],"execution_count":1},{"id":"377fef56-484d-491c-b19e-1be6931e44eb","cell_type":"code","source":"import pickle","metadata":{"trusted":true},"outputs":[],"execution_count":2},{"id":"92e3921b-2bb8-4333-8cfe-4bd27f785d24","cell_type":"code","source":"from python_workflow_definition.cwl.export import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":3},{"id":"5303c059-8ae4-4557-858e-b4bd64eac711","cell_type":"code","source":"load_workflow_json(file_name=\"workflow.json\")","metadata":{"trusted":true},"outputs":[],"execution_count":4},{"id":"0192ca74-3971-464b-9435-c156e0b6e623","cell_type":"code","source":"os.environ[\"ESPRESSO_PSEUDO\"] = os.path.abspath(os.path.join(\"../../espresso/pseudo\"))","metadata":{"trusted":true},"outputs":[],"execution_count":5},{"id":"df302bd2-e9b6-4595-979c-67c46414d986","cell_type":"code","source":"! cwltool --preserve-environment=ESPRESSO_PSEUDO workflow.cwl workflow.yml","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"/srv/conda/envs/notebook/bin/cwltool:11: DeprecationWarning: Nesting argument groups is deprecated.\n sys.exit(run())\n\u001b[1;30mINFO\u001b[0m /srv/conda/envs/notebook/bin/cwltool 3.1.20250110105449\n\u001b[1;30mINFO\u001b[0m Resolved 'workflow.cwl' to 'file:///home/jovyan/example_workflows/quantum_espresso/workflow.cwl'\n\u001b[1;30mINFO\u001b[0m [workflow ] start\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_bulk_structure_0\n\u001b[1;30mINFO\u001b[0m [step get_bulk_structure_0] start\n\u001b[1;30mINFO\u001b[0m [job get_bulk_structure_0] /tmp/o5wvbmxj$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/3uzk268a/stgc8824dc9-f08d-44b9-9d9c-518dc0415f4d/workflow.py \\\n --function=workflow.get_bulk_structure \\\n --arg_a=/tmp/3uzk268a/stgbcb5cab3-e9ef-46e5-8f20-1a73f5b242e9/a.pickle \\\n --arg_element=/tmp/3uzk268a/stgb2678406-1178-4bf6-ba96-2b623c74f408/element.pickle \\\n --arg_cubic=/tmp/3uzk268a/stg7d836c8f-b499-4ea6-a341-97ff1a366b8f/cubic.pickle\n\u001b[1;30mINFO\u001b[0m [job get_bulk_structure_0] Max memory used: 100MiB\n\u001b[1;30mINFO\u001b[0m [job get_bulk_structure_0] completed success\n\u001b[1;30mINFO\u001b[0m [step get_bulk_structure_0] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_dict_13\n\u001b[1;30mINFO\u001b[0m [step get_dict_13] start\n\u001b[1;30mINFO\u001b[0m [job get_dict_13] /tmp/que74c90$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_dict \\\n --arg_kpts=/tmp/k_sm4nc9/stg029d45cb-9d3e-4d26-a4f6-95cb0fb162d5/kpts.pickle \\\n --arg_structure=/tmp/k_sm4nc9/stgbde5b9c0-6926-4c08-886c-a7425fdf5571/result.pickle \\\n --arg_pseudopotentials=/tmp/k_sm4nc9/stg6717dca8-3880-4d1b-bed2-4078dca65fdb/pseudopotentials.pickle \\\n --arg_calculation=/tmp/k_sm4nc9/stg0e219c86-340f-4f1a-bc25-e366286137ea/calculation_0.pickle \\\n --arg_smearing=/tmp/k_sm4nc9/stg299bfaba-cb42-4aa5-b372-3101efa15660/smearing.pickle\n\u001b[1;30mINFO\u001b[0m [job get_dict_13] completed success\n\u001b[1;30mINFO\u001b[0m [step get_dict_13] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step calculate_qe_1\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_1] start\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_1] /tmp/9ew4474v$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/3hzmn68a/stgf58c4a14-7b0f-4efc-bedd-3561b9fdb227/workflow.py \\\n --function=workflow.calculate_qe \\\n --arg_input_dict=/tmp/3hzmn68a/stg41392280-cfd6-4e67-ab11-915bfcbd0304/result.pickle \\\n --arg_working_directory=/tmp/3hzmn68a/stg6aa2aff2-eca4-41b7-bef5-a917188e0853/working_directory_0.pickle\n[jupyter-pythonworkflow-fl--x---ff79c5c6:01664] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_1] Max memory used: 250MiB\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_1] completed success\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_1] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step generate_structures_2\n\u001b[1;30mINFO\u001b[0m [step generate_structures_2] start\n\u001b[1;30mINFO\u001b[0m [job generate_structures_2] /tmp/l8wkgfu9$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/aibj9rr4/stgb715e49b-e705-4371-8c63-5bc9f8addb97/workflow.py \\\n --function=workflow.generate_structures \\\n --arg_structure=/tmp/aibj9rr4/stg88fe1a91-5cb6-4f9c-9fff-a63e477a0ccb/structure.pickle \\\n --arg_strain_lst=/tmp/aibj9rr4/stg3489de05-cdc1-4f7f-af63-cb8612cefe10/strain_lst.pickle\n\u001b[1;30mINFO\u001b[0m [job generate_structures_2] Max memory used: 108MiB\n\u001b[1;30mINFO\u001b[0m [job generate_structures_2] completed success\n\u001b[1;30mINFO\u001b[0m [step generate_structures_2] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_dict_20\n\u001b[1;30mINFO\u001b[0m [step get_dict_20] start\n\u001b[1;30mINFO\u001b[0m [job get_dict_20] /tmp/0ls08hbo$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_dict \\\n --arg_kpts=/tmp/yb9nqwe7/stg475a5983-0e88-40ee-b3a4-0c9b916f08f1/kpts.pickle \\\n --arg_structure=/tmp/yb9nqwe7/stg4f27a44f-d76c-4278-855d-ab654ebe6e32/s_0.pickle \\\n --arg_pseudopotentials=/tmp/yb9nqwe7/stgbfcb2c02-776a-4101-b906-06a8ef758140/pseudopotentials.pickle \\\n --arg_calculation=/tmp/yb9nqwe7/stgd91d742a-82f8-459e-939b-2cd0876f41ee/calculation_1.pickle \\\n --arg_smearing=/tmp/yb9nqwe7/stga65643a5-b1bb-416e-bc6c-ee7b52c28bd8/smearing.pickle\n\u001b[1;30mINFO\u001b[0m [job get_dict_20] completed success\n\u001b[1;30mINFO\u001b[0m [step get_dict_20] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step calculate_qe_3\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_3] start\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_3] /tmp/nzsw1xr6$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/gjwsgfmj/stg3069d789-517f-41bf-8d4b-07e51f9e924d/workflow.py \\\n --function=workflow.calculate_qe \\\n --arg_input_dict=/tmp/gjwsgfmj/stg9575b9dd-3336-4e26-a106-7c52c1e41f91/result.pickle \\\n --arg_working_directory=/tmp/gjwsgfmj/stg28ff68ea-caa6-47bf-be5e-67394ab72f64/working_directory_1.pickle\n[jupyter-pythonworkflow-fl--x---ff79c5c6:01846] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_3] Max memory used: 239MiB\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_3] completed success\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_3] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_dict_25\n\u001b[1;30mINFO\u001b[0m [step get_dict_25] start\n\u001b[1;30mINFO\u001b[0m [job get_dict_25] /tmp/zrs68r3d$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_dict \\\n --arg_kpts=/tmp/jj2_r82j/stg13ba3696-061f-46f7-b012-bb7ab8a40796/kpts.pickle \\\n --arg_structure=/tmp/jj2_r82j/stg5ca9e71a-98fe-44df-8ca5-550908525a4e/s_2.pickle \\\n --arg_pseudopotentials=/tmp/jj2_r82j/stg9e1fe3f0-1192-4cde-9772-6be8dfa314fe/pseudopotentials.pickle \\\n --arg_calculation=/tmp/jj2_r82j/stgde664722-b13c-4ae9-9562-aec2063d548a/calculation_1.pickle \\\n --arg_smearing=/tmp/jj2_r82j/stg799e1f26-f08e-40bf-a7a9-c58e907674c8/smearing.pickle\n\u001b[1;30mINFO\u001b[0m [job get_dict_25] completed success\n\u001b[1;30mINFO\u001b[0m [step get_dict_25] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_dict_23\n\u001b[1;30mINFO\u001b[0m [step get_dict_23] start\n\u001b[1;30mINFO\u001b[0m [job get_dict_23] /tmp/7qgifyu_$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_dict \\\n --arg_kpts=/tmp/ojouuynn/stg7b99ab8b-9235-4007-b6b7-dd4170bd1760/kpts.pickle \\\n --arg_structure=/tmp/ojouuynn/stg8bfd87ff-e83d-4918-9a0d-71b958c46154/s_1.pickle \\\n --arg_pseudopotentials=/tmp/ojouuynn/stg0bcc3451-1700-46e0-9a5e-fca9968c8521/pseudopotentials.pickle \\\n --arg_calculation=/tmp/ojouuynn/stgb9d8de26-7d88-4119-ad13-8374ff498f72/calculation_1.pickle \\\n --arg_smearing=/tmp/ojouuynn/stg01fe91f8-b588-4adc-b207-9d9d4e0956e3/smearing.pickle\n\u001b[1;30mINFO\u001b[0m [job get_dict_23] completed success\n\u001b[1;30mINFO\u001b[0m [step get_dict_23] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_dict_27\n\u001b[1;30mINFO\u001b[0m [step get_dict_27] start\n\u001b[1;30mINFO\u001b[0m [job get_dict_27] /tmp/v83aah99$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_dict \\\n --arg_kpts=/tmp/_wg2md43/stgee67f0e5-87a9-4768-82ed-cb6c6cf74b6d/kpts.pickle \\\n --arg_structure=/tmp/_wg2md43/stgb39aa4d9-0909-4ffb-bfe8-c448a3d77669/s_3.pickle \\\n --arg_pseudopotentials=/tmp/_wg2md43/stgc7b618ee-6e90-4b93-a079-785fab087237/pseudopotentials.pickle \\\n --arg_calculation=/tmp/_wg2md43/stg738933fd-9a31-4c0e-ac99-984d1002fd0a/calculation_1.pickle \\\n --arg_smearing=/tmp/_wg2md43/stg174b61ec-1446-4c63-a433-49a940e48f39/smearing.pickle\n\u001b[1;30mINFO\u001b[0m [job get_dict_27] completed success\n\u001b[1;30mINFO\u001b[0m [step get_dict_27] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step calculate_qe_4\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_4] start\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_4] /tmp/abio0491$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/pltanpun/stg312b10e4-f5f2-4770-b776-12c659453bef/workflow.py \\\n --function=workflow.calculate_qe \\\n --arg_input_dict=/tmp/pltanpun/stgceefbf38-ae57-4369-872d-4ea0895a6976/result.pickle \\\n --arg_working_directory=/tmp/pltanpun/stg6abc0b8b-ed0d-4692-b100-b510fe7de457/working_directory_2.pickle\n[jupyter-pythonworkflow-fl--x---ff79c5c6:01928] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_4] Max memory used: 244MiB\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_4] completed success\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_4] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_dict_29\n\u001b[1;30mINFO\u001b[0m [step get_dict_29] start\n\u001b[1;30mINFO\u001b[0m [job get_dict_29] /tmp/li__ygke$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_dict \\\n --arg_kpts=/tmp/20wxb6mt/stg4f9c6f04-e0da-4f1d-9b91-17867111fcec/kpts.pickle \\\n --arg_structure=/tmp/20wxb6mt/stg27aeb042-5dba-406d-ad3f-c1a15414d462/s_4.pickle \\\n --arg_pseudopotentials=/tmp/20wxb6mt/stg00a521b7-0d6f-44fa-849f-611e82b816d2/pseudopotentials.pickle \\\n --arg_calculation=/tmp/20wxb6mt/stg51a4e465-9a6d-465e-a329-ec65c9631dca/calculation_1.pickle \\\n --arg_smearing=/tmp/20wxb6mt/stg72f4b40d-8647-441e-94ce-b4d60eb84837/smearing.pickle\n\u001b[1;30mINFO\u001b[0m [job get_dict_29] completed success\n\u001b[1;30mINFO\u001b[0m [step get_dict_29] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step calculate_qe_7\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_7] start\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_7] /tmp/26ontyw1$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/3e7bw8e2/stg86169764-ce4e-40f6-9b63-ef5e11208258/workflow.py \\\n --function=workflow.calculate_qe \\\n --arg_input_dict=/tmp/3e7bw8e2/stge349561c-3f24-48da-94fa-2d16e4df4779/result.pickle \\\n --arg_working_directory=/tmp/3e7bw8e2/stg30622d6d-1c64-4440-9d76-7a70e081ff5e/working_directory_5.pickle\n[jupyter-pythonworkflow-fl--x---ff79c5c6:02006] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_7] Max memory used: 243MiB\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_7] completed success\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_7] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step calculate_qe_5\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_5] start\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_5] /tmp/nwlrltkk$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/q_jjx0ja/stgb8b85eae-791f-4e34-9696-ceb277f72155/workflow.py \\\n --function=workflow.calculate_qe \\\n --arg_input_dict=/tmp/q_jjx0ja/stg2efa8335-1138-47c3-b9e9-d8ea2d8754c0/result.pickle \\\n --arg_working_directory=/tmp/q_jjx0ja/stg4f726c25-7adb-4ce0-970e-bd9ddc5191cc/working_directory_3.pickle\n[jupyter-pythonworkflow-fl--x---ff79c5c6:02086] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_5] Max memory used: 246MiB\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_5] completed success\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_5] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step calculate_qe_6\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_6] start\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_6] /tmp/9yp55i6u$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/1c_enl4x/stg2aeb7a80-3364-48a0-810d-955e246aa33b/workflow.py \\\n --function=workflow.calculate_qe \\\n --arg_input_dict=/tmp/1c_enl4x/stg19bec723-020c-4d0e-bca0-8124f6962c7e/result.pickle \\\n --arg_working_directory=/tmp/1c_enl4x/stgc36f93b3-f6f6-4fec-8eb2-0de1ca22f3cd/working_directory_4.pickle\n[jupyter-pythonworkflow-fl--x---ff79c5c6:02164] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_6] Max memory used: 243MiB\n\u001b[1;30mINFO\u001b[0m [job calculate_qe_6] completed success\n\u001b[1;30mINFO\u001b[0m [step calculate_qe_6] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_list_31\n\u001b[1;30mINFO\u001b[0m [step get_list_31] start\n\u001b[1;30mINFO\u001b[0m [job get_list_31] /tmp/xgme48c0$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_list \\\n --arg_2=/tmp/u2adengu/stg036d73c4-50fd-4b84-9054-ef00e66e0ea8/energy.pickle \\\n --arg_4=/tmp/u2adengu/stg8736d42d-b110-43b3-8e1d-da22b9883ff5/energy.pickle \\\n --arg_1=/tmp/u2adengu/stg8cac7eb7-6132-4085-865d-673027e26baa/energy.pickle \\\n --arg_3=/tmp/u2adengu/stg55b370b4-c852-4493-bb7b-4acaa0588aa8/energy.pickle \\\n --arg_0=/tmp/u2adengu/stg5d319d67-b4fc-4c4d-886a-529b9bfd2458/energy.pickle\n\u001b[1;30mINFO\u001b[0m [job get_list_31] completed success\n\u001b[1;30mINFO\u001b[0m [step get_list_31] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step get_list_30\n\u001b[1;30mINFO\u001b[0m [step get_list_30] start\n\u001b[1;30mINFO\u001b[0m [job get_list_30] /tmp/1xbu0rii$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --function=python_workflow_definition.shared.get_list \\\n --arg_2=/tmp/0ads3xzd/stg24fc8770-2739-45d1-b535-89924a6234ed/volume.pickle \\\n --arg_4=/tmp/0ads3xzd/stgfb79ea5f-6109-4f2e-a1d6-0817648a5350/volume.pickle \\\n --arg_1=/tmp/0ads3xzd/stg0b1e9637-dba2-4e42-9dbe-c605e7469714/volume.pickle \\\n --arg_3=/tmp/0ads3xzd/stgdf4991ae-4266-4ab6-82be-8f87d46fa7f6/volume.pickle \\\n --arg_0=/tmp/0ads3xzd/stg623a06b5-d2a0-44fd-9473-833452ad5762/volume.pickle\n\u001b[1;30mINFO\u001b[0m [job get_list_30] completed success\n\u001b[1;30mINFO\u001b[0m [step get_list_30] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] starting step plot_energy_volume_curve_8\n\u001b[1;30mINFO\u001b[0m [step plot_energy_volume_curve_8] start\n\u001b[1;30mINFO\u001b[0m [job plot_energy_volume_curve_8] /tmp/vcffw_nr$ python \\\n -m \\\n python_workflow_definition.cwl \\\n --workflowfile=/tmp/6s32h5zf/stg79d5980d-139d-4481-9277-3f58bb27d49f/workflow.py \\\n --function=workflow.plot_energy_volume_curve \\\n --arg_volume_lst=/tmp/6s32h5zf/stg67eb8ed0-f498-4181-a5a5-c7e74ecd3276/result.pickle \\\n --arg_energy_lst=/tmp/6s32h5zf/stg3ebdb224-bd9a-4ae7-ad4e-03489dcfb661/result.pickle\n\u001b[1;30mINFO\u001b[0m [job plot_energy_volume_curve_8] Max memory used: 110MiB\n\u001b[1;30mINFO\u001b[0m [job plot_energy_volume_curve_8] completed success\n\u001b[1;30mINFO\u001b[0m [step plot_energy_volume_curve_8] completed success\n\u001b[1;30mINFO\u001b[0m [workflow ] completed success\n{\n \"result_file\": {\n \"location\": \"file:///home/jovyan/example_workflows/quantum_espresso/result.pickle\",\n \"basename\": \"result.pickle\",\n \"class\": \"File\",\n \"checksum\": \"sha1$dbc1aaddc8b7343d6d33b34edcf608b8f8801918\",\n \"size\": 4,\n \"path\": \"/home/jovyan/example_workflows/quantum_espresso/result.pickle\"\n }\n}\u001b[1;30mINFO\u001b[0m Final process status is success\n"}],"execution_count":6},{"id":"2942dbba-ea0a-4d20-be5c-ed9992d09ff8","cell_type":"code","source":"with open(\"result.pickle\", \"rb\") as f:\n print(pickle.load(f))","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"None\n"}],"execution_count":7},{"id":"60e909ee-d0d0-4bd1-81c8-dd5274ae5834","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]} \ No newline at end of file diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/__init__.py b/python_workflow_definition/src/python_workflow_definition/cwl/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py b/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py new file mode 100644 index 0000000..ec1c6c4 --- /dev/null +++ b/python_workflow_definition/src/python_workflow_definition/cwl/__main__.py @@ -0,0 +1,51 @@ +import sys +import pickle +from ast import literal_eval +import importlib.util + + +def load_function(file_name, funct): + spec = importlib.util.spec_from_file_location("workflow", file_name) + module = importlib.util.module_from_spec(spec) + sys.modules["workflow"] = module + spec.loader.exec_module(module) + return getattr(module, funct.split(".")[-1]) + + +def convert_argument(arg): + if ".pickle" in arg: + with open(arg, "rb") as f: + return pickle.load(f) + else: + return literal_eval(arg) + + +if __name__ == "__main__": + # load input + argument_lst = sys.argv[1:] + funct_lst = [arg.split("=")[-1] for arg in argument_lst if "--function=" in arg] + file_lst = [arg.split("=")[-1] for arg in argument_lst if "--workflowfile=" in arg] + if len(file_lst) > 0: + workflow_function = load_function(file_name=file_lst[0], funct=funct_lst[0]) + internal_function = False + else: + m, p = funct_lst[0].rsplit(".", 1) + workflow_function = getattr(importlib.import_module(m), p) + internal_function = True + kwargs = { + arg.split("=")[0][6:]: convert_argument(arg=arg.split("=")[-1]) + for arg in argument_lst + if "--arg_" in arg + } + + # evaluate function + result = workflow_function(**kwargs) + + # store output + if isinstance(result, dict) and not internal_function: + for k, v in result.items(): + with open(k + ".pickle", "wb") as f: + pickle.dump(v, f) + else: + with open("result.pickle", "wb") as f: + pickle.dump(result, f) diff --git a/python_workflow_definition/src/python_workflow_definition/cwl/export.py b/python_workflow_definition/src/python_workflow_definition/cwl/export.py new file mode 100644 index 0000000..ee87a7c --- /dev/null +++ b/python_workflow_definition/src/python_workflow_definition/cwl/export.py @@ -0,0 +1,240 @@ +import json +import pickle +from yaml import CDumper as Dumper, dump + + +from python_workflow_definition.purepython import ( + group_edges, + resort_total_lst, +) +from python_workflow_definition.shared import ( + convert_nodes_list_to_dict, + remove_result, + EDGES_LABEL, + NODES_LABEL, + TARGET_LABEL, + TARGET_PORT_LABEL, + SOURCE_LABEL, + SOURCE_PORT_LABEL, +) + + +def _get_function_argument(argument: str, position: int = 3) -> dict: + return { + argument + + "_file": { + "type": "File", + "inputBinding": { + "prefix": "--arg_" + argument + "=", + "separate": False, + "position": position, + }, + }, + } + + +def _get_function_template(function_name: str) -> dict: + return { + "function": { + "default": function_name, + "inputBinding": {"position": 3, "prefix": "--function=", "separate": False}, + "type": "string", + }, + } + + +def _get_output_name(output_name: str) -> dict: + return { + output_name + + "_file": {"type": "File", "outputBinding": {"glob": output_name + ".pickle"}} + } + + +def _get_function(workflow): + function_nodes_dict = { + n["id"]: n["value"] for n in workflow[NODES_LABEL] if n["type"] == "function" + } + funct_dict = {} + for funct_id in function_nodes_dict.keys(): + target_ports = list( + set( + [ + e[TARGET_PORT_LABEL] + for e in workflow[EDGES_LABEL] + if e["target"] == funct_id + ] + ) + ) + source_ports = list( + set( + [ + e[SOURCE_PORT_LABEL] + for e in workflow[EDGES_LABEL] + if e["source"] == funct_id + ] + ) + ) + funct_dict[funct_id] = { + "targetPorts": target_ports, + "sourcePorts": source_ports, + } + return function_nodes_dict, funct_dict + + +def _write_function_cwl(workflow): + function_nodes_dict, funct_dict = _get_function(workflow) + + for i in function_nodes_dict.keys(): + template = { + "cwlVersion": "v1.2", + "class": "CommandLineTool", + "baseCommand": "python", + "inputs": { + "wrapper": { + "type": "string", + "inputBinding": {"position": 1, "prefix": "-m"}, + "default": "python_workflow_definition.cwl", + }, + "workflowfile": { + "type": "File", + "inputBinding": { + "position": 2, + "prefix": "--workflowfile=", + "separate": False, + }, + "default": {"class": "File", "location": "workflow.py"}, + }, + }, + "outputs": {}, + } + file_name = function_nodes_dict[i].split(".")[-1] + "_" + str(i) + ".cwl" + if function_nodes_dict[i].split(".")[0] != "python_workflow_definition": + template["inputs"]["workflowfile"]["default"]["location"] = ( + function_nodes_dict[i].split(".")[0] + ".py" + ) + else: + del template["inputs"]["workflowfile"] + template["inputs"].update( + _get_function_template(function_name=function_nodes_dict[i]) + ) + for j, arg in enumerate(funct_dict[i]["targetPorts"]): + template["inputs"].update( + _get_function_argument(argument=arg, position=4 + j) + ) + for out in funct_dict[i]["sourcePorts"]: + if out is None: + template["outputs"].update(_get_output_name(output_name="result")) + else: + template["outputs"].update(_get_output_name(output_name=out)) + with open(file_name, "w") as f: + dump(template, f, Dumper=Dumper) + + +def _write_workflow_config(workflow): + input_dict = { + n["name"]: n["value"] for n in workflow[NODES_LABEL] if n["type"] == "input" + } + with open("workflow.yml", "w") as f: + dump( + { + k + "_file": {"class": "File", "path": k + ".pickle"} + for k in input_dict.keys() + }, + f, + Dumper=Dumper, + ) + for k, v in input_dict.items(): + with open(k + ".pickle", "wb") as f: + pickle.dump(v, f) + + +def _write_workflow(workflow): + workflow_template = { + "cwlVersion": "v1.2", + "class": "Workflow", + "inputs": {}, + "steps": {}, + "outputs": {}, + } + input_dict = { + n["name"]: n["value"] for n in workflow[NODES_LABEL] if n["type"] == "input" + } + function_nodes_dict, funct_dict = _get_function(workflow) + result_id = [n["id"] for n in workflow[NODES_LABEL] if n["type"] == "output"][0] + last_compute_id = [ + e[SOURCE_LABEL] for e in workflow[EDGES_LABEL] if e[TARGET_LABEL] == result_id + ][0] + workflow_template["inputs"].update({k + "_file": "File" for k in input_dict.keys()}) + if funct_dict[last_compute_id]["sourcePorts"] == [None]: + workflow_template["outputs"] = { + "result_file": { + "type": "File", + "outputSource": function_nodes_dict[last_compute_id].split(".")[-1] + + "_" + + str(last_compute_id) + + "/result_file", + }, + } + else: + raise ValueError() + + content = remove_result(workflow_dict=workflow) + edges_new_lst = content[EDGES_LABEL] + total_lst = group_edges(edges_new_lst) + nodes_new_dict = { + int(k): v + for k, v in convert_nodes_list_to_dict(nodes_list=content[NODES_LABEL]).items() + } + total_new_lst = resort_total_lst(total_lst=total_lst, nodes_dict=nodes_new_dict) + step_name_lst = { + t[0]: function_nodes_dict[t[0]].split(".")[-1] for t in total_new_lst + } + input_id_dict = { + n["id"]: n["name"] for n in workflow[NODES_LABEL] if n["type"] == "input" + } + for t in total_new_lst: + ind = t[0] + node_script = step_name_lst[ind] + "_" + str(ind) + ".cwl" + output = [ + o + "_file" if o is not None else "result_file" + for o in funct_dict[ind]["sourcePorts"] + ] + in_dict = {} + for k, v in t[1].items(): + if v[SOURCE_LABEL] in input_id_dict: + in_dict[k + "_file"] = input_id_dict[v[SOURCE_LABEL]] + "_file" + else: + if v["sourcePort"] is None: + in_dict[k + "_file"] = ( + step_name_lst[v[SOURCE_LABEL]] + + "_" + + str(v[SOURCE_LABEL]) + + "/result_file" + ) + else: + in_dict[k + "_file"] = ( + step_name_lst[v[SOURCE_LABEL]] + + "_" + + str(v[SOURCE_LABEL]) + + "/" + + v[SOURCE_PORT_LABEL] + + "_file" + ) + workflow_template["steps"].update( + { + step_name_lst[ind] + + "_" + + str(ind): {"run": node_script, "in": in_dict, "out": output} + } + ) + with open("workflow.cwl", "w") as f: + dump(workflow_template, f, Dumper=Dumper) + + +def load_workflow_json(file_name: str): + with open(file_name, "r") as f: + workflow = json.load(f) + + _write_function_cwl(workflow=workflow) + _write_workflow_config(workflow=workflow) + _write_workflow(workflow=workflow)