From eff11efa294f8eb6197f0fcebaa62d8ca933a1fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Fri, 3 Oct 2025 22:48:43 +0200 Subject: [PATCH 01/12] Fix integration notebook --- notebooks/5-1-gpaw.ipynb | 116 ++++++++++++++++----------- notebooks/5-2-quantum-espresso.ipynb | 116 ++++++++++++++++----------- 2 files changed, 137 insertions(+), 95 deletions(-) diff --git a/notebooks/5-1-gpaw.ipynb b/notebooks/5-1-gpaw.ipynb index 98e94eb5..98b5506d 100644 --- a/notebooks/5-1-gpaw.ipynb +++ b/notebooks/5-1-gpaw.ipynb @@ -11,16 +11,16 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "21852c01-efda-43fe-add9-d08123a82995", "metadata": {}, "outputs": [], "source": [ "import subprocess\n", "from ase.build import bulk\n", - "from atomistics.workflows.evcurve.helper import (\n", - " analyse_structures_helper as evcurve_analyse_structures,\n", - " generate_structures_helper as evcurve_generate_structures,\n", + "from atomistics.workflows import (\n", + " analyse_results_for_energy_volume_curve,\n", + " get_tasks_for_energy_volume_curve,\n", ")\n", "import matplotlib.pyplot as plt\n", "import pprint\n", @@ -29,10 +29,12 @@ ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "The only function which is executed using [mpi4py](https://mpi4py.readthedocs.io) is the `evaluate_with_gpaw()` function:", - "id": "e15eed0da3c6547b" + "id": "e15eed0da3c6547b", + "metadata": {}, + "source": [ + "The only function which is executed using [mpi4py](https://mpi4py.readthedocs.io) is the `evaluate_with_gpaw()` function:" + ] }, { "cell_type": "code", @@ -54,19 +56,21 @@ ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "As a first step of the workflow the equilibrium structure of four Aluminium atoms is strained by 5%.", - "id": "e9936ca22151d490" + "id": "e9936ca22151d490", + "metadata": {}, + "source": [ + "As a first step of the workflow the equilibrium structure of four Aluminium atoms is strained by 5%." + ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "a1d4e9a1-7275-4d9a-b038-db27185f00ae", "metadata": {}, "outputs": [], "source": [ - "structure_dict = evcurve_generate_structures(\n", + "task_dict = get_tasks_for_energy_volume_curve(\n", " structure=bulk(\"Al\", a=4.05, cubic=True),\n", " num_points=7,\n", " vol_range=0.05,\n", @@ -75,26 +79,30 @@ ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "The resulting dictionary of structures `structure_dict` is transformed to simplify the parallel execution:", - "id": "8bf60b7b0f5af31a" + "id": "8bf60b7b0f5af31a", + "metadata": {}, + "source": [ + "The resulting dictionary of structures `task_dict` is transformed to simplify the parallel execution:" + ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "73ad1043-eacd-4758-a7b4-f64969beabab", "metadata": {}, "outputs": [], "source": [ - "task_loop_dict = {k: {\"calc_energy\": v} for k, v in structure_dict.items()}" + "task_loop_dict = {k: {\"calc_energy\": v} for k, v in task_dict[\"calc_energy\"].items()}" ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "The status of the flux cluster is validated using the `flux resource list` command and the `flux jobs -a` command, just to highlight flux was initialized correctly and has access to the available resources.", - "id": "af5ba456f3fafd39" + "id": "af5ba456f3fafd39", + "metadata": {}, + "source": [ + "The status of the flux cluster is validated using the `flux resource list` command and the `flux jobs -a` command, just to highlight flux was initialized correctly and has access to the available resources." + ] }, { "cell_type": "code", @@ -157,14 +165,16 @@ ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "The for each strained structure a calculation task is submitted to the `FluxClusterExecutor`. After the successful submission the current status of the flux queue is printed using `flux jobs -a`. Finally, the results are collected by gathering the [concurrent.futures.Future](https://docs.python.org/3/library/concurrent.futures.html#future-objects) objects.", - "id": "2670c218fbde5b02" + "id": "2670c218fbde5b02", + "metadata": {}, + "source": [ + "The for each strained structure a calculation task is submitted to the `FluxClusterExecutor`. After the successful submission the current status of the flux queue is printed using `flux jobs -a`. Finally, the results are collected by gathering the [concurrent.futures.Future](https://docs.python.org/3/library/concurrent.futures.html#future-objects) objects." + ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "11d7a7f3-5fbf-4377-ac98-71cd29caf433", "metadata": {}, "outputs": [ @@ -224,7 +234,7 @@ " sleep(1)\n", " pprint.pp(subprocess.check_output([\"flux\", \"jobs\", \"-a\"], universal_newlines=True).split(\"\\n\"))\n", " result_dict = {\n", - " k: f.result()[-1] \n", + " k: f.result() \n", " for k, f in tqdm(future_dict.items())\n", " }\n", " sleep(1)\n", @@ -232,31 +242,35 @@ ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "The resulting energies for the different volumes are fitted using a 3rd order polynomial to derive the bulk modulus as second derivative multiplied by the equilibrium volume.", - "id": "8c80e3bcb483b069" + "id": "8c80e3bcb483b069", + "metadata": {}, + "source": [ + "The resulting energies for the different volumes are fitted using a 3rd order polynomial to derive the bulk modulus as second derivative multiplied by the equilibrium volume." + ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "5a5822c7-5ab6-483b-a3ab-8cef5fcaf269", "metadata": {}, "outputs": [], "source": [ - "fit_dict = evcurve_analyse_structures(\n", + "fit_dict = analyse_results_for_energy_volume_curve(\n", " output_dict={\"energy\": result_dict},\n", - " structure_dict=structure_dict,\n", + " task_dict=task_dict,\n", " fit_type=\"polynomial\",\n", " fit_order=3,\n", ")" ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "The final energy volume curve plot summarizes the results of this calculation.", - "id": "571b7c25dcca8bb" + "id": "571b7c25dcca8bb", + "metadata": {}, + "source": [ + "The final energy volume curve plot summarizes the results of this calculation." + ] }, { "cell_type": "code", @@ -322,14 +336,16 @@ ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "The for each strained structure a calculation task is submitted to the `FluxJobExecutor`. After the successful submission the current status of the flux queue is printed using `flux jobs -a`. Finally, the results are collected by gathering the [concurrent.futures.Future](https://docs.python.org/3/library/concurrent.futures.html#future-objects) objects.", - "id": "21bdd0d42f9806e4" + "id": "21bdd0d42f9806e4", + "metadata": {}, + "source": [ + "The for each strained structure a calculation task is submitted to the `FluxJobExecutor`. After the successful submission the current status of the flux queue is printed using `flux jobs -a`. Finally, the results are collected by gathering the [concurrent.futures.Future](https://docs.python.org/3/library/concurrent.futures.html#future-objects) objects." + ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "2b0c559b-a117-4766-b679-314ab9e65e97", "metadata": {}, "outputs": [ @@ -421,7 +437,7 @@ " sleep(1)\n", " pprint.pp(subprocess.check_output([\"flux\", \"jobs\", \"-a\"], universal_newlines=True).split(\"\\n\"))\n", " result_dict = {\n", - " k: f.result()[-1] \n", + " k: f.result() \n", " for k, f in tqdm(future_dict.items())\n", " }\n", " sleep(1)\n", @@ -429,31 +445,35 @@ ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "The resulting energies for the different volumes are fitted using a 3rd order polynomial to derive the bulk modulus as second derivative multiplied by the equilibrium volume.", - "id": "ca353da1941c1c7c" + "id": "ca353da1941c1c7c", + "metadata": {}, + "source": [ + "The resulting energies for the different volumes are fitted using a 3rd order polynomial to derive the bulk modulus as second derivative multiplied by the equilibrium volume." + ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "8bb73863-bfde-4e96-9323-5330b2051632", "metadata": {}, "outputs": [], "source": [ - "fit_dict = evcurve_analyse_structures(\n", + "fit_dict = analyse_results_for_energy_volume_curve(\n", " output_dict={\"energy\": result_dict},\n", - " structure_dict=structure_dict,\n", + " task_dict=task_dict,\n", " fit_type=\"polynomial\",\n", " fit_order=3,\n", ")" ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "The final energy volume curve plot summarizes the results of this calculation.", - "id": "3d4189b5c74ba2a2" + "id": "3d4189b5c74ba2a2", + "metadata": {}, + "source": [ + "The final energy volume curve plot summarizes the results of this calculation." + ] }, { "cell_type": "code", diff --git a/notebooks/5-2-quantum-espresso.ipynb b/notebooks/5-2-quantum-espresso.ipynb index 02958042..491706ce 100644 --- a/notebooks/5-2-quantum-espresso.ipynb +++ b/notebooks/5-2-quantum-espresso.ipynb @@ -11,7 +11,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "e9003ea3-59da-442e-9af0-9d1b2bc72511", "metadata": {}, "outputs": [], @@ -19,9 +19,9 @@ "import os\n", "import subprocess\n", "from ase.build import bulk\n", - "from atomistics.workflows.evcurve.helper import (\n", - " analyse_structures_helper as evcurve_analyse_structures,\n", - " generate_structures_helper as evcurve_generate_structures,\n", + "from atomistics.workflows import (\n", + " analyse_results_for_energy_volume_curve,\n", + " get_tasks_for_energy_volume_curve,\n", ")\n", "import matplotlib.pyplot as plt\n", "import pprint\n", @@ -30,10 +30,12 @@ ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "The function which handles the parallel execution is the `evaluate_with_quantum_espresso()` function, internally it calls `flux run pw.x` to execute the quantum espresso Fortran executable in parallel:", - "id": "7e44687dee535c05" + "id": "7e44687dee535c05", + "metadata": {}, + "source": [ + "The function which handles the parallel execution is the `evaluate_with_quantum_espresso()` function, internally it calls `flux run pw.x` to execute the quantum espresso Fortran executable in parallel:" + ] }, { "cell_type": "code", @@ -62,10 +64,12 @@ ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "The pseudo potential is located in `tests/integration`, here the absolute path to the pseudo potential is given as: `/home/runner/work/executorlib/executorlib/tests/integration`.", - "id": "89e65ac14c2ae034" + "id": "89e65ac14c2ae034", + "metadata": {}, + "source": [ + "The pseudo potential is located in `tests/integration`, here the absolute path to the pseudo potential is given as: `/home/runner/work/executorlib/executorlib/tests/integration`." + ] }, { "cell_type": "code", @@ -78,19 +82,21 @@ ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "As a first step of the workflow the equilibrium structure of four Aluminium atoms is strained by 5%.", - "id": "246aac73af27fcb3" + "id": "246aac73af27fcb3", + "metadata": {}, + "source": [ + "As a first step of the workflow the equilibrium structure of four Aluminium atoms is strained by 5%." + ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "ed98740b-920e-48f1-b4e9-247de631c794", "metadata": {}, "outputs": [], "source": [ - "structure_dict = evcurve_generate_structures(\n", + "task_dict = get_tasks_for_energy_volume_curve(\n", " structure=bulk(\"Al\", a=4.15, cubic=True),\n", " num_points=7,\n", " vol_range=0.05,\n", @@ -99,26 +105,30 @@ ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "The resulting dictionary of structures `structure_dict` is transformed to simplify the parallel execution:", - "id": "91dc9dc13d40835e" + "id": "91dc9dc13d40835e", + "metadata": {}, + "source": [ + "The resulting dictionary of structures `task_dict` is transformed to simplify the parallel execution:" + ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "944ad3fb-20e1-4956-a77a-f492e1477841", "metadata": {}, "outputs": [], "source": [ - "task_loop_dict = {k: {\"calc_energy\": v} for k, v in structure_dict.items()}" + "task_loop_dict = {k: {\"calc_energy\": v} for k, v in task_dict[\"calc_energy\"].items()}" ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "The status of the flux cluster is validated using the `flux resource list` command and the `flux jobs -a` command, just to highlight flux was initialized correctly and has access to the available resources.", - "id": "80b40994805c5e10" + "id": "80b40994805c5e10", + "metadata": {}, + "source": [ + "The status of the flux cluster is validated using the `flux resource list` command and the `flux jobs -a` command, just to highlight flux was initialized correctly and has access to the available resources." + ] }, { "cell_type": "code", @@ -181,10 +191,12 @@ ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "The for each strained structure a calculation task is submitted to the `FluxClusterExecutor`. After the successful submission the current status of the flux queue is printed using `flux jobs -a`. Finally, the results are collected by gathering the [concurrent.futures.Future](https://docs.python.org/3/library/concurrent.futures.html#future-objects) objects.", - "id": "21f9e2a13fafd6fd" + "id": "21f9e2a13fafd6fd", + "metadata": {}, + "source": [ + "The for each strained structure a calculation task is submitted to the `FluxClusterExecutor`. After the successful submission the current status of the flux queue is printed using `flux jobs -a`. Finally, the results are collected by gathering the [concurrent.futures.Future](https://docs.python.org/3/library/concurrent.futures.html#future-objects) objects." + ] }, { "cell_type": "code", @@ -261,31 +273,35 @@ ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "The resulting energies for the different volumes are fitted using a 3rd order polynomial to derive the bulk modulus as second derivative multiplied by the equilibrium volume.", - "id": "ab134a77a64251a3" + "id": "ab134a77a64251a3", + "metadata": {}, + "source": [ + "The resulting energies for the different volumes are fitted using a 3rd order polynomial to derive the bulk modulus as second derivative multiplied by the equilibrium volume." + ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "a340eec1-7e7c-4d0b-9503-df443f38550e", "metadata": {}, "outputs": [], "source": [ - "fit_dict = evcurve_analyse_structures(\n", + "fit_dict = analyse_results_for_energy_volume_curve(\n", " output_dict={\"energy\": result_dict},\n", - " structure_dict=structure_dict,\n", + " task_dict=task_dict,\n", " fit_type=\"polynomial\",\n", " fit_order=3,\n", ")" ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "The final energy volume curve plot summarizes the results of this calculation.", - "id": "175dd036cb6e4bdd" + "id": "175dd036cb6e4bdd", + "metadata": {}, + "source": [ + "The final energy volume curve plot summarizes the results of this calculation." + ] }, { "cell_type": "code", @@ -351,10 +367,12 @@ ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "The for each strained structure a calculation task is submitted to the `FluxJobExecutor`. After the successful submission the current status of the flux queue is printed using `flux jobs -a`. Finally, the results are collected by gathering the [concurrent.futures.Future](https://docs.python.org/3/library/concurrent.futures.html#future-objects) objects.", - "id": "800c01d8a5347bb1" + "id": "800c01d8a5347bb1", + "metadata": {}, + "source": [ + "The for each strained structure a calculation task is submitted to the `FluxJobExecutor`. After the successful submission the current status of the flux queue is printed using `flux jobs -a`. Finally, the results are collected by gathering the [concurrent.futures.Future](https://docs.python.org/3/library/concurrent.futures.html#future-objects) objects." + ] }, { "cell_type": "code", @@ -463,31 +481,35 @@ ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "The resulting energies for the different volumes are fitted using a 3rd order polynomial to derive the bulk modulus as second derivative multiplied by the equilibrium volume.", - "id": "8a9c8658af1dda5a" + "id": "8a9c8658af1dda5a", + "metadata": {}, + "source": [ + "The resulting energies for the different volumes are fitted using a 3rd order polynomial to derive the bulk modulus as second derivative multiplied by the equilibrium volume." + ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "35faea8e-8240-4b88-aa0f-1a55684675ea", "metadata": {}, "outputs": [], "source": [ - "fit_dict = evcurve_analyse_structures(\n", + "fit_dict = analyse_results_for_energy_volume_curve(\n", " output_dict={\"energy\": result_dict},\n", - " structure_dict=structure_dict,\n", + " task_dict=task_dict,\n", " fit_type=\"polynomial\",\n", " fit_order=3,\n", ")" ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "The final energy volume curve plot summarizes the results of this calculation.", - "id": "9c3358ea83647f01" + "id": "9c3358ea83647f01", + "metadata": {}, + "source": [ + "The final energy volume curve plot summarizes the results of this calculation." + ] }, { "cell_type": "code", From 328ced4abb11eae82070b29aaf3f313673ba7ccc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Fri, 3 Oct 2025 23:11:56 +0200 Subject: [PATCH 02/12] limit maximum number of workers --- notebooks/5-1-gpaw.ipynb | 4 ++-- notebooks/5-2-quantum-espresso.ipynb | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/notebooks/5-1-gpaw.ipynb b/notebooks/5-1-gpaw.ipynb index 98b5506d..21fa904e 100644 --- a/notebooks/5-1-gpaw.ipynb +++ b/notebooks/5-1-gpaw.ipynb @@ -222,7 +222,7 @@ ], "source": [ "future_dict = {}\n", - "with FluxClusterExecutor() as exe:\n", + "with FluxClusterExecutor(max_workers=2) as exe:\n", " for k, v in task_loop_dict.items():\n", " future_dict[k] = exe.submit(\n", " evaluate_with_gpaw, \n", @@ -425,7 +425,7 @@ ], "source": [ "future_dict = {}\n", - "with FluxJobExecutor() as exe:\n", + "with FluxJobExecutor(max_workers=2) as exe:\n", " for k, v in task_loop_dict.items():\n", " future_dict[k] = exe.submit(\n", " evaluate_with_gpaw, \n", diff --git a/notebooks/5-2-quantum-espresso.ipynb b/notebooks/5-2-quantum-espresso.ipynb index 491706ce..d3e9e912 100644 --- a/notebooks/5-2-quantum-espresso.ipynb +++ b/notebooks/5-2-quantum-espresso.ipynb @@ -200,7 +200,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "4f7e27df-ca14-4fc6-971a-054716de38b6", "metadata": {}, "outputs": [ @@ -248,7 +248,7 @@ ], "source": [ "future_dict = {}\n", - "with FluxClusterExecutor() as exe:\n", + "with FluxClusterExecutor(max_workers=2) as exe:\n", " for k, v in task_loop_dict.items():\n", " os.makedirs(os.path.abspath((\"strain_%0.2f\" % k).replace(\".\", \"_\")), exist_ok=True)\n", " future_dict[k] = exe.submit(\n", @@ -376,7 +376,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "cb3c60b7-e9f4-4ac9-9ed1-cd372e802353", "metadata": {}, "outputs": [ @@ -456,7 +456,7 @@ ], "source": [ "future_dict = {}\n", - "with FluxJobExecutor(flux_executor_nesting=True) as exe:\n", + "with FluxJobExecutor(max_workers=2, flux_executor_nesting=True) as exe:\n", " for k, v in task_loop_dict.items():\n", " os.makedirs(os.path.abspath((\"strain_%0.2f\" % k).replace(\".\", \"_\")), exist_ok=True)\n", " future_dict[k] = exe.submit(\n", From 4839ffa960d7794dfa5327468fa4d3197e632837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Fri, 3 Oct 2025 23:14:12 +0200 Subject: [PATCH 03/12] fix cluster --- notebooks/5-1-gpaw.ipynb | 2 +- notebooks/5-2-quantum-espresso.ipynb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/notebooks/5-1-gpaw.ipynb b/notebooks/5-1-gpaw.ipynb index 21fa904e..f6c5ab0a 100644 --- a/notebooks/5-1-gpaw.ipynb +++ b/notebooks/5-1-gpaw.ipynb @@ -222,7 +222,7 @@ ], "source": [ "future_dict = {}\n", - "with FluxClusterExecutor(max_workers=2) as exe:\n", + "with FluxClusterExecutor() as exe:\n", " for k, v in task_loop_dict.items():\n", " future_dict[k] = exe.submit(\n", " evaluate_with_gpaw, \n", diff --git a/notebooks/5-2-quantum-espresso.ipynb b/notebooks/5-2-quantum-espresso.ipynb index d3e9e912..04659213 100644 --- a/notebooks/5-2-quantum-espresso.ipynb +++ b/notebooks/5-2-quantum-espresso.ipynb @@ -248,7 +248,7 @@ ], "source": [ "future_dict = {}\n", - "with FluxClusterExecutor(max_workers=2) as exe:\n", + "with FluxClusterExecutor() as exe:\n", " for k, v in task_loop_dict.items():\n", " os.makedirs(os.path.abspath((\"strain_%0.2f\" % k).replace(\".\", \"_\")), exist_ok=True)\n", " future_dict[k] = exe.submit(\n", From 98565391c321c93890a7d1a4393d024c29eaab3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 4 Oct 2025 07:43:43 +0200 Subject: [PATCH 04/12] restrict threads --- notebooks/5-1-gpaw.ipynb | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/notebooks/5-1-gpaw.ipynb b/notebooks/5-1-gpaw.ipynb index f6c5ab0a..951940d5 100644 --- a/notebooks/5-1-gpaw.ipynb +++ b/notebooks/5-1-gpaw.ipynb @@ -9,6 +9,17 @@ "The [gpaw](https://gpaw.readthedocs.io/index.html) density-functional theory (DFT) simulation code provides a Python interface supporting the [message passing interface (MPI) for Python - mpi4py](https://mpi4py.readthedocs.io) for parallelization. So `executorlib` is used to orchestrate multiple [gpaw](https://gpaw.readthedocs.io/index.html) simulation each using multiple CPU cores for parallelization. These kind of hierarchical workflows are one of the core strength of `executorlib`. While the same could be achieved by writing the whole simulation workflow using [mpi4py](https://mpi4py.readthedocs.io) this would drastically increase the complexity. With `executorlib` the user can quickly up-scale their simulation workflow without the need to address the parallel execution explicitly, rather the parallelization is introduced on a per-function level, by submitting the functions to the `FluxClusterExecutor` or `FluxJobExecutor`." ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "99a8ca1c", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "os.environ[\"OMP_NUM_THREADS\"] = \"1\" # Ensures that each task uses only one thread" + ] + }, { "cell_type": "code", "execution_count": null, From 3ecb8f2defe38669edba25e9f41484db03498f7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 4 Oct 2025 07:48:38 +0200 Subject: [PATCH 05/12] inside function --- notebooks/5-1-gpaw.ipynb | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/notebooks/5-1-gpaw.ipynb b/notebooks/5-1-gpaw.ipynb index 951940d5..925457b6 100644 --- a/notebooks/5-1-gpaw.ipynb +++ b/notebooks/5-1-gpaw.ipynb @@ -9,17 +9,6 @@ "The [gpaw](https://gpaw.readthedocs.io/index.html) density-functional theory (DFT) simulation code provides a Python interface supporting the [message passing interface (MPI) for Python - mpi4py](https://mpi4py.readthedocs.io) for parallelization. So `executorlib` is used to orchestrate multiple [gpaw](https://gpaw.readthedocs.io/index.html) simulation each using multiple CPU cores for parallelization. These kind of hierarchical workflows are one of the core strength of `executorlib`. While the same could be achieved by writing the whole simulation workflow using [mpi4py](https://mpi4py.readthedocs.io) this would drastically increase the complexity. With `executorlib` the user can quickly up-scale their simulation workflow without the need to address the parallel execution explicitly, rather the parallelization is introduced on a per-function level, by submitting the functions to the `FluxClusterExecutor` or `FluxJobExecutor`." ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "99a8ca1c", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "os.environ[\"OMP_NUM_THREADS\"] = \"1\" # Ensures that each task uses only one thread" - ] - }, { "cell_type": "code", "execution_count": null, @@ -49,12 +38,15 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "aa435760-390f-4eff-88f4-96b15777de3e", "metadata": {}, "outputs": [], "source": [ "def evaluate_with_gpaw(task_dict, kpts, encut):\n", + " import os\n", + " os.environ[\"OMP_NUM_THREADS\"] = \"1\" # Ensures that each task uses only one thread\n", + "\n", " from gpaw import GPAW, PW\n", "\n", " structure = task_dict[\"calc_energy\"].copy()\n", From 25da5b5d2c2b8a8820d5f0558601f61f9a44b3ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 4 Oct 2025 07:54:20 +0200 Subject: [PATCH 06/12] list available resources --- .github/workflows/pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 987c884a..17aee847 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -186,6 +186,7 @@ jobs: shell: bash -l {0} timeout-minutes: 20 run: | + flux start flux resource list flux start papermill notebooks/5-1-gpaw.ipynb notebooks/5-1-gpaw-out.ipynb -k python3 flux start papermill notebooks/5-2-quantum-espresso.ipynb notebooks/5-2-quantum-espresso-out.ipynb -k python3 From e70c1e4f1a7ef112b622200b139f6322e220bd4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 4 Oct 2025 08:00:07 +0200 Subject: [PATCH 07/12] restrict number of cores --- notebooks/5-1-gpaw.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notebooks/5-1-gpaw.ipynb b/notebooks/5-1-gpaw.ipynb index 925457b6..27c85168 100644 --- a/notebooks/5-1-gpaw.ipynb +++ b/notebooks/5-1-gpaw.ipynb @@ -232,7 +232,7 @@ " task_dict=v, \n", " kpts=(3, 3, 3), \n", " encut=300,\n", - " resource_dict={\"cores\": 2},\n", + " resource_dict={\"cores\": 1},\n", " )\n", " sleep(1)\n", " pprint.pp(subprocess.check_output([\"flux\", \"jobs\", \"-a\"], universal_newlines=True).split(\"\\n\"))\n", @@ -435,7 +435,7 @@ " task_dict=v, \n", " kpts=(3, 3, 3), \n", " encut=300,\n", - " resource_dict={\"cores\": 2},\n", + " resource_dict={\"cores\": 1},\n", " )\n", " sleep(1)\n", " pprint.pp(subprocess.check_output([\"flux\", \"jobs\", \"-a\"], universal_newlines=True).split(\"\\n\"))\n", From c1aa2d368ef904ccd45bc212a69eb2a3346d172f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 4 Oct 2025 08:17:59 +0200 Subject: [PATCH 08/12] mpi again --- notebooks/5-1-gpaw.ipynb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/notebooks/5-1-gpaw.ipynb b/notebooks/5-1-gpaw.ipynb index 27c85168..98e3dd7f 100644 --- a/notebooks/5-1-gpaw.ipynb +++ b/notebooks/5-1-gpaw.ipynb @@ -47,12 +47,13 @@ " import os\n", " os.environ[\"OMP_NUM_THREADS\"] = \"1\" # Ensures that each task uses only one thread\n", "\n", - " from gpaw import GPAW, PW\n", + " from gpaw import GPAW, PW, mpi\n", "\n", " structure = task_dict[\"calc_energy\"].copy()\n", " structure.calc = GPAW(\n", " xc=\"PBE\",\n", " mode=PW(encut),\n", + " communicator=[mpi.world.rank],\n", " kpts=kpts,\n", " )\n", " return structure.get_potential_energy()" @@ -232,7 +233,7 @@ " task_dict=v, \n", " kpts=(3, 3, 3), \n", " encut=300,\n", - " resource_dict={\"cores\": 1},\n", + " resource_dict={\"cores\": 2},\n", " )\n", " sleep(1)\n", " pprint.pp(subprocess.check_output([\"flux\", \"jobs\", \"-a\"], universal_newlines=True).split(\"\\n\"))\n", @@ -435,7 +436,7 @@ " task_dict=v, \n", " kpts=(3, 3, 3), \n", " encut=300,\n", - " resource_dict={\"cores\": 1},\n", + " resource_dict={\"cores\": 2},\n", " )\n", " sleep(1)\n", " pprint.pp(subprocess.check_output([\"flux\", \"jobs\", \"-a\"], universal_newlines=True).split(\"\\n\"))\n", From be3870bc0985d22bd3f8cd2b6c511aa8f13d9c63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 4 Oct 2025 08:25:53 +0200 Subject: [PATCH 09/12] use world communicator --- notebooks/5-1-gpaw.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/5-1-gpaw.ipynb b/notebooks/5-1-gpaw.ipynb index 98e3dd7f..9e57c11d 100644 --- a/notebooks/5-1-gpaw.ipynb +++ b/notebooks/5-1-gpaw.ipynb @@ -53,7 +53,7 @@ " structure.calc = GPAW(\n", " xc=\"PBE\",\n", " mode=PW(encut),\n", - " communicator=[mpi.world.rank],\n", + " communicator=mpi.world,\n", " kpts=kpts,\n", " )\n", " return structure.get_potential_energy()" From 241a4ca1cdf4153c54dc16ec21d23cadff40051e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 4 Oct 2025 08:43:09 +0200 Subject: [PATCH 10/12] set mpi4py flag --- notebooks/5-1-gpaw.ipynb | 1 + 1 file changed, 1 insertion(+) diff --git a/notebooks/5-1-gpaw.ipynb b/notebooks/5-1-gpaw.ipynb index 9e57c11d..90d62147 100644 --- a/notebooks/5-1-gpaw.ipynb +++ b/notebooks/5-1-gpaw.ipynb @@ -46,6 +46,7 @@ "def evaluate_with_gpaw(task_dict, kpts, encut):\n", " import os\n", " os.environ[\"OMP_NUM_THREADS\"] = \"1\" # Ensures that each task uses only one thread\n", + " os.environ[\"GPAW_MPI4PY\"] = \"1\" # Ensures that GPAW uses MPI via mpi4py\n", "\n", " from gpaw import GPAW, PW, mpi\n", "\n", From b28a4136690192000d760d85e81fa9e343992cce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 4 Oct 2025 08:45:35 +0200 Subject: [PATCH 11/12] import mpi4py --- notebooks/5-1-gpaw.ipynb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/notebooks/5-1-gpaw.ipynb b/notebooks/5-1-gpaw.ipynb index 90d62147..68322212 100644 --- a/notebooks/5-1-gpaw.ipynb +++ b/notebooks/5-1-gpaw.ipynb @@ -48,13 +48,14 @@ " os.environ[\"OMP_NUM_THREADS\"] = \"1\" # Ensures that each task uses only one thread\n", " os.environ[\"GPAW_MPI4PY\"] = \"1\" # Ensures that GPAW uses MPI via mpi4py\n", "\n", - " from gpaw import GPAW, PW, mpi\n", + " from gpaw import GPAW, PW\n", + " from mpi4py import MPI\n", "\n", " structure = task_dict[\"calc_energy\"].copy()\n", " structure.calc = GPAW(\n", " xc=\"PBE\",\n", " mode=PW(encut),\n", - " communicator=mpi.world,\n", + " communicator=MPI.COMM_WORLD,\n", " kpts=kpts,\n", " )\n", " return structure.get_potential_energy()" From 240165b1e87061d52033d3012a8d2a35e59893f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 4 Oct 2025 08:53:09 +0200 Subject: [PATCH 12/12] just environmet variable --- notebooks/5-1-gpaw.ipynb | 2 -- 1 file changed, 2 deletions(-) diff --git a/notebooks/5-1-gpaw.ipynb b/notebooks/5-1-gpaw.ipynb index 68322212..8e25be8c 100644 --- a/notebooks/5-1-gpaw.ipynb +++ b/notebooks/5-1-gpaw.ipynb @@ -49,13 +49,11 @@ " os.environ[\"GPAW_MPI4PY\"] = \"1\" # Ensures that GPAW uses MPI via mpi4py\n", "\n", " from gpaw import GPAW, PW\n", - " from mpi4py import MPI\n", "\n", " structure = task_dict[\"calc_energy\"].copy()\n", " structure.calc = GPAW(\n", " xc=\"PBE\",\n", " mode=PW(encut),\n", - " communicator=MPI.COMM_WORLD,\n", " kpts=kpts,\n", " )\n", " return structure.get_potential_energy()"