From a1b8d523a38a837175753351dbfa0850919ddbb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 26 Apr 2025 12:10:20 +0200 Subject: [PATCH 1/9] Provide additional functions for import --- executorlib/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/executorlib/__init__.py b/executorlib/__init__.py index 3674d0f0..caee5666 100644 --- a/executorlib/__init__.py +++ b/executorlib/__init__.py @@ -9,14 +9,24 @@ SlurmJobExecutor, ) from executorlib.standalone.cache import get_cache_data +from executorlib.standalone.command import get_command_path +from executorlib.standalone.interactive.communication import SocketInterface, interface_bootup +from executorlib.standalone.interactive.spawner import MpiExecSpawner, SubprocessSpawner +from executorlib.standalone.queue import cancel_items_in_queue __all__: list[str] = [ + "cancel_items_in_queue", "get_cache_data", + "get_command_path", + "interface_bootup", "FluxJobExecutor", "FluxClusterExecutor", + "MpiExecSpawner", "SingleNodeExecutor", "SlurmJobExecutor", "SlurmClusterExecutor", + "SocketInterface", + "SubprocessSpawner", ] __version__ = _get_versions()["version"] From 99ef1cc3f1f64b98aa8c22a515baeff758427fe3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 26 Apr 2025 10:13:41 +0000 Subject: [PATCH 2/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- executorlib/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/executorlib/__init__.py b/executorlib/__init__.py index caee5666..dabd0694 100644 --- a/executorlib/__init__.py +++ b/executorlib/__init__.py @@ -10,7 +10,10 @@ ) from executorlib.standalone.cache import get_cache_data from executorlib.standalone.command import get_command_path -from executorlib.standalone.interactive.communication import SocketInterface, interface_bootup +from executorlib.standalone.interactive.communication import ( + SocketInterface, + interface_bootup, +) from executorlib.standalone.interactive.spawner import MpiExecSpawner, SubprocessSpawner from executorlib.standalone.queue import cancel_items_in_queue From bb86f67bf2256250de1e1947daa77d48c8e2866b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 26 Apr 2025 12:15:29 +0200 Subject: [PATCH 3/9] add cloudpickle_register to API --- executorlib/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/executorlib/__init__.py b/executorlib/__init__.py index caee5666..d98253d5 100644 --- a/executorlib/__init__.py +++ b/executorlib/__init__.py @@ -13,9 +13,11 @@ from executorlib.standalone.interactive.communication import SocketInterface, interface_bootup from executorlib.standalone.interactive.spawner import MpiExecSpawner, SubprocessSpawner from executorlib.standalone.queue import cancel_items_in_queue +from executorlib.standalone.serialize import cloudpickle_register __all__: list[str] = [ "cancel_items_in_queue", + "cloudpickle_register", "get_cache_data", "get_command_path", "interface_bootup", From 429d8ba2edba4fa790e9e08df2aa87b4f2f6054a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 26 Apr 2025 12:19:50 +0200 Subject: [PATCH 4/9] add socket functions --- executorlib/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/executorlib/__init__.py b/executorlib/__init__.py index 31ccb2e1..9f8f4aeb 100644 --- a/executorlib/__init__.py +++ b/executorlib/__init__.py @@ -13,6 +13,10 @@ from executorlib.standalone.interactive.communication import ( SocketInterface, interface_bootup, + interface_connect, + interface_receive, + interface_send, + interface_shutdown, ) from executorlib.standalone.interactive.spawner import MpiExecSpawner, SubprocessSpawner from executorlib.standalone.queue import cancel_items_in_queue @@ -24,6 +28,10 @@ "get_cache_data", "get_command_path", "interface_bootup", + "interface_connect", + "interface_receive", + "interface_send", + "interface_shutdown", "FluxJobExecutor", "FluxClusterExecutor", "MpiExecSpawner", From 9f6174367f79fb8f3f2bf87afd3f467fc97be6d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 26 Apr 2025 12:28:14 +0200 Subject: [PATCH 5/9] Separate user interface and developer interface --- executorlib/__init__.py | 25 +------------------------ executorlib/standalone/__init__.py | 20 +++++++++++++++----- 2 files changed, 16 insertions(+), 29 deletions(-) diff --git a/executorlib/__init__.py b/executorlib/__init__.py index 9f8f4aeb..abbaa132 100644 --- a/executorlib/__init__.py +++ b/executorlib/__init__.py @@ -9,37 +9,14 @@ SlurmJobExecutor, ) from executorlib.standalone.cache import get_cache_data -from executorlib.standalone.command import get_command_path -from executorlib.standalone.interactive.communication import ( - SocketInterface, - interface_bootup, - interface_connect, - interface_receive, - interface_send, - interface_shutdown, -) -from executorlib.standalone.interactive.spawner import MpiExecSpawner, SubprocessSpawner -from executorlib.standalone.queue import cancel_items_in_queue -from executorlib.standalone.serialize import cloudpickle_register __all__: list[str] = [ - "cancel_items_in_queue", - "cloudpickle_register", "get_cache_data", - "get_command_path", - "interface_bootup", - "interface_connect", - "interface_receive", - "interface_send", - "interface_shutdown", "FluxJobExecutor", "FluxClusterExecutor", - "MpiExecSpawner", "SingleNodeExecutor", "SlurmJobExecutor", "SlurmClusterExecutor", - "SocketInterface", - "SubprocessSpawner", ] -__version__ = _get_versions()["version"] +__version__ = _get_versions()["version"] \ No newline at end of file diff --git a/executorlib/standalone/__init__.py b/executorlib/standalone/__init__.py index 757663a8..c8637977 100644 --- a/executorlib/standalone/__init__.py +++ b/executorlib/standalone/__init__.py @@ -1,8 +1,11 @@ """ Submodules in the executorlib.standalone module do not depend on other modules of the executorlib package. This strict -separation simplifies the development, testing and debugging. +separation simplifies the development, testing and debugging. The functionality in executorlib.standalone is designed +to be used independently in other libraries. """ +from executorlib.standalone.cache import get_cache_data +from executorlib.standalone.command import get_command_path from executorlib.standalone.interactive.communication import ( SocketInterface, interface_bootup, @@ -11,14 +14,21 @@ interface_send, interface_shutdown, ) -from executorlib.standalone.interactive.spawner import MpiExecSpawner +from executorlib.standalone.interactive.spawner import MpiExecSpawner, SubprocessSpawner +from executorlib.standalone.queue import cancel_items_in_queue +from executorlib.standalone.serialize import cloudpickle_register -__all__ = [ - "SocketInterface", +__all__: list[str] = [ + "cancel_items_in_queue", + "cloudpickle_register", + "get_cache_data", + "get_command_path", "interface_bootup", "interface_connect", + "interface_receive", "interface_send", "interface_shutdown", - "interface_receive", "MpiExecSpawner", + "SocketInterface", + "SubprocessSpawner", ] From 4b39060bf855dbdd0e5ae570e93e5ea577c7ad23 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 26 Apr 2025 10:28:29 +0000 Subject: [PATCH 6/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- executorlib/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executorlib/__init__.py b/executorlib/__init__.py index abbaa132..3674d0f0 100644 --- a/executorlib/__init__.py +++ b/executorlib/__init__.py @@ -19,4 +19,4 @@ "SlurmClusterExecutor", ] -__version__ = _get_versions()["version"] \ No newline at end of file +__version__ = _get_versions()["version"] From 63cbf2419a75f4dc516c9b3e8fb56063b6361364 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 26 Apr 2025 12:40:02 +0200 Subject: [PATCH 7/9] add documentation --- executorlib/standalone/__init__.py | 2 -- notebooks/4-developer.ipynb | 32 +++++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/executorlib/standalone/__init__.py b/executorlib/standalone/__init__.py index c8637977..9ff19574 100644 --- a/executorlib/standalone/__init__.py +++ b/executorlib/standalone/__init__.py @@ -4,7 +4,6 @@ to be used independently in other libraries. """ -from executorlib.standalone.cache import get_cache_data from executorlib.standalone.command import get_command_path from executorlib.standalone.interactive.communication import ( SocketInterface, @@ -21,7 +20,6 @@ __all__: list[str] = [ "cancel_items_in_queue", "cloudpickle_register", - "get_cache_data", "get_command_path", "interface_bootup", "interface_connect", diff --git a/notebooks/4-developer.ipynb b/notebooks/4-developer.ipynb index 29653b93..6906a0c4 100644 --- a/notebooks/4-developer.ipynb +++ b/notebooks/4-developer.ipynb @@ -109,7 +109,37 @@ { "id": "7bc073aa-6036-48e7-9696-37af050d438a", "cell_type": "markdown", - "source": "## Communication\nThe key functionality of the `executorlib` package is the up-scaling of python functions with thread based parallelism, MPI based parallelism or by assigning GPUs to individual python functions. In the background this is realized using a combination of the [zero message queue](https://zeromq.org) and [cloudpickle](https://github.com/cloudpipe/cloudpickle) \nto communicate binary python objects. The `executorlib.standalone.interactive.communication.SocketInterface` is an abstraction of this \ninterface, which is used in the other classes inside `executorlib` and might also be helpful for other projects. It comes with a series of utility functions:\n\n* `executorlib.standalone.interactive.communication.interface_bootup()`: To initialize the interface\n* `executorlib.standalone.interactive.communication.interface_connect()`: To connect the interface to another instance\n* `executorlib.standalone.interactive.communication.interface_send()`: To send messages via this interface \n* `executorlib.standalone.interactive.communication.interface_receive()`: To receive messages via this interface \n* `executorlib.standalone.interactive.communication.interface_shutdown()`: To shutdown the interface\n\nWhile `executorlib` was initially designed for up-scaling python functions for HPC, the same functionality can be \nleveraged to up-scale any executable independent of the programming language it is developed in.", + "source": [ + "## Communication\n", + "The key functionality of the executorlib package is the up-scaling of python functions with thread based parallelism, MPI based parallelism or by assigning GPUs to individual python functions. In the background this is realized using a combination of the [zero message queue](https://zeromq.org) and [cloudpickle](https://github.com/cloudpipe/cloudpickle)\n", + "to communicate binary python objects. The `executorlib.standalone.interactive.communication.SocketInterface` is an abstraction of this \n", + "interface, which is used in the other classes inside `executorlib` and might also be helpful for other projects. It comes with a series of utility functions:\n", + "\n", + "* `executorlib.standalone.interactive.communication.interface_bootup()`: To initialize the interface\n", + "* `executorlib.standalone.interactive.communication.interface_connect()`: To connect the interface to another instance\n", + "* `executorlib.standalone.interactive.communication.interface_send()`: To send messages via this interface \n", + "* `executorlib.standalone.interactive.communication.interface_receive()`: To receive messages via this interface \n", + "* `executorlib.standalone.interactive.communication.interface_shutdown()`: To shutdown the interface\n", + "\n", + "While executorlib was initially designed for up-scaling python functions for HPC, the same functionality can be\n", + "leveraged to up-scale any executable independent of the programming language it is developed in.\n", + "\n", + "## External Libraries\n", + "For external libraries executorlib provides a standardized interface for a subset of its internal functionality, which is designed to remain stable with minor version updates. Developers can import the following functionality from `executorlib.standalone`:\n", + "* `cancel_items_in_queue()`\n", + "* `cloudpickle_register()`\n", + "* `get_command_path()`\n", + "* `interface_bootup()`\n", + "* `interface_connect()`\n", + "* `interface_receive()`\n", + "* `interface_send()`\n", + "* `interface_shutdown()`\n", + "* `MpiExecSpawner`\n", + "* `SocketInterface`\n", + "* `SubprocessSpawner`\n", + "\n", + "It is not recommended to import components from other parts of executorlib in other libraries, only the interfaces in `executorlib` and `executorlib.standalone` are designed to be stable. All other classes and functions are considered for internal use only." + ], "metadata": {} }, { From 51a6854218ae5cc756ba33c0e7511ec68d2133ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 26 Apr 2025 12:41:08 +0200 Subject: [PATCH 8/9] add links in readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index d1c475ca..e0e61c27 100644 --- a/README.md +++ b/README.md @@ -158,5 +158,6 @@ as hierarchical job scheduler within the allocations. * [Modules](https://executorlib.readthedocs.io/en/latest/4-developer.html#modules) * [Interface Class Hierarchy](https://executorlib.readthedocs.io/en/latest/4-developer.html#interface-class-hierarchy) * [Communication](https://executorlib.readthedocs.io/en/latest/4-developer.html#communication) + * [External Libraries](https://executorlib.readthedocs.io/en/latest/4-developer.html#external-libraries) * [External Executables](https://executorlib.readthedocs.io/en/latest/4-developer.html#external-executables) * [Interface](https://executorlib.readthedocs.io/en/latest/api.html) From df3f68878c7ef8396ff6fbc4adab552301e1cb35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 26 Apr 2025 13:21:54 +0200 Subject: [PATCH 9/9] add explanation --- notebooks/4-developer.ipynb | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/notebooks/4-developer.ipynb b/notebooks/4-developer.ipynb index 6906a0c4..7747a941 100644 --- a/notebooks/4-developer.ipynb +++ b/notebooks/4-developer.ipynb @@ -126,17 +126,17 @@ "\n", "## External Libraries\n", "For external libraries executorlib provides a standardized interface for a subset of its internal functionality, which is designed to remain stable with minor version updates. Developers can import the following functionality from `executorlib.standalone`:\n", - "* `cancel_items_in_queue()`\n", - "* `cloudpickle_register()`\n", - "* `get_command_path()`\n", - "* `interface_bootup()`\n", - "* `interface_connect()`\n", - "* `interface_receive()`\n", - "* `interface_send()`\n", - "* `interface_shutdown()`\n", - "* `MpiExecSpawner`\n", - "* `SocketInterface`\n", - "* `SubprocessSpawner`\n", + "* `cancel_items_in_queue()` - Cancel items which are still waiting in the Python standard library queue - `queue.queue`.\n", + "* `cloudpickle_register()` - Cloudpickle can either pickle by value or pickle by reference. The functions which are communicated have to be pickled by value rather than by reference, so the module which calls the map function is pickled by value.\n", + "* `get_command_path()` - Get path of the backend executable script `executorlib.backend`.\n", + "* `interface_bootup()` - Start interface for ZMQ communication.\n", + "* `interface_connect()` - Connect to an existing `SocketInterface` instance by providing the hostname and the port as strings.\n", + "* `interface_receive()` - Receive instructions from a `SocketInterface` instance.\n", + "* `interface_send()` - Send results to a `SocketInterface` instance.\n", + "* `interface_shutdown()` - Close the connection to a `SocketInterface` instance.\n", + "* `MpiExecSpawner` - Subprocess interface to start `mpi4py` parallel process.\n", + "* `SocketInterface` - The `SocketInterface` is an abstraction layer on top of the zero message queue.\n", + "* `SubprocessSpawner` - Subprocess interface to start serial Python process.\n", "\n", "It is not recommended to import components from other parts of executorlib in other libraries, only the interfaces in `executorlib` and `executorlib.standalone` are designed to be stable. All other classes and functions are considered for internal use only." ],