From db3bfabc0c1b98378ae4a457e1754e51d0aa9d6e Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 7 Nov 2023 18:32:21 +0100 Subject: [PATCH] Update DocStrings --- pympipool/flux/executor.py | 54 +++++++++++++++++++------------------ pympipool/mpi/executor.py | 51 ++++++++++++++++++----------------- pympipool/slurm/executor.py | 51 ++++++++++++++++++----------------- 3 files changed, 82 insertions(+), 74 deletions(-) diff --git a/pympipool/flux/executor.py b/pympipool/flux/executor.py index 95425118..db604c9e 100644 --- a/pympipool/flux/executor.py +++ b/pympipool/flux/executor.py @@ -14,6 +14,11 @@ class PyFluxExecutor(ExecutorBase): """ + The pympipool.flux.PyFluxExecutor leverages the flux framework to distribute python tasks within a queuing system + allocation. In analogy to the pympipool.slurm.PySlurmExecutur it provides the option to specify the number of + threads per worker as well as the number of GPUs per worker in addition to specifying the number of cores per + worker. + Args: max_workers (int): defines the number workers which can execute functions in parallel cores_per_worker (int): number of MPI cores to be used for each function call @@ -23,6 +28,27 @@ class PyFluxExecutor(ExecutorBase): cwd (str/None): current working directory where the parallel python task is executed sleep_interval (float): synchronization interval - default 0.1 executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux + + Examples: + ``` + >>> import numpy as np + >>> from pympipool.flux import PyFluxExecutor + >>> + >>> def calc(i, j, k): + >>> from mpi4py import MPI + >>> size = MPI.COMM_WORLD.Get_size() + >>> rank = MPI.COMM_WORLD.Get_rank() + >>> return np.array([i, j, k]), size, rank + >>> + >>> def init_k(): + >>> return {"k": 3} + >>> + >>> with PyFluxExecutor(cores=2, init_function=init_k) as p: + >>> fs = p.submit(calc, 2, j=4) + >>> print(fs.result()) + + [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] + ``` """ def __init__( @@ -59,11 +85,7 @@ def __init__( class PyFluxSingleTaskExecutor(ExecutorBase): """ - The pympipool.Executor behaves like the concurrent.futures.Executor but it uses mpi4py to execute parallel tasks. - In contrast to the mpi4py.futures.MPIPoolExecutor the pympipool.Executor can be executed in a serial python process - and does not require the python script to be executed with MPI. Still internally the pympipool.Executor uses the - mpi4py.futures.MPIPoolExecutor, consequently it is primarily an abstraction of its functionality to improve the - usability in particular when used in combination with Jupyter notebooks. + The pympipool.flux.PyFluxSingleTaskExecutor is the internal worker for the pympipool.flux.PyFluxExecutor. Args: cores (int): defines the number of MPI ranks to use for each function call @@ -71,27 +93,7 @@ class PyFluxSingleTaskExecutor(ExecutorBase): gpus_per_task (int): number of GPUs per MPI rank - defaults to 0 init_function (None): optional function to preset arguments for functions which are submitted later cwd (str/None): current working directory where the parallel python task is executed - - Examples: - ``` - >>> import numpy as np - >>> from pympipool.flux.executor import PyFluxSingleTaskExecutor - >>> - >>> def calc(i, j, k): - >>> from mpi4py import MPI - >>> size = MPI.COMM_WORLD.Get_size() - >>> rank = MPI.COMM_WORLD.Get_rank() - >>> return np.array([i, j, k]), size, rank - >>> - >>> def init_k(): - >>> return {"k": 3} - >>> - >>> with PyFluxSingleTaskExecutor(cores=2, init_function=init_k) as p: - >>> fs = p.submit(calc, 2, j=4) - >>> print(fs.result()) - - [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] - ``` + executor (flux.job.FluxExecutor): Flux Python interface to submit the workers to flux """ def __init__( diff --git a/pympipool/mpi/executor.py b/pympipool/mpi/executor.py index 5d137789..1c5fa2e6 100644 --- a/pympipool/mpi/executor.py +++ b/pympipool/mpi/executor.py @@ -10,6 +10,12 @@ class PyMPIExecutor(ExecutorBase): """ + The pympipool.mpi.PyMPIExecutor leverages the message passing interface MPI to distribute python tasks within an + MPI allocation. In contrast to the mpi4py.futures.MPIPoolExecutor the pympipool.mpi.PyMPIExecutor can be executed + in a serial python process and does not require the python script to be executed with MPI. Consequently, it is + primarily an abstraction of its functionality to improve the usability in particular when used in combination with \ + Jupyter notebooks. + Args: max_workers (int): defines the number workers which can execute functions in parallel cores_per_worker (int): number of MPI cores to be used for each function call @@ -17,6 +23,26 @@ class PyMPIExecutor(ExecutorBase): init_function (None): optional function to preset arguments for functions which are submitted later cwd (str/None): current working directory where the parallel python task is executed sleep_interval (float): synchronization interval - default 0.1 + + Examples: + ``` + >>> import numpy as np + >>> from pympipool.mpi import PyMPIExecutor + >>> + >>> def calc(i, j, k): + >>> from mpi4py import MPI + >>> size = MPI.COMM_WORLD.Get_size() + >>> rank = MPI.COMM_WORLD.Get_rank() + >>> return np.array([i, j, k]), size, rank + >>> + >>> def init_k(): + >>> return {"k": 3} + >>> + >>> with PyMPIExecutor(cores=2, init_function=init_k) as p: + >>> fs = p.submit(calc, 2, j=4) + >>> print(fs.result()) + [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] + ``` """ def __init__( @@ -49,11 +75,7 @@ def __init__( class PyMPISingleTaskExecutor(ExecutorBase): """ - The pympipool.Executor behaves like the concurrent.futures.Executor but it uses mpi4py to execute parallel tasks. - In contrast to the mpi4py.futures.MPIPoolExecutor the pympipool.Executor can be executed in a serial python process - and does not require the python script to be executed with MPI. Still internally the pympipool.Executor uses the - mpi4py.futures.MPIPoolExecutor, consequently it is primarily an abstraction of its functionality to improve the - usability in particular when used in combination with Jupyter notebooks. + The pympipool.mpi.PyMPISingleTaskExecutor is the internal worker for the pympipool.mpi.PyMPIExecutor. Args: cores (int): defines the number of MPI ranks to use for each function call @@ -61,25 +83,6 @@ class PyMPISingleTaskExecutor(ExecutorBase): init_function (None): optional function to preset arguments for functions which are submitted later cwd (str/None): current working directory where the parallel python task is executed - Examples: - ``` - >>> import numpy as np - >>> from pympipool.mpi.executor import PyMPISingleTaskExecutor - >>> - >>> def calc(i, j, k): - >>> from mpi4py import MPI - >>> size = MPI.COMM_WORLD.Get_size() - >>> rank = MPI.COMM_WORLD.Get_rank() - >>> return np.array([i, j, k]), size, rank - >>> - >>> def init_k(): - >>> return {"k": 3} - >>> - >>> with PyMPISingleTaskExecutor(cores=2, init_function=init_k) as p: - >>> fs = p.submit(calc, 2, j=4) - >>> print(fs.result()) - [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] - ``` """ def __init__( diff --git a/pympipool/slurm/executor.py b/pympipool/slurm/executor.py index 33688506..f31ad713 100644 --- a/pympipool/slurm/executor.py +++ b/pympipool/slurm/executor.py @@ -18,6 +18,11 @@ class PySlurmExecutor(ExecutorBase): """ + The pympipool.slurm.PySlurmExecutor leverages the srun command to distribute python tasks within a SLURM queuing + system allocation. In analogy to the pympipool.flux.PyFluxExecutor it provides the option to specify the number of + threads per worker as well as the number of GPUs per worker in addition to specifying the number of cores per + worker. + Args: max_workers (int): defines the number workers which can execute functions in parallel cores_per_worker (int): number of MPI cores to be used for each function call @@ -27,6 +32,27 @@ class PySlurmExecutor(ExecutorBase): init_function (None): optional function to preset arguments for functions which are submitted later cwd (str/None): current working directory where the parallel python task is executed sleep_interval (float): synchronization interval - default 0.1 + + Examples: + ``` + >>> import numpy as np + >>> from pympipool.slurm import PySlurmExecutor + >>> + >>> def calc(i, j, k): + >>> from mpi4py import MPI + >>> size = MPI.COMM_WORLD.Get_size() + >>> rank = MPI.COMM_WORLD.Get_rank() + >>> return np.array([i, j, k]), size, rank + >>> + >>> def init_k(): + >>> return {"k": 3} + >>> + >>> with PySlurmExecutor(cores=2, init_function=init_k) as p: + >>> fs = p.submit(calc, 2, j=4) + >>> print(fs.result()) + + [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] + ``` """ def __init__( @@ -63,11 +89,7 @@ def __init__( class PySlurmSingleTaskExecutor(ExecutorBase): """ - The pympipool.Executor behaves like the concurrent.futures.Executor but it uses mpi4py to execute parallel tasks. - In contrast to the mpi4py.futures.MPIPoolExecutor the pympipool.Executor can be executed in a serial python process - and does not require the python script to be executed with MPI. Still internally the pympipool.Executor uses the - mpi4py.futures.MPIPoolExecutor, consequently it is primarily an abstraction of its functionality to improve the - usability in particular when used in combination with Jupyter notebooks. + The pympipool.slurm.PySlurmSingleTaskExecutor is the internal worker for the pympipool.slurm.PySlurmExecutor. Args: cores (int): defines the number of MPI ranks to use for each function call @@ -77,25 +99,6 @@ class PySlurmSingleTaskExecutor(ExecutorBase): init_function (None): optional function to preset arguments for functions which are submitted later cwd (str/None): current working directory where the parallel python task is executed - Examples: - ``` - >>> import numpy as np - >>> from pympipool.mpi.executor import PyMPISingleTaskExecutor - >>> - >>> def calc(i, j, k): - >>> from mpi4py import MPI - >>> size = MPI.COMM_WORLD.Get_size() - >>> rank = MPI.COMM_WORLD.Get_rank() - >>> return np.array([i, j, k]), size, rank - >>> - >>> def init_k(): - >>> return {"k": 3} - >>> - >>> with PyMPISingleTaskExecutor(cores=2, init_function=init_k) as p: - >>> fs = p.submit(calc, 2, j=4) - >>> print(fs.result()) - [(array([2, 4, 3]), 2, 0), (array([2, 4, 3]), 2, 1)] - ``` """ def __init__(