diff --git a/pympipool/__init__.py b/pympipool/__init__.py index f6e0ee3d..7c42f113 100644 --- a/pympipool/__init__.py +++ b/pympipool/__init__.py @@ -19,6 +19,9 @@ class Executor: an interactive Jupyter notebook. Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the number of + cores which can be used in parallel - just like the max_cores parameter. Using max_cores is + recommended, as computers have a limited number of compute cores. max_cores (int): defines the number cores which can be used in parallel cores_per_worker (int): number of MPI cores to be used for each function call threads_per_core (int): number of OpenMP threads to be used for each function call @@ -64,6 +67,7 @@ class Executor: def __init__( self, + max_workers: int = 1, max_cores: int = 1, cores_per_worker: int = 1, threads_per_core: int = 1, @@ -84,6 +88,7 @@ def __init__( def __new__( cls, + max_workers: int = 1, max_cores: int = 1, cores_per_worker: int = 1, threads_per_core: int = 1, @@ -108,6 +113,9 @@ def __new__( requires the SLURM workload manager to be installed on the system. Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the + number of cores which can be used in parallel - just like the max_cores parameter. Using + max_cores is recommended, as computers have a limited number of compute cores. max_cores (int): defines the number cores which can be used in parallel cores_per_worker (int): number of MPI cores to be used for each function call threads_per_core (int): number of OpenMP threads to be used for each function call @@ -135,6 +143,7 @@ def __new__( """ if not disable_dependencies: return ExecutorWithDependencies( + max_workers=max_workers, max_cores=max_cores, cores_per_worker=cores_per_worker, threads_per_core=threads_per_core, @@ -152,6 +161,7 @@ def __new__( else: _check_refresh_rate(refresh_rate=refresh_rate) return create_executor( + max_workers=max_workers, max_cores=max_cores, cores_per_worker=cores_per_worker, threads_per_core=threads_per_core, diff --git a/pympipool/scheduler/__init__.py b/pympipool/scheduler/__init__.py index ff8d2f00..3138bc2f 100644 --- a/pympipool/scheduler/__init__.py +++ b/pympipool/scheduler/__init__.py @@ -14,6 +14,7 @@ check_executor, check_backend, check_init_function, + validate_number_of_cores, ) from pympipool.scheduler.slurm import ( PySlurmExecutor, @@ -36,6 +37,7 @@ def create_executor( + max_workers: int = 1, max_cores: int = 1, cores_per_worker: int = 1, threads_per_core: int = 1, @@ -58,6 +60,9 @@ def create_executor( requires the SLURM workload manager to be installed on the system. Args: + max_workers (int): for backwards compatibility with the standard library, max_workers also defines the number of + cores which can be used in parallel - just like the max_cores parameter. Using max_cores is + recommended, as computers have a limited number of compute cores. max_cores (int): defines the number cores which can be used in parallel cores_per_worker (int): number of MPI cores to be used for each function call threads_per_core (int): number of OpenMP threads to be used for each function call @@ -65,12 +70,11 @@ def create_executor( oversubscribe (bool): adds the `--oversubscribe` command line flag (OpenMPI and SLURM only) - default False cwd (str/None): current working directory where the parallel python task is executed hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the - context of an HPC cluster this essential to be able to communicate to an - Executor running on a different compute node within the same allocation. And - in principle any computer should be able to resolve that their own hostname - points to the same address as localhost. Still MacOS >= 12 seems to disable - this look up for security reasons. So on MacOS it is required to set this - option to true + context of an HPC cluster this essential to be able to communicate to an Executor + running on a different compute node within the same allocation. And in principle + any computer should be able to resolve that their own hostname points to the same + address as localhost. Still MacOS >= 12 seems to disable this look up for security + reasons. So on MacOS it is required to set this option to true backend (str): Switch between the different backends "flux", "mpi" or "slurm". Alternatively, when "auto" is selected (the default) the available backend is determined automatically. block_allocation (boolean): To accelerate the submission of a series of python functions with the same @@ -81,6 +85,7 @@ def create_executor( command_line_argument_lst (list): Additional command line arguments for the srun call (SLURM only) """ + max_cores = validate_number_of_cores(max_cores=max_cores, max_workers=max_workers) check_init_function(block_allocation=block_allocation, init_function=init_function) check_backend(backend=backend) if backend == "flux" or (backend == "auto" and flux_installed): diff --git a/pympipool/shared/inputcheck.py b/pympipool/shared/inputcheck.py index 69e6f87e..fded2483 100644 --- a/pympipool/shared/inputcheck.py +++ b/pympipool/shared/inputcheck.py @@ -78,3 +78,10 @@ def check_backend(backend): def check_init_function(block_allocation, init_function): if not block_allocation and init_function is not None: raise ValueError("") + + +def validate_number_of_cores(max_cores, max_workers): + # only overwrite max_cores when it is set to 1 + if max_workers != 1 and max_cores == 1: + return max_workers + return max_cores diff --git a/tests/test_executor_backend_mpi.py b/tests/test_executor_backend_mpi.py index 27798b5c..2cb3b876 100644 --- a/tests/test_executor_backend_mpi.py +++ b/tests/test_executor_backend_mpi.py @@ -43,7 +43,7 @@ def test_meta_executor_single(self): def test_meta_executor_parallel(self): with Executor( - max_cores=2, + max_workers=2, cores_per_worker=2, hostname_localhost=True, backend="mpi",