From c0d9e103d8955a79d4883abf13575ddf05b143d6 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 7 Nov 2023 16:25:18 +0100 Subject: [PATCH 1/4] Implement universal executor class --- pympipool/__init__.py | 45 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/pympipool/__init__.py b/pympipool/__init__.py index e344cbe0..6979129a 100644 --- a/pympipool/__init__.py +++ b/pympipool/__init__.py @@ -4,8 +4,53 @@ try: # The PyFluxExecutor requires flux-core to be installed. from pympipool.flux.executor import PyFluxExecutor + flux_installed = True except ImportError: + flux_installed = False pass +try: # The PySlurmExecutor requires the srun command to be available. + from pympipool.slurm.executor import PySlurmExecutor + slurm_installed = True +except ImportError: + slurm_installed = False + pass + + __version__ = get_versions()["version"] del get_versions + + +class Executor: + def __new__( + cls, + max_workers, + cores_per_worker=1, + init_function=None, + cwd=None, + sleep_interval=0.1 + ): + if flux_installed: + return PyFluxExecutor( + max_workers=max_workers, + cores_per_worker=cores_per_worker, + init_function=init_function, + cwd=cwd, + sleep_interval=sleep_interval, + ) + elif slurm_installed: + return PySlurmExecutor( + max_workers=max_workers, + cores_per_worker=cores_per_worker, + init_function=init_function, + cwd=cwd, + sleep_interval=sleep_interval, + ) + else: + return PyMPIExecutor( + max_workers=max_workers, + cores_per_worker=cores_per_worker, + init_function=init_function, + cwd=cwd, + sleep_interval=sleep_interval, + ) From f8b2846b2e5882c0fa19f696e4c24e127fe26ac0 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 7 Nov 2023 16:26:21 +0100 Subject: [PATCH 2/4] black formatting --- pympipool/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pympipool/__init__.py b/pympipool/__init__.py index fa95fae3..46c2ed85 100644 --- a/pympipool/__init__.py +++ b/pympipool/__init__.py @@ -3,6 +3,7 @@ try: # The PyFluxExecutor requires flux-core to be installed. from pympipool.flux.executor import PyFluxExecutor + flux_installed = True except ImportError: flux_installed = False @@ -10,6 +11,7 @@ try: # The PySlurmExecutor requires the srun command to be available. from pympipool.slurm.executor import PySlurmExecutor + slurm_installed = True except ImportError: slurm_installed = False @@ -27,7 +29,7 @@ def __new__( cores_per_worker=1, init_function=None, cwd=None, - sleep_interval=0.1 + sleep_interval=0.1, ): if flux_installed: return PyFluxExecutor( From 64e8c392d332eb96abec8f8d108c9e23eccf755b Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 7 Nov 2023 16:32:59 +0100 Subject: [PATCH 3/4] only enable flux when flux was started --- pympipool/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pympipool/__init__.py b/pympipool/__init__.py index 46c2ed85..3fdd9a5e 100644 --- a/pympipool/__init__.py +++ b/pympipool/__init__.py @@ -1,10 +1,11 @@ +import os from ._version import get_versions from pympipool.mpi.executor import PyMPIExecutor try: # The PyFluxExecutor requires flux-core to be installed. from pympipool.flux.executor import PyFluxExecutor - flux_installed = True + flux_installed = "FLUX_URI" in os.environ except ImportError: flux_installed = False pass From e401b414560efacf077575b2d2744d4cfa57be6a Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 7 Nov 2023 19:16:03 +0100 Subject: [PATCH 4/4] Update pympipool/__init__.py --- pympipool/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pympipool/__init__.py b/pympipool/__init__.py index 3fdd9a5e..77561022 100644 --- a/pympipool/__init__.py +++ b/pympipool/__init__.py @@ -26,7 +26,7 @@ class Executor: def __new__( cls, - max_workers, + max_workers=1, cores_per_worker=1, init_function=None, cwd=None,