From 2c1574b7d7873476499758413cdea8c63a5ceb4d Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sat, 10 Feb 2024 22:17:15 +0100 Subject: [PATCH 1/3] Introduce launcher modifiers --- reframe/core/launchers/__init__.py | 47 +++++++++++++++++++++++++---- reframe/core/launchers/rsh.py | 8 ++++- unittests/test_launchers.py | 48 ++++++++++++++++++++---------- 3 files changed, 81 insertions(+), 22 deletions(-) diff --git a/reframe/core/launchers/__init__.py b/reframe/core/launchers/__init__.py index 3c1a3362cf..f89be28e4c 100644 --- a/reframe/core/launchers/__init__.py +++ b/reframe/core/launchers/__init__.py @@ -4,12 +4,17 @@ # SPDX-License-Identifier: BSD-3-Clause import abc - import reframe.core.fields as fields import reframe.utility.typecheck as typ +from reframe.core.meta import RegressionTestMeta +from reframe.core.warnings import user_deprecation_warning + +class _JobLauncherMeta(RegressionTestMeta, abc.ABCMeta): + '''Job launcher metaclass.''' -class JobLauncher(abc.ABC): + +class JobLauncher(metaclass=_JobLauncherMeta): '''Abstract base class for job launchers. A job launcher is the executable that actually launches a distributed @@ -30,7 +35,28 @@ class JobLauncher(abc.ABC): #: #: :type: :class:`List[str]` #: :default: ``[]`` - options = fields.TypedField(typ.List[str]) + options = variable(typ.List[str], value=[]) + + #: Optional modifier of the launcher command. + #: + #: This will be combined with the :attr:`modifier_options` and prepended to + #: the parallel launch command. + #: + #: :type: :class:`str` + #: :default: ``''`` + #: + #: .. versionadded:: 4.6.0 + modifier = variable(str, value='') + + #: Options to be passed to the launcher :attr:`modifier`. + #: + #: If the modifier is empty, these options will be ignored. + #: + #: :type: :clas:`List[str]` + #: :default: ``[]`` + #: + #: :versionadded:: 4.6.0 + modifier_options = variable(typ.List[str], value=[]) def __init__(self): self.options = [] @@ -53,7 +79,13 @@ def run_command(self, job): :param job: a job descriptor. :returns: the launcher command as a string. ''' - return ' '.join(self.command(job) + self.options) + cmd_tokens = [] + if self.modifier: + cmd_tokens.append(self.modifier) + cmd_tokens += self.modifier_options + + cmd_tokens += self.command(job) + self.options + return ' '.join(cmd_tokens) class LauncherWrapper(JobLauncher): @@ -90,8 +122,13 @@ def set_launcher(self): ''' - def __init__(self, target_launcher, wrapper_command, wrapper_options=[]): + def __init__(self, target_launcher, wrapper_command, wrapper_options=None): super().__init__() + user_deprecation_warning("'LauncherWrapper is deprected; " + "please use the launcher's 'modifier' and " + "'modifier_options' instead") + + wrapper_options = wrapper_options or [] self.options = target_launcher.options self._target_launcher = target_launcher self._wrapper_command = [wrapper_command] + wrapper_options diff --git a/reframe/core/launchers/rsh.py b/reframe/core/launchers/rsh.py index 0025eac012..909b416de1 100644 --- a/reframe/core/launchers/rsh.py +++ b/reframe/core/launchers/rsh.py @@ -29,5 +29,11 @@ def command(self, job): return ['ssh', '-o BatchMode=yes'] + ssh_opts + [hostname] def run_command(self, job): + cmd_tokens = [] + if self.modifier: + cmd_tokens.append(self.modifier) + cmd_tokens += self.modifier_options + # self.options is processed specially above - return ' '.join(self.command(job)) + cmd_tokens += self.command(job) + return ' '.join(cmd_tokens) diff --git a/unittests/test_launchers.py b/unittests/test_launchers.py index 1ea59d951e..a6b91fbb36 100644 --- a/unittests/test_launchers.py +++ b/unittests/test_launchers.py @@ -8,6 +8,7 @@ import reframe.core.launchers as launchers from reframe.core.backends import getlauncher from reframe.core.schedulers import Job, JobScheduler +from reframe.core.warnings import ReframeDeprecationWarning @pytest.fixture(params=[ @@ -20,9 +21,10 @@ def launcher(request): # convenience for the rest of the unit tests wrapper_cls = launchers.LauncherWrapper wrapper_cls.registered_name = 'launcherwrapper' - return wrapper_cls( - getlauncher('alps')(), 'ddt', ['--offline'] - ) + with pytest.warns(ReframeDeprecationWarning): + return wrapper_cls( + getlauncher('alps')(), 'ddt', ['--offline'] + ) return getlauncher(request.param)() @@ -154,38 +156,52 @@ def test_run_command(job): assert command == 'lrun -N 2 -T 2 -M "-gpu" --foo' -def test_run_command_minimal(minimal_job): +@pytest.fixture(params=['modifiers', 'plain']) +def use_modifiers(request): + return request.param == 'modifiers' + + +def test_run_command_minimal(minimal_job, use_modifiers): launcher_name = type(minimal_job.launcher).registered_name # This is relevant only for the srun launcher, because it may # run in different platforms with older versions of Slurm minimal_job.launcher.use_cpus_per_task = True + if use_modifiers and launcher_name != 'launcherwrapper': + minimal_job.launcher.modifier = 'ddt' + minimal_job.launcher.modifier_options = ['--offline'] + prefix = 'ddt --offline' + if launcher_name != 'local': + prefix += ' ' + else: + prefix = '' + command = minimal_job.launcher.run_command(minimal_job) if launcher_name == 'alps': - assert command == 'aprun -n 1 --foo' + assert command == f'{prefix}aprun -n 1 --foo' elif launcher_name == 'launcherwrapper': assert command == 'ddt --offline aprun -n 1 --foo' elif launcher_name == 'local': - assert command == '' + assert command == f'{prefix}' elif launcher_name == 'mpiexec': - assert command == 'mpiexec -n 1 --foo' + assert command == f'{prefix}mpiexec -n 1 --foo' elif launcher_name == 'mpirun': - assert command == 'mpirun -np 1 --foo' + assert command == f'{prefix}mpirun -np 1 --foo' elif launcher_name == 'srun': - assert command == 'srun --foo' + assert command == f'{prefix}srun --foo' elif launcher_name == 'srunalloc': - assert command == ('srun ' + assert command == (f'{prefix}srun ' '--job-name=fake_job ' '--ntasks=1 ' '--foo') elif launcher_name == 'ssh': - assert command == 'ssh -o BatchMode=yes --foo host' + assert command == f'{prefix}ssh -o BatchMode=yes --foo host' elif launcher_name in ('clush', 'pdsh'): - assert command == f'{launcher_name} -w host --foo' + assert command == f'{prefix}{launcher_name} -w host --foo' elif launcher_name == 'upcrun': - assert command == 'upcrun -n 1 --foo' + assert command == f'{prefix}upcrun -n 1 --foo' elif launcher_name == 'upcxx-run': - assert command == 'upcxx-run -n 1 --foo' + assert command == f'{prefix}upcxx-run -n 1 --foo' elif launcher_name == 'lrun': - assert command == 'lrun -N 1 -T 1 --foo' + assert command == f'{prefix}lrun -N 1 -T 1 --foo' elif launcher_name == 'lrun-gpu': - assert command == 'lrun -N 1 -T 1 -M "-gpu" --foo' + assert command == f'{prefix}lrun -N 1 -T 1 -M "-gpu" --foo' From 75559ceef807081cb9c3c7f75569fee9718b576d Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sun, 11 Feb 2024 20:45:38 +0100 Subject: [PATCH 2/3] Remove unused imports --- reframe/core/launchers/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/reframe/core/launchers/__init__.py b/reframe/core/launchers/__init__.py index f89be28e4c..8fcac175da 100644 --- a/reframe/core/launchers/__init__.py +++ b/reframe/core/launchers/__init__.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: BSD-3-Clause import abc -import reframe.core.fields as fields import reframe.utility.typecheck as typ from reframe.core.meta import RegressionTestMeta from reframe.core.warnings import user_deprecation_warning From 9eb84092ca4bcb41b6b1ccc6a3c08fe9dcee68e4 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 15 Feb 2024 22:46:02 +0100 Subject: [PATCH 3/3] Apply suggestions from code review Co-authored-by: Theofilos Manitaras --- reframe/core/launchers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reframe/core/launchers/__init__.py b/reframe/core/launchers/__init__.py index 8fcac175da..ff45a22c96 100644 --- a/reframe/core/launchers/__init__.py +++ b/reframe/core/launchers/__init__.py @@ -123,7 +123,7 @@ def set_launcher(self): def __init__(self, target_launcher, wrapper_command, wrapper_options=None): super().__init__() - user_deprecation_warning("'LauncherWrapper is deprected; " + user_deprecation_warning("'LauncherWrapper is deprecated; " "please use the launcher's 'modifier' and " "'modifier_options' instead")