From 12846e4f905b49b582f3dccf9b711d8b0c4f2219 Mon Sep 17 00:00:00 2001 From: rafael Date: Thu, 21 Jan 2021 16:56:28 +0100 Subject: [PATCH 1/2] differentiate cli and test job options --- reframe/core/launchers/mpi.py | 2 +- reframe/core/schedulers/__init__.py | 3 ++- reframe/core/schedulers/pbs.py | 2 +- reframe/core/schedulers/slurm.py | 14 +++++++++----- unittests/test_launchers.py | 4 ++-- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/reframe/core/launchers/mpi.py b/reframe/core/launchers/mpi.py index 24d2ed88a6..ca4f007954 100644 --- a/reframe/core/launchers/mpi.py +++ b/reframe/core/launchers/mpi.py @@ -117,7 +117,7 @@ def command(self, job): hint = 'multithread' if job.use_smt else 'nomultithread' ret += ['--hint=%s' % hint] - for opt in job.options: + for opt in job.options + job.sched_options: if opt.startswith('#'): continue diff --git a/reframe/core/schedulers/__init__.py b/reframe/core/schedulers/__init__.py index f236632111..2a1e5eb57b 100644 --- a/reframe/core/schedulers/__init__.py +++ b/reframe/core/schedulers/__init__.py @@ -180,7 +180,8 @@ def __init__(self, self.num_cpus_per_task = None self.use_smt = None self.time_limit = None - self.options = list(sched_options) if sched_options else [] + self.sched_options = list(sched_options) if sched_options else [] + self.options = [] self._name = name self._workdir = workdir diff --git a/reframe/core/schedulers/pbs.py b/reframe/core/schedulers/pbs.py index 566d9e4a8a..f4619a32b8 100644 --- a/reframe/core/schedulers/pbs.py +++ b/reframe/core/schedulers/pbs.py @@ -91,7 +91,7 @@ def _emit_lselect_option(self, job): # Options starting with `-` are emitted in separate lines rem_opts = [] verb_opts = [] - for opt in (*job.sched_access, *job.options): + for opt in (*job.sched_access, *job.options, *job.sched_options): if opt.startswith('-'): rem_opts.append(opt) elif opt.startswith('#'): diff --git a/reframe/core/schedulers/slurm.py b/reframe/core/schedulers/slurm.py index 67be4a77eb..cd9d521a8b 100644 --- a/reframe/core/schedulers/slurm.py +++ b/reframe/core/schedulers/slurm.py @@ -149,10 +149,12 @@ def emit_preamble(self, job): ] # Determine if job refers to a Slurm job array, by looking into the - # job.options + # job.options and job.sched_options jobarr_parser = ArgumentParser() jobarr_parser.add_argument('-a', '--array') - parsed_args, _ = jobarr_parser.parse_known_args(job.options) + parsed_args, _ = jobarr_parser.parse_known_args( + job.options + job.sched_options + ) if parsed_args.array: job._is_array = True self.log('Slurm job is a job array') @@ -197,7 +199,9 @@ def emit_preamble(self, job): # NOTE: Here last of the passed --constraint job options is taken # into account in order to respect the behavior of slurm. - parsed_options, _ = constraint_parser.parse_known_args(job.options) + parsed_options, _ = constraint_parser.parse_known_args( + job.options + job.sched_options + ) if parsed_options.constraint: constraints.append(parsed_options.constraint.strip()) @@ -208,7 +212,7 @@ def emit_preamble(self, job): preamble.append(self._format_option(hint, '--hint={0}')) prefix_patt = re.compile(r'(#\w+)') - for opt in job.options: + for opt in job.options + job.sched_options: if opt.strip().startswith(('-C', '--constraint')): # Constraints are already processed continue @@ -267,7 +271,7 @@ def filternodes(self, job, nodes): # Collect options that restrict node selection, but we need to first # create a mutable list out of the immutable SequenceView that # sched_access is - options = list(job.sched_access + job.options) + options = list(job.sched_access + job.options + job.sched_options) option_parser = ArgumentParser() option_parser.add_argument('--reservation') option_parser.add_argument('-p', '--partition') diff --git a/unittests/test_launchers.py b/unittests/test_launchers.py index 4777f05661..4f09d2594e 100644 --- a/unittests/test_launchers.py +++ b/unittests/test_launchers.py @@ -85,7 +85,7 @@ def job(make_job, launcher): job.num_cpus_per_task = 2 job.use_smt = True job.time_limit = '10m' - job.options += ['--gres=gpu:4', '#DW jobdw anything'] + job.options = ['--gres=gpu:4', '#DW jobdw anything'] job.launcher.options = ['--foo'] return job @@ -130,8 +130,8 @@ def test_run_command(job): '--cpus-per-task=2 ' '--exclusive ' '--hint=multithread ' - '--fake ' '--gres=gpu:4 ' + '--fake ' '--foo') elif launcher_name == 'ssh': assert command == 'ssh -o BatchMode=yes -l user -p 22222 --foo host' From d8d41ca67b7cc91d5796c14377b622d28c34adb2 Mon Sep 17 00:00:00 2001 From: rafael Date: Thu, 21 Jan 2021 18:24:15 +0100 Subject: [PATCH 2/2] change variable name --- reframe/core/launchers/mpi.py | 2 +- reframe/core/schedulers/__init__.py | 2 +- reframe/core/schedulers/pbs.py | 2 +- reframe/core/schedulers/slurm.py | 10 +++++----- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/reframe/core/launchers/mpi.py b/reframe/core/launchers/mpi.py index ca4f007954..1364b02afc 100644 --- a/reframe/core/launchers/mpi.py +++ b/reframe/core/launchers/mpi.py @@ -117,7 +117,7 @@ def command(self, job): hint = 'multithread' if job.use_smt else 'nomultithread' ret += ['--hint=%s' % hint] - for opt in job.options + job.sched_options: + for opt in job.options + job.cli_options: if opt.startswith('#'): continue diff --git a/reframe/core/schedulers/__init__.py b/reframe/core/schedulers/__init__.py index 2a1e5eb57b..d4872a541e 100644 --- a/reframe/core/schedulers/__init__.py +++ b/reframe/core/schedulers/__init__.py @@ -180,7 +180,7 @@ def __init__(self, self.num_cpus_per_task = None self.use_smt = None self.time_limit = None - self.sched_options = list(sched_options) if sched_options else [] + self.cli_options = list(sched_options) if sched_options else [] self.options = [] self._name = name diff --git a/reframe/core/schedulers/pbs.py b/reframe/core/schedulers/pbs.py index f4619a32b8..03c980404e 100644 --- a/reframe/core/schedulers/pbs.py +++ b/reframe/core/schedulers/pbs.py @@ -91,7 +91,7 @@ def _emit_lselect_option(self, job): # Options starting with `-` are emitted in separate lines rem_opts = [] verb_opts = [] - for opt in (*job.sched_access, *job.options, *job.sched_options): + for opt in (*job.sched_access, *job.options, *job.cli_options): if opt.startswith('-'): rem_opts.append(opt) elif opt.startswith('#'): diff --git a/reframe/core/schedulers/slurm.py b/reframe/core/schedulers/slurm.py index cd9d521a8b..90c4b9ba21 100644 --- a/reframe/core/schedulers/slurm.py +++ b/reframe/core/schedulers/slurm.py @@ -149,11 +149,11 @@ def emit_preamble(self, job): ] # Determine if job refers to a Slurm job array, by looking into the - # job.options and job.sched_options + # job.options and job.cli_options jobarr_parser = ArgumentParser() jobarr_parser.add_argument('-a', '--array') parsed_args, _ = jobarr_parser.parse_known_args( - job.options + job.sched_options + job.options + job.cli_options ) if parsed_args.array: job._is_array = True @@ -200,7 +200,7 @@ def emit_preamble(self, job): # NOTE: Here last of the passed --constraint job options is taken # into account in order to respect the behavior of slurm. parsed_options, _ = constraint_parser.parse_known_args( - job.options + job.sched_options + job.options + job.cli_options ) if parsed_options.constraint: constraints.append(parsed_options.constraint.strip()) @@ -212,7 +212,7 @@ def emit_preamble(self, job): preamble.append(self._format_option(hint, '--hint={0}')) prefix_patt = re.compile(r'(#\w+)') - for opt in job.options + job.sched_options: + for opt in job.options + job.cli_options: if opt.strip().startswith(('-C', '--constraint')): # Constraints are already processed continue @@ -271,7 +271,7 @@ def filternodes(self, job, nodes): # Collect options that restrict node selection, but we need to first # create a mutable list out of the immutable SequenceView that # sched_access is - options = list(job.sched_access + job.options + job.sched_options) + options = list(job.sched_access + job.options + job.cli_options) option_parser = ArgumentParser() option_parser.add_argument('--reservation') option_parser.add_argument('-p', '--partition')