From 0e99cd7ac9222491333db9caf0e8a9de22834f00 Mon Sep 17 00:00:00 2001 From: rafael Date: Wed, 10 Jun 2020 15:59:36 +0200 Subject: [PATCH 1/2] remove sched_* variables --- reframe/core/launchers/mpi.py | 15 -------- reframe/core/schedulers/__init__.py | 30 ---------------- reframe/core/schedulers/pbs.py | 4 --- reframe/core/schedulers/slurm.py | 19 ---------- reframe/frontend/cli.py | 11 +++--- reframe/frontend/executors/__init__.py | 5 --- reframe/frontend/executors/policies.py | 10 ------ unittests/test_launchers.py | 10 ------ unittests/test_schedulers.py | 48 +++----------------------- 9 files changed, 9 insertions(+), 143 deletions(-) diff --git a/reframe/core/launchers/mpi.py b/reframe/core/launchers/mpi.py index 082e87ce98..798d376422 100644 --- a/reframe/core/launchers/mpi.py +++ b/reframe/core/launchers/mpi.py @@ -110,9 +110,6 @@ def command(self, job): if job.num_cpus_per_task: ret += ['--cpus-per-task=%s' % str(job.num_cpus_per_task)] - if job.sched_partition: - ret += ['--partition=%s' % str(job.sched_partition)] - if job.sched_exclusive_access: ret += ['--exclusive'] @@ -120,18 +117,6 @@ def command(self, job): hint = 'multithread' if job.use_smt else 'nomultithread' ret += ['--hint=%s' % hint] - if job.sched_partition: - ret += ['--partition=%s' % str(job.sched_partition)] - - if job.sched_account: - ret += ['--account=%s' % str(job.sched_account)] - - if job.sched_nodelist: - ret += ['--nodelist=%s' % str(job.sched_nodelist)] - - if job.sched_exclude_nodelist: - ret += ['--exclude=%s' % str(job.sched_exclude_nodelist)] - for opt in job.options: if opt.startswith('#'): continue diff --git a/reframe/core/schedulers/__init__.py b/reframe/core/schedulers/__init__.py index 312d1f6267..0a437f5494 100644 --- a/reframe/core/schedulers/__init__.py +++ b/reframe/core/schedulers/__init__.py @@ -200,11 +200,6 @@ def __init__(self, max_pending_time=None, sched_flex_alloc_nodes=None, sched_access=[], - sched_account=None, - sched_partition=None, - sched_reservation=None, - sched_nodelist=None, - sched_exclude_nodelist=None, sched_exclusive_access=None, sched_options=None): @@ -236,11 +231,6 @@ def __init__(self, # Backend scheduler related information self._sched_flex_alloc_nodes = sched_flex_alloc_nodes self._sched_access = sched_access - self._sched_nodelist = sched_nodelist - self._sched_exclude_nodelist = sched_exclude_nodelist - self._sched_partition = sched_partition - self._sched_reservation = sched_reservation - self._sched_account = sched_account self._sched_exclusive_access = sched_exclusive_access @classmethod @@ -281,26 +271,6 @@ def sched_flex_alloc_nodes(self): def sched_access(self): return self._sched_access - @property - def sched_nodelist(self): - return self._sched_nodelist - - @property - def sched_exclude_nodelist(self): - return self._sched_exclude_nodelist - - @property - def sched_partition(self): - return self._sched_partition - - @property - def sched_reservation(self): - return self._sched_reservation - - @property - def sched_account(self): - return self._sched_account - @property def sched_exclusive_access(self): return self._sched_exclusive_access diff --git a/reframe/core/schedulers/pbs.py b/reframe/core/schedulers/pbs.py index c5eb2dc5bb..db174bdbb6 100644 --- a/reframe/core/schedulers/pbs.py +++ b/reframe/core/schedulers/pbs.py @@ -100,10 +100,6 @@ def emit_preamble(self, job): preamble.append( self._format_option('-l walltime=%d:%d:%d' % (h, m, s))) - if job.sched_partition: - preamble.append( - self._format_option('-q %s' % job.sched_partition)) - preamble += self._emit_lselect_option(job) # PBS starts the job in the home directory by default diff --git a/reframe/core/schedulers/slurm.py b/reframe/core/schedulers/slurm.py index 55a910d3c0..1a5f30efba 100644 --- a/reframe/core/schedulers/slurm.py +++ b/reframe/core/schedulers/slurm.py @@ -161,11 +161,6 @@ def emit_preamble(self, job): self._format_option(job.num_tasks_per_socket, '--ntasks-per-socket={0}'), self._format_option(job.num_cpus_per_task, '--cpus-per-task={0}'), - self._format_option(job.sched_partition, '--partition={0}'), - self._format_option(job.sched_account, '--account={0}'), - self._format_option(job.sched_nodelist, '--nodelist={0}'), - self._format_option(job.sched_exclude_nodelist, '--exclude={0}'), - self._format_option(job.sched_reservation, '--reservation={0}') ] # Slurm replaces '%a' by the corresponding SLURM_ARRAY_TASK_ID @@ -254,20 +249,6 @@ def filternodes(self, job, nodes): # create a mutable list out of the immutable SequenceView that # sched_access is options = list(job.sched_access + job.options) - if job.sched_partition: - options.append('--partition=%s' % job.sched_partition) - - if job.sched_account: - options.append('--account=%s' % job.sched_account) - - if job.sched_nodelist: - options.append('--nodelist=%s' % job.sched_nodelist) - - if job.sched_exclude_nodelist: - options.append('--exclude=%s' % job.sched_exclude_nodelist) - - if job.sched_reservation: - options.append('--reservation=%s' % job.sched_reservation) option_parser = ArgumentParser() option_parser.add_argument('--reservation') diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index f84615c93f..ee7459746a 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -595,11 +595,13 @@ def print_infoline(param, value): options.flex_alloc_nodes = options.flex_alloc_nodes or 'idle' if options.account: + options.job_options.append(f'account={options.account}') printer.warning(f"`--account' is deprecated and " f"will be removed in the future; you should " f"use `-J account={options.account}'") if options.partition: + options.job_options.append(f'partition={options.partition}') printer.warning(f"`--partition' is deprecated and " f"will be removed in the future; you should " f"use `-J partition={options.partition}' " @@ -607,16 +609,19 @@ def print_infoline(param, value): f"scheduler") if options.reservation: + options.job_options.append(f'reservation={options.reservation}') printer.warning(f"`--reservation' is deprecated and " f"will be removed in the future; you should " f"use `-J reservation={options.reservation}'") if options.nodelist: + options.job_options.append(f'nodelist={options.nodelist}') printer.warning(f"`--nodelist' is deprecated and " f"will be removed in the future; you should " f"use `-J nodelist={options.nodelist}'") if options.exclude_nodes: + options.job_options.append(f'exclude={options.exclude_nodes}') printer.warning(f"`--exclude-nodes' is deprecated and " f"will be removed in the future; you should " f"use `-J exclude={options.exclude_nodes}'") @@ -663,12 +668,6 @@ def print_infoline(param, value): sched_flex_alloc_nodes = options.flex_alloc_nodes exec_policy.sched_flex_alloc_nodes = sched_flex_alloc_nodes - exec_policy.flex_alloc_nodes = options.flex_alloc_nodes - exec_policy.sched_account = options.account - exec_policy.sched_partition = options.partition - exec_policy.sched_reservation = options.reservation - exec_policy.sched_nodelist = options.nodelist - exec_policy.sched_exclude_nodelist = options.exclude_nodes parsed_job_options = [] for opt in options.job_options: if opt.startswith('-') or opt.startswith('#'): diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py index 022bfd54f0..b589b6f34c 100644 --- a/reframe/frontend/executors/__init__.py +++ b/reframe/frontend/executors/__init__.py @@ -454,11 +454,6 @@ def __init__(self): # Scheduler options self.sched_flex_alloc_nodes = None - self.sched_account = None - self.sched_partition = None - self.sched_reservation = None - self.sched_nodelist = None - self.sched_exclude_nodelist = None self.sched_options = [] # Task event listeners diff --git a/reframe/frontend/executors/policies.py b/reframe/frontend/executors/policies.py index 83240978bd..405fa7a9fc 100644 --- a/reframe/frontend/executors/policies.py +++ b/reframe/frontend/executors/policies.py @@ -56,11 +56,6 @@ def runcase(self, case): task.setup(partition, environ, sched_flex_alloc_nodes=self.sched_flex_alloc_nodes, - sched_account=self.sched_account, - sched_partition=self.sched_partition, - sched_reservation=self.sched_reservation, - sched_nodelist=self.sched_nodelist, - sched_exclude_nodelist=self.sched_exclude_nodelist, sched_options=self.sched_options) task.compile() @@ -257,11 +252,6 @@ def _setup_task(self, task): task.setup(task.testcase.partition, task.testcase.environ, sched_flex_alloc_nodes=self.sched_flex_alloc_nodes, - sched_account=self.sched_account, - sched_partition=self.sched_partition, - sched_reservation=self.sched_reservation, - sched_nodelist=self.sched_nodelist, - sched_exclude_nodelist=self.sched_exclude_nodelist, sched_options=self.sched_options) except TaskExit: return False diff --git a/unittests/test_launchers.py b/unittests/test_launchers.py index 90554fffe9..c9fa6f4a74 100644 --- a/unittests/test_launchers.py +++ b/unittests/test_launchers.py @@ -48,11 +48,6 @@ def setUp(self): script_filename='fake_script', stdout='fake_stdout', stderr='fake_stderr', - sched_account='fake_account', - sched_partition='fake_partition', - sched_reservation='fake_reservation', - sched_nodelist="mynode", - sched_exclude_nodelist='fake_exclude_nodelist', sched_exclusive_access='fake_exclude_access', sched_options=['--fake']) self.job.num_tasks = 4 @@ -128,13 +123,8 @@ def expected_command(self): '--ntasks-per-core=1 ' '--ntasks-per-socket=1 ' '--cpus-per-task=2 ' - '--partition=fake_partition ' '--exclusive ' '--hint=multithread ' - '--partition=fake_partition ' - '--account=fake_account ' - '--nodelist=mynode ' - '--exclude=fake_exclude_nodelist ' '--fake ' '--gres=gpu:4 ' '--foo') diff --git a/unittests/test_schedulers.py b/unittests/test_schedulers.py index ecf02555f6..a48798c809 100644 --- a/unittests/test_schedulers.py +++ b/unittests/test_schedulers.py @@ -105,11 +105,6 @@ def minimal_job(make_job): @pytest.fixture def fake_job(make_job): ret = make_job( - sched_nodelist='nid000[00-17]', - sched_exclude_nodelist='nid00016', - sched_partition='foo', - sched_reservation='bar', - sched_account='spam', sched_exclusive_access=True ) ret.time_limit = '5m' @@ -160,11 +155,6 @@ def _expected_slurm_directives(job): '#SBATCH --ntasks-per-socket=%s' % job.num_tasks_per_socket, '#SBATCH --cpus-per-task=%s' % job.num_cpus_per_task, '#SBATCH --hint=multithread', - '#SBATCH --nodelist=%s' % job.sched_nodelist, - '#SBATCH --exclude=%s' % job.sched_exclude_nodelist, - '#SBATCH --partition=%s' % job.sched_partition, - '#SBATCH --reservation=%s' % job.sched_reservation, - '#SBATCH --account=%s' % job.sched_account, '#SBATCH --exclusive', # Custom options and directives '#SBATCH --gres=gpu:4', @@ -188,7 +178,7 @@ def _expected_pbs_directives(job): ':mem=100GB:cpu_type=haswell' % (num_nodes, job.num_tasks_per_node, num_cpus_per_node), - '#PBS -q %s' % job.sched_partition, + # '#PBS -q %s' % job.sched_partition, '#PBS --gres=gpu:4', '#DW jobdw capacity=100GB', '#DW stage_in source=/foo' @@ -205,7 +195,6 @@ def _expected_torque_directives(job): '#PBS -e %s' % job.stderr, '#PBS -l nodes=%s:ppn=%s:haswell' % (num_nodes, num_cpus_per_node), '#PBS -l mem=100GB', - '#PBS -q %s' % job.sched_partition, '#PBS --gres=gpu:4', '#DW jobdw capacity=100GB', '#DW stage_in source=/foo' @@ -681,8 +670,7 @@ def test_flex_alloc_sched_access_idle_sequence_view(make_flexible_job): from reframe.utility import SequenceView job = make_flexible_job('idle', - sched_access=SequenceView(['--constraint=f3']), - sched_partition='p3') + sched_access=SequenceView(['--constraint=f3'])) prepare_job(job) assert job.num_tasks == 4 @@ -715,7 +703,8 @@ def test_flex_alloc_constraint_idle(make_flexible_job): def test_flex_alloc_partition_idle(make_flexible_job): - job = make_flexible_job('idle', sched_partition='p2') + job = make_flexible_job('idle') + job.options = ['--partition=p2'] with pytest.raises(JobError): prepare_job(job) @@ -734,12 +723,6 @@ def test_flex_alloc_valid_multiple_constraints(make_flexible_job): assert job.num_tasks == 4 -def test_flex_alloc_valid_partition_cmd(make_flexible_job): - job = make_flexible_job('all', sched_partition='p2') - prepare_job(job) - assert job.num_tasks == 8 - - def test_flex_alloc_valid_partition_opt(make_flexible_job): job = make_flexible_job('all') job.options = ['-p p2'] @@ -761,12 +744,6 @@ def test_flex_alloc_valid_constraint_partition(make_flexible_job): assert job.num_tasks == 4 -def test_flex_alloc_invalid_partition_cmd(make_flexible_job): - job = make_flexible_job('all', sched_partition='invalid') - with pytest.raises(JobError): - prepare_job(job) - - def test_flex_alloc_invalid_partition_opt(make_flexible_job): job = make_flexible_job('all') job.options = ['--partition=invalid'] @@ -781,15 +758,6 @@ def test_flex_alloc_invalid_constraint(make_flexible_job): prepare_job(job) -def test_flex_alloc_valid_reservation_cmd(make_flexible_job): - job = make_flexible_job('all', - sched_access=['--constraint=f2'], - sched_reservation='dummy') - - prepare_job(job) - assert job.num_tasks == 4 - - def test_flex_alloc_valid_reservation_option(make_flexible_job): job = make_flexible_job('all', sched_access=['--constraint=f2']) job.options = ['--reservation=dummy'] @@ -797,14 +765,6 @@ def test_flex_alloc_valid_reservation_option(make_flexible_job): assert job.num_tasks == 4 -def test_flex_alloc_exclude_nodes_cmd(make_flexible_job): - job = make_flexible_job('all', - sched_access=['--constraint=f1'], - sched_exclude_nodelist='nid00001') - prepare_job(job) - assert job.num_tasks == 8 - - def test_flex_alloc_exclude_nodes_opt(make_flexible_job): job = make_flexible_job('all', sched_access=['--constraint=f1']) job.options = ['-x nid00001'] From d221bf6955f4de7787d35ffbfd847b57d9413b56 Mon Sep 17 00:00:00 2001 From: rafael Date: Fri, 12 Jun 2020 09:24:30 +0200 Subject: [PATCH 2/2] remove commented line --- unittests/test_schedulers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/unittests/test_schedulers.py b/unittests/test_schedulers.py index a48798c809..d672c712b0 100644 --- a/unittests/test_schedulers.py +++ b/unittests/test_schedulers.py @@ -178,7 +178,6 @@ def _expected_pbs_directives(job): ':mem=100GB:cpu_type=haswell' % (num_nodes, job.num_tasks_per_node, num_cpus_per_node), - # '#PBS -q %s' % job.sched_partition, '#PBS --gres=gpu:4', '#DW jobdw capacity=100GB', '#DW stage_in source=/foo'