From a1d26af2dc77248950744283a469e0e5a786f802 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 6 Jun 2018 13:35:29 +0200 Subject: [PATCH 1/5] PBS scheduler backend --- config/cscs-pbs.py | 161 ++++++++++++++++++++++++++++ config/cscs.py | 1 - cscs-checks/cuda/cuda_checks.py | 2 +- reframe/core/pipeline.py | 3 +- reframe/core/schedulers/pbs.py | 114 ++++++++++++++++++++ reframe/core/schedulers/registry.py | 1 + unittests/test_schedulers.py | 128 ++++++++++++++++------ 7 files changed, 372 insertions(+), 38 deletions(-) create mode 100644 config/cscs-pbs.py create mode 100644 reframe/core/schedulers/pbs.py diff --git a/config/cscs-pbs.py b/config/cscs-pbs.py new file mode 100644 index 0000000000..ffbcd8bb03 --- /dev/null +++ b/config/cscs-pbs.py @@ -0,0 +1,161 @@ +# +# Minimal CSCS configuration for testing the PBS backend +# + + +class ReframeSettings: + _reframe_module = 'reframe' + _job_poll_intervals = [1, 2, 3] + _job_submit_timeout = 60 + _checks_path = ['checks/'] + _checks_path_recurse = True + _site_configuration = { + 'systems': { + 'dom': { + 'descr': 'Dom TDS', + 'hostnames': ['dom'], + 'modules_system': 'tmod', + 'resourcesdir': '/apps/common/regression/resources', + 'partitions': { + 'login': { + 'scheduler': 'local', + 'modules': [], + 'access': [], + 'environs': ['PrgEnv-cray', 'PrgEnv-gnu', + 'PrgEnv-intel', 'PrgEnv-pgi'], + 'descr': 'Login nodes', + 'max_jobs': 4 + }, + + 'gpu': { + 'scheduler': 'pbs+mpiexec', + 'modules': ['daint-gpu'], + 'access': ['proc=gpu'], + 'environs': ['PrgEnv-cray', 'PrgEnv-gnu', + 'PrgEnv-intel', 'PrgEnv-pgi'], + 'descr': 'Hybrid nodes (Haswell/P100)', + 'max_jobs': 100, + }, + + 'mc': { + 'scheduler': 'pbs+mpiexec', + 'modules': ['daint-mc'], + 'access': ['proc=mc'], + 'environs': ['PrgEnv-cray', 'PrgEnv-gnu', + 'PrgEnv-intel', 'PrgEnv-pgi'], + 'descr': 'Multicore nodes (Broadwell)', + 'max_jobs': 100, + }, + } + }, + + 'generic': { + 'descr': 'Generic example system', + 'partitions': { + 'login': { + 'scheduler': 'local', + 'modules': [], + 'access': [], + 'environs': ['builtin-gcc'], + 'descr': 'Login nodes' + } + } + } + }, + + 'environments': { + '*': { + 'PrgEnv-cray': { + 'type': 'ProgEnvironment', + 'modules': ['PrgEnv-cray'], + }, + + 'PrgEnv-gnu': { + 'type': 'ProgEnvironment', + 'modules': ['PrgEnv-gnu'], + }, + + 'PrgEnv-intel': { + 'type': 'ProgEnvironment', + 'modules': ['PrgEnv-intel'], + }, + + 'PrgEnv-pgi': { + 'type': 'ProgEnvironment', + 'modules': ['PrgEnv-pgi'], + }, + + 'builtin': { + 'type': 'ProgEnvironment', + 'cc': 'cc', + 'cxx': '', + 'ftn': '', + }, + + 'builtin-gcc': { + 'type': 'ProgEnvironment', + 'cc': 'gcc', + 'cxx': 'g++', + 'ftn': 'gfortran', + } + } + }, + } + + _logging_config = { + 'level': 'DEBUG', + 'handlers': { + 'reframe.log': { + 'level': 'DEBUG', + 'format': '[%(asctime)s] %(levelname)s: ' + '%(check_info)s: %(message)s', + 'append': False, + }, + + # Output handling + '&1': { + 'level': 'INFO', + 'format': '%(message)s' + }, + 'reframe.out': { + 'level': 'INFO', + 'format': '%(message)s', + 'append': False, + } + } + } + + @property + def version(self): + return self._version + + @property + def reframe_module(self): + return self._reframe_module + + @property + def job_poll_intervals(self): + return self._job_poll_intervals + + @property + def job_submit_timeout(self): + return self._job_submit_timeout + + @property + def checks_path(self): + return self._checks_path + + @property + def checks_path_recurse(self): + return self._checks_path_recurse + + @property + def site_configuration(self): + return self._site_configuration + + @property + def logging_config(self): + return self._logging_config + + +settings = ReframeSettings() diff --git a/config/cscs.py b/config/cscs.py index ce2dbe6742..786dc8c081 100644 --- a/config/cscs.py +++ b/config/cscs.py @@ -178,7 +178,6 @@ class ReframeSettings: } }, - # Generic system used for cli unit tests 'generic': { 'descr': 'Generic example system', 'partitions': { diff --git a/cscs-checks/cuda/cuda_checks.py b/cscs-checks/cuda/cuda_checks.py index 2c05e5571f..895ebd6b9c 100644 --- a/cscs-checks/cuda/cuda_checks.py +++ b/cscs-checks/cuda/cuda_checks.py @@ -12,7 +12,7 @@ def __init__(self, name, **kwargs): self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-gnu'] self.sourcesdir = os.path.join(self.current_system.resourcesdir, 'CUDA', 'essentials') - self.modules = ['cudatoolkit'] + self.modules = ['craype-accel-nvidia60'] self.maintainers = ['AJ', 'VK'] self.num_gpus_per_node = 1 self.tags = {'production'} diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 3529f2775d..59f678b23b 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -811,8 +811,7 @@ def _setup_job(self, **job_opts): pre_run=self.pre_run, post_run=self.post_run, sched_exclusive_access=self.exclusive_access, - **job_opts - ) + **job_opts) # Get job options from managed resources and prepend them to # job_opts. We want any user supplied options to be able to diff --git a/reframe/core/schedulers/pbs.py b/reframe/core/schedulers/pbs.py new file mode 100644 index 0000000000..856fe3341d --- /dev/null +++ b/reframe/core/schedulers/pbs.py @@ -0,0 +1,114 @@ +# +# PBS backend +# +# - Initial version submitted by Rafael Escovar, ASML +# + +import os +import itertools +import re +import time +from datetime import datetime + +import reframe.core.schedulers as sched +import reframe.utility.os_ext as os_ext +from reframe.core.config import settings +from reframe.core.exceptions import (SpawnedProcessError, JobError) +from reframe.core.logging import getlogger +from reframe.core.schedulers.registry import register_scheduler + + +# Time to wait after a job is finished for its standard output/error to be +# written to the corresponding files. +PBS_OUTPUT_WRITEBACK_WAIT = 3 + + +@register_scheduler('pbs') +class PbsJob(sched.Job): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._prefix = '#PBS' + self._time_finished = None + + # Optional part of the job id refering to the PBS server + self._pbs_server = None + + def _emit_lselect_option(self, builder): + num_tasks_per_node = self._num_tasks_per_node or 1 + num_nodes = self._num_tasks // num_tasks_per_node + ret = '-l select=%s:mpiprocs=%s' % (num_nodes, num_tasks_per_node) + if self._num_cpus_per_task: + num_cpus_per_node = num_tasks_per_node * self._num_cpus_per_task + ret += ':ncpus=%s' % num_cpus_per_node + + if self.options: + ret += ':' + ':'.join(self.options) + + self._emit_job_option(ret, builder) + + def _emit_job_option(self, option, builder): + builder.verbatim(self._prefix + ' ' + option) + + def _run_command(self, cmd, timeout=None): + """Run command cmd and re-raise any exception as a JobError.""" + try: + return os_ext.run_command(cmd, check=True, timeout=timeout) + except SpawnedProcessError as e: + raise JobError(jobid=self._jobid) from e + + def emit_preamble(self, builder): + self._emit_job_option('-N "%s"' % self.name, builder) + self._emit_lselect_option(builder) + self._emit_job_option('-l walltime=%d:%d:%d' % self.time_limit, + builder) + if self.sched_partition: + self._emit_job_option('-q %s' % self.sched_partition, builder) + + self._emit_job_option('-o %s' % self.stdout, builder) + self._emit_job_option('-e %s' % self.stderr, builder) + + # PBS starts the job in the home directory by default + builder.verbatim('cd %s' % self.workdir) + + def submit(self): + # `-o` and `-e` options are only recognized in command line by the PBS + # Slurm wrappers. + cmd = 'qsub -o %s -e %s %s' % (self.stdout, self.stderr, + self.script_filename) + completed = self._run_command(cmd, settings().job_submit_timeout) + jobid_match = re.search('^(?P\S+)', completed.stdout) + if not jobid_match: + raise JobError('could not retrieve the job id ' + 'of the submitted job') + + jobid, *info = jobid_match.group('jobid').split('.', maxsplit=2) + self._jobid = int(jobid) + if info: + self._pbs_server = info[0] + + def wait(self): + super().wait() + intervals = itertools.cycle(settings().job_poll_intervals) + while not self.finished(): + time.sleep(next(intervals)) + + def cancel(self): + super().cancel() + + # Recreate the full job id + jobid = str(self._jobid) + if self._pbs_server: + jobid += ':' + self._pbs_server + + getlogger().debug('cancelling job (id=%s)' % jobid) + self._run_command('qdel %s' % jobid, settings().job_submit_timeout) + + def finished(self): + super().finished() + done = os.path.exists(self.stdout) and os.path.exists(self.stderr) + if done: + t_now = datetime.now() + self._time_finished = self._time_finished or t_now + time_from_finish = (t_now - self._time_finished).total_seconds() + + return done and time_from_finish > PBS_OUTPUT_WRITEBACK_WAIT diff --git a/reframe/core/schedulers/registry.py b/reframe/core/schedulers/registry.py index b8ac4b149b..f8772422ec 100644 --- a/reframe/core/schedulers/registry.py +++ b/reframe/core/schedulers/registry.py @@ -32,3 +32,4 @@ def getscheduler(name): # Import the schedulers modules to trigger their registration import reframe.core.schedulers.local import reframe.core.schedulers.slurm +import reframe.core.schedulers.pbs diff --git a/unittests/test_schedulers.py b/unittests/test_schedulers.py index afe61a1c9d..acf85b7264 100644 --- a/unittests/test_schedulers.py +++ b/unittests/test_schedulers.py @@ -75,6 +75,25 @@ def assertScriptSanity(self, script_file): self.assertEqual(['echo prerun', 'hostname', 'echo postrun'], matches) + def setup_job(self): + # Mock up a job submission + self.testjob._time_limit = (0, 5, 0) + self.testjob._num_tasks = 16 + self.testjob._num_tasks_per_node = 2 + self.testjob._num_tasks_per_core = 1 + self.testjob._num_tasks_per_socket = 1 + self.testjob._num_cpus_per_task = 18 + self.testjob._use_smt = True + self.testjob._sched_nodelist = 'nid000[00-17]' + self.testjob._sched_exclude_nodelist = 'nid00016' + self.testjob._sched_partition = 'foo' + self.testjob._sched_reservation = 'bar' + self.testjob._sched_account = 'spam' + self.testjob._sched_exclusive_access = True + self.testjob.options = ['--gres=gpu:4', + '#DW jobdw capacity=100GB', + '#DW stage_in source=/foo'] + def test_prepare(self): self.testjob.prepare(self.builder) self.assertScriptSanity(self.testjob.script_filename) @@ -86,7 +105,6 @@ def test_submit(self): self.testjob.submit() self.assertIsNotNone(self.testjob.jobid) self.testjob.wait() - self.assertEqual(0, self.testjob.exitcode) @fixtures.switch_to_user_runtime def test_submit_timelimit(self, check_elapsed_time=True): @@ -168,6 +186,10 @@ def setup_user(self, msg=None): # Local scheduler is by definition available pass + def test_submit(self): + super().test_submit() + self.assertEqual(0, self.testjob.exitcode) + def test_submit_timelimit(self): from reframe.core.schedulers.local import LOCAL_JOB_TIMEOUT @@ -276,23 +298,7 @@ def setup_user(self, msg=None): super().setup_user(msg='SLURM (with sacct) not configured') def test_prepare(self): - # Mock up a job submission - self.testjob._time_limit = (0, 5, 0) - self.testjob._num_tasks = 16 - self.testjob._num_tasks_per_node = 2 - self.testjob._num_tasks_per_core = 1 - self.testjob._num_tasks_per_socket = 1 - self.testjob._num_cpus_per_task = 18 - self.testjob._use_smt = True - self.testjob._sched_nodelist = 'nid000[00-17]' - self.testjob._sched_exclude_nodelist = 'nid00016' - self.testjob._sched_partition = 'foo' - self.testjob._sched_reservation = 'bar' - self.testjob._sched_account = 'spam' - self.testjob._sched_exclusive_access = True - self.testjob.options = ['--gres=gpu:4', - '#DW jobdw capacity=100GB', - '#DW stage_in source=/foo'] + self.setup_job() super().test_prepare() expected_directives = set([ '#SBATCH --job-name="rfm_testjob"', @@ -324,29 +330,37 @@ def test_prepare(self): self.assertEqual(expected_directives, found_directives) def test_prepare_no_exclusive(self): + self.setup_job() self.testjob._sched_exclusive_access = False super().test_prepare() with open(self.testjob.script_filename) as fp: self.assertIsNone(re.search(r'--exclusive', fp.read())) def test_prepare_no_smt(self): + self.setup_job() self.testjob._use_smt = None super().test_prepare() with open(self.testjob.script_filename) as fp: self.assertIsNone(re.search(r'--hint', fp.read())) def test_prepare_with_smt(self): + self.setup_job() self.testjob._use_smt = True super().test_prepare() with open(self.testjob.script_filename) as fp: self.assertIsNotNone(re.search(r'--hint=multithread', fp.read())) def test_prepare_without_smt(self): + self.setup_job() self.testjob._use_smt = False super().test_prepare() with open(self.testjob.script_filename) as fp: self.assertIsNotNone(re.search(r'--hint=nomultithread', fp.read())) + def test_submit(self): + super().test_submit() + self.assertEqual(0, self.testjob.exitcode) + def test_submit_timelimit(self): # Skip this test for Slurm, since we the minimum time limit is 1min self.skipTest("SLURM's minimum time limit is 60s") @@ -357,8 +371,68 @@ def test_cancel(self): super().test_cancel() self.assertEqual(self.testjob.state, SLURM_JOB_CANCELLED) - def test_poll(self): - super().test_poll() + +class TestSqueueJob(TestSlurmJob): + @property + def sched_name(self): + return 'squeue' + + def setup_user(self, msg=None): + partition = (fixtures.partition_with_scheduler(self.sched_name) or + fixtures.partition_with_scheduler('slurm')) + if partition is None: + self.skipTest('SLURM not configured') + + self.testjob.options += partition.access + + def test_submit(self): + # Squeue backend may not set the exitcode; bypass ourp parent's submit + _TestJob.test_submit(self) + + +class TestPbsJob(_TestJob, unittest.TestCase): + @property + def sched_name(self): + return 'pbs' + + @property + def sched_configured(self): + return fixtures.partition_with_scheduler('pbs') is not None + + @property + def launcher(self): + return LocalLauncher() + + def setup_user(self, msg=None): + super().setup_user(msg='PBS not configured') + + def test_prepare(self): + self.setup_job() + self.testjob.options = ['mem=100GB', 'cpu_type=haswell'] + super().test_prepare() + num_nodes = self.testjob.num_tasks // self.testjob.num_tasks_per_node + num_cpus_per_node = (self.testjob.num_cpus_per_task * + self.testjob.num_tasks_per_node) + expected_directives = set([ + '#PBS -N "rfm_testjob"', + '#PBS -l walltime=0:5:0', + '#PBS -o %s' % self.testjob.stdout, + '#PBS -e %s' % self.testjob.stderr, + '#PBS -l select=%s:mpiprocs=%s:ncpus=%s' + ':mem=100GB:cpu_type=haswell' % (num_nodes, + self.testjob.num_tasks_per_node, + num_cpus_per_node), + '#PBS -q %s' % self.testjob.sched_partition, + ]) + with open(self.testjob.script_filename) as fp: + found_directives = set(re.findall(r'^\#\w+ .*', fp.read(), + re.MULTILINE)) + + self.assertEqual(expected_directives, found_directives) + + def test_submit_timelimit(self): + # Skip this test for PBS, since we the minimum time limit is 1min + self.skipTest("PBS minimum time limit is 60s") class TestSlurmFlexibleNodeAllocation(unittest.TestCase): @@ -550,17 +624,3 @@ def test_attributes(self): def test_str(self): self.assertEqual('nid00001', str(self.node)) - - -class TestSqueueJob(TestSlurmJob): - @property - def sched_name(self): - return 'squeue' - - def setup_user(self, msg=None): - partition = (fixtures.partition_with_scheduler(self.sched_name) or - fixtures.partition_with_scheduler('slurm')) - if partition is None: - self.skipTest('SLURM not configured') - - self.testjob.options += partition.access From ead0f87a86307f82c783dce7dca97db086aa2554 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 6 Jun 2018 18:26:44 +0200 Subject: [PATCH 2/5] PR fixes --- reframe/core/schedulers/pbs.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/reframe/core/schedulers/pbs.py b/reframe/core/schedulers/pbs.py index 856fe3341d..ff8cbc79b9 100644 --- a/reframe/core/schedulers/pbs.py +++ b/reframe/core/schedulers/pbs.py @@ -35,12 +35,12 @@ def __init__(self, *args, **kwargs): def _emit_lselect_option(self, builder): num_tasks_per_node = self._num_tasks_per_node or 1 + num_cpus_per_task = self._num_cpus_per_task or 1 num_nodes = self._num_tasks // num_tasks_per_node - ret = '-l select=%s:mpiprocs=%s' % (num_nodes, num_tasks_per_node) - if self._num_cpus_per_task: - num_cpus_per_node = num_tasks_per_node * self._num_cpus_per_task - ret += ':ncpus=%s' % num_cpus_per_node - + num_cpus_per_node = num_tasks_per_node * num_cpus_per_task + ret = '-l select=%s:mpiprocs=%s:ncpus=%s' % (num_nodes, + num_tasks_per_node, + num_cpus_per_node) if self.options: ret += ':' + ':'.join(self.options) @@ -98,7 +98,7 @@ def cancel(self): # Recreate the full job id jobid = str(self._jobid) if self._pbs_server: - jobid += ':' + self._pbs_server + jobid += '.' + self._pbs_server getlogger().debug('cancelling job (id=%s)' % jobid) self._run_command('qdel %s' % jobid, settings().job_submit_timeout) From e99ccdb9a4101c9e4609349624c03930b96bc815 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 7 Jun 2018 10:32:31 +0200 Subject: [PATCH 3/5] Add another unit test for PBS backend - This test checks if the `ncpus` resource option is emitted correctly when `num_cpus_per_task` is not defined. - Also some cosmetic changes in `reframe/__init__.py`. --- reframe/__init__.py | 6 +++--- unittests/test_schedulers.py | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/reframe/__init__.py b/reframe/__init__.py index 274fe3bfe9..681096a3ea 100644 --- a/reframe/__init__.py +++ b/reframe/__init__.py @@ -3,13 +3,13 @@ VERSION = '2.13-dev1' -_required_pyver = (3, 5, 0) INSTALL_PREFIX = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +MIN_PYTHON_VERSION = (3, 5, 0) # Check python version -if sys.version_info[:3] < _required_pyver: +if sys.version_info[:3] < MIN_PYTHON_VERSION: sys.stderr.write('Unsupported Python version: ' - 'Python >= %d.%d.%d is required\n' % _required_pyver) + 'Python >= %d.%d.%d is required\n' % MIN_PYTHON_VERSION) sys.exit(1) diff --git a/unittests/test_schedulers.py b/unittests/test_schedulers.py index acf85b7264..d5faa74f3b 100644 --- a/unittests/test_schedulers.py +++ b/unittests/test_schedulers.py @@ -430,6 +430,30 @@ def test_prepare(self): self.assertEqual(expected_directives, found_directives) + def test_prepare_no_cpus(self): + self.setup_job() + self.testjob._num_cpus_per_task = None + self.testjob.options = ['mem=100GB', 'cpu_type=haswell'] + super().test_prepare() + num_nodes = self.testjob.num_tasks // self.testjob.num_tasks_per_node + num_cpus_per_node = self.testjob.num_tasks_per_node + expected_directives = set([ + '#PBS -N "rfm_testjob"', + '#PBS -l walltime=0:5:0', + '#PBS -o %s' % self.testjob.stdout, + '#PBS -e %s' % self.testjob.stderr, + '#PBS -l select=%s:mpiprocs=%s:ncpus=%s' + ':mem=100GB:cpu_type=haswell' % (num_nodes, + self.testjob.num_tasks_per_node, + num_cpus_per_node), + '#PBS -q %s' % self.testjob.sched_partition, + ]) + with open(self.testjob.script_filename) as fp: + found_directives = set(re.findall(r'^\#\w+ .*', fp.read(), + re.MULTILINE)) + + self.assertEqual(expected_directives, found_directives) + def test_submit_timelimit(self): # Skip this test for PBS, since we the minimum time limit is 1min self.skipTest("PBS minimum time limit is 60s") From 3b2e76463840409db1aba806911750d63546f3aa Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 7 Jun 2018 11:05:04 +0200 Subject: [PATCH 4/5] Enable CI for PBS backend --- ci-scripts/ci-runner.bash | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ci-scripts/ci-runner.bash b/ci-scripts/ci-runner.bash index 839c5d44de..be0e73ca58 100644 --- a/ci-scripts/ci-runner.bash +++ b/ci-scripts/ci-runner.bash @@ -183,6 +183,18 @@ else checked_exec ./test_reframe.py --rfm-user-config=config/cscs.py + echo "===================================" + echo "Running unit tests with PBS backend" + echo "===================================" + + if [[ $(hostname) =~ dom ]]; then + PATH_save=$PATH + export PATH=/apps/dom/UES/karakasv/slurm-wrappers/bin:$PATH + checked_exec ./test_reframe.py --rfm-user-config=config/cscs-pbs.py + export PATH=$PATH_save + fi + + # Find modified or added user checks userchecks=( $(git log --name-status --oneline --no-merges -1 | \ awk '/^[AM]/ { print $2 } /^R0[0-9][0-9]/ { print $3 }' | \ From a05a30b7be4dad8adbf8bf907a986557e1551161 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 7 Jun 2018 13:43:11 +0200 Subject: [PATCH 5/5] Update documentation + PBS fixes PBS fixes: - Treat separately options and `-lselect` resources. - Treat custom prefixes. Though I don't know of cases that need that, it adds more flexibility and is in accordance with the Slurm backend. --- docs/configure.rst | 75 ++++++++++++++++++++-------------- reframe/core/schedulers/pbs.py | 28 +++++++++---- unittests/test_schedulers.py | 10 ++++- 3 files changed, 74 insertions(+), 39 deletions(-) diff --git a/docs/configure.rst b/docs/configure.rst index a1b61a1ed0..b44d1d723b 100644 --- a/docs/configure.rst +++ b/docs/configure.rst @@ -128,36 +128,14 @@ The available partition attributes are the following: * ``descr``: A detailed description of the partition (default is the partition name). * ``scheduler``: The job scheduler and parallel program launcher combination that is used on this partition to launch jobs. - The syntax of this attribute is ``+``. The available values for the job scheduler are the following: - - * ``slurm``: Jobs on this partition will be launched using `Slurm `__. - This scheduler relies on job accounting (``sacct`` command) in order to reliably query the job status. - * ``squeue``: Jobs on this partition will be launched using `Slurm `__, but no job accounting is required. - The job status is obtained using the ``squeue`` command. - This scheduler is less reliable than the one based on the ``sacct`` command, but the framework does its best to query the job state as reliably as possible. - * ``local``: Jobs on this partition will be launched locally as OS processes. - - The available values for the parallel program launchers are the following: - - * ``srun``: Programs on this partition will be launched using a bare ``srun`` command *without* any job allocation options passed to it. - This launcher may only be used with the ``slurm`` scheduler. - - * ``srunalloc``: Programs on this partition will be launched using the ``srun`` command *with* job allocation options passed automatically to it. - This launcher may also be used with the ``local`` scheduler. - * ``alps``: Programs on this partition will be launched using the ``aprun`` command. - * ``mpirun``: Programs on this partition will be launched using the ``mpirun`` command. - * ``mpiexec``: Programs on this partition will be launched using the ``mpiexec`` command. - * ``local``: Programs on this partition will be launched as-is without using any parallel program launcher. - - There exist also the following aliases for specific combinations of job schedulers and parallel program launchers: - - * ``nativeslurm``: This is equivalent to ``slurm+srun``. - * ``local``: This is equivalent to ``local+local``. + The syntax of this attribute is ``+``. + A list of the supported `schedulers <#supported-scheduler-backends>`__ and `parallel launchers <#supported-parallel-launchers>`__ can be found at the end of this section. * ``access``: A list of scheduler options that will be passed to the generated job script for gaining access to that logical partition (default ``[]``). * ``environs``: A list of environments, with which ReFrame will try to run any regression tests written for this partition (default ``[]``). The environment names must be resolved inside the ``environments`` section of the ``_site_configuration`` dictionary (see `Environments Configuration <#environments-configuration>`__ for more information). + * ``modules``: A list of modules to be loaded before running a regression test on that partition (default ``[]``). * ``variables``: A set of environment variables to be set before running a regression test on that partition (default ``{}``). @@ -227,20 +205,57 @@ The available partition attributes are the following: } } +.. note:: + For the `PBS <#supported-scheduler-backends>`__ backend, options accepted in the ``access`` and ``resources`` attributes may either refer to actual ``qsub`` options or be just resources specifications to be passed to the ``-l select`` option. + The backend assumes a ``qsub`` option, if the options passed in these attributes start with a ``-``. + .. note:: .. versionchanged:: 2.8 A new syntax for the ``scheduler`` values was introduced as well as more parallel program launchers. The old values for the ``scheduler`` key will continue to be supported. -.. note:: - .. versionadded:: 2.8.1 - The ``squeue`` backend scheduler was added. - -.. note:: .. versionchanged:: 2.9 Better support for custom job resources. + +Supported scheduler backends +============================ + +ReFrame supports the following job schedulers: + + +* ``slurm``: Jobs on the configured partition will be launched using `Slurm `__. + This scheduler relies on job accounting (``sacct`` command) in order to reliably query the job status. +* ``squeue``: *[new in 2.8.1]* + Jobs on the configured partition will be launched using `Slurm `__, but no job accounting is required. + The job status is obtained using the ``squeue`` command. + This scheduler is less reliable than the one based on the ``sacct`` command, but the framework does its best to query the job state as reliably as possible. + +* ``pbs``: *[new in 2.13]* Jobs on the configured partition will be launched using a `PBS-based `__ scheduler. +* ``local``: Jobs on the configured partition will be launched locally as OS processes. + + +Supported parallel launchers +============================ + +ReFrame supports the following parallel job launchers: + +* ``srun``: Programs on the configured partition will be launched using a bare ``srun`` command *without* any job allocation options passed to it. + This launcher may only be used with the ``slurm`` scheduler. +* ``srunalloc``: Programs on the configured partition will be launched using the ``srun`` command *with* job allocation options passed automatically to it. + This launcher may also be used with the ``local`` scheduler. +* ``alps``: Programs on the configured partition will be launched using the ``aprun`` command. +* ``mpirun``: Programs on the configured partition will be launched using the ``mpirun`` command. +* ``mpiexec``: Programs on the configured partition will be launched using the ``mpiexec`` command. +* ``local``: Programs on the configured partition will be launched as-is without using any parallel program launcher. + +There exist also the following aliases for specific combinations of job schedulers and parallel program launchers: + +* ``nativeslurm``: This is equivalent to ``slurm+srun``. +* ``local``: This is equivalent to ``local+local``. + + Environments Configuration -------------------------- diff --git a/reframe/core/schedulers/pbs.py b/reframe/core/schedulers/pbs.py index ff8cbc79b9..766d174f52 100644 --- a/reframe/core/schedulers/pbs.py +++ b/reframe/core/schedulers/pbs.py @@ -38,13 +38,27 @@ def _emit_lselect_option(self, builder): num_cpus_per_task = self._num_cpus_per_task or 1 num_nodes = self._num_tasks // num_tasks_per_node num_cpus_per_node = num_tasks_per_node * num_cpus_per_task - ret = '-l select=%s:mpiprocs=%s:ncpus=%s' % (num_nodes, - num_tasks_per_node, - num_cpus_per_node) - if self.options: - ret += ':' + ':'.join(self.options) - - self._emit_job_option(ret, builder) + select_opt = '-l select=%s:mpiprocs=%s:ncpus=%s' % (num_nodes, + num_tasks_per_node, + num_cpus_per_node) + + # Options starting with `-` are emitted in separate lines + rem_opts = [] + verb_opts = [] + for opt in self.options: + if opt.startswith('-'): + rem_opts.append(opt) + elif opt.startswith('#'): + verb_opts.append(opt) + else: + select_opt += ':' + opt + + self._emit_job_option(select_opt, builder) + for opt in rem_opts: + self._emit_job_option(opt, builder) + + for opt in verb_opts: + builder.verbatim(opt) def _emit_job_option(self, option, builder): builder.verbatim(self._prefix + ' ' + option) diff --git a/unittests/test_schedulers.py b/unittests/test_schedulers.py index d5faa74f3b..5e47c1ffbe 100644 --- a/unittests/test_schedulers.py +++ b/unittests/test_schedulers.py @@ -408,7 +408,7 @@ def setup_user(self, msg=None): def test_prepare(self): self.setup_job() - self.testjob.options = ['mem=100GB', 'cpu_type=haswell'] + self.testjob.options += ['mem=100GB', 'cpu_type=haswell'] super().test_prepare() num_nodes = self.testjob.num_tasks // self.testjob.num_tasks_per_node num_cpus_per_node = (self.testjob.num_cpus_per_task * @@ -423,6 +423,9 @@ def test_prepare(self): self.testjob.num_tasks_per_node, num_cpus_per_node), '#PBS -q %s' % self.testjob.sched_partition, + '#PBS --gres=gpu:4', + '#DW jobdw capacity=100GB', + '#DW stage_in source=/foo' ]) with open(self.testjob.script_filename) as fp: found_directives = set(re.findall(r'^\#\w+ .*', fp.read(), @@ -433,7 +436,7 @@ def test_prepare(self): def test_prepare_no_cpus(self): self.setup_job() self.testjob._num_cpus_per_task = None - self.testjob.options = ['mem=100GB', 'cpu_type=haswell'] + self.testjob.options += ['mem=100GB', 'cpu_type=haswell'] super().test_prepare() num_nodes = self.testjob.num_tasks // self.testjob.num_tasks_per_node num_cpus_per_node = self.testjob.num_tasks_per_node @@ -447,6 +450,9 @@ def test_prepare_no_cpus(self): self.testjob.num_tasks_per_node, num_cpus_per_node), '#PBS -q %s' % self.testjob.sched_partition, + '#PBS --gres=gpu:4', + '#DW jobdw capacity=100GB', + '#DW stage_in source=/foo' ]) with open(self.testjob.script_filename) as fp: found_directives = set(re.findall(r'^\#\w+ .*', fp.read(),