Skip to content

[bugfix] Add support for heterogeneous jobs in Slurm backend #2376

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Jan 13, 2022
Merged

[bugfix] Add support for heterogeneous jobs in Slurm backend #2376

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7745dba
Make Slurm polling aware of Heterogeneous Jobs
Jan 12, 2022
ed3177c
Fix job status check for heterogeneous Slurm jobs
Jan 12, 2022
efe3cc1
Address PR comments
Jan 13, 2022
07f4597
Merge branch 'master' into bugfix/poll_hetjob_slurm
Jan 13, 2022
98596f3
Merge branch 'master' into bugfix/poll_hetjob_slurm
Jan 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 12 additions & 7 deletions reframe/core/schedulers/slurm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,12 +90,16 @@ class SlurmJobScheduler(sched.JobScheduler):
# standard job state polling using sacct.
SACCT_SQUEUE_RATIO = 10

# This matches the format for both normal jobs as well as job arrays.
# This matches the format for both normal and heterogeneous jobs,
# as well as job arrays.
# For heterogeneous jobs, the job_id has the following format:
# <het_job_id>+<het_job_offset>
# (https://slurm.schedmd.com/heterogeneous_jobs.html)
# For job arrays the job_id has one of the following formats:
# * <job_id>_<array_task_id>
# * <job_id>_[<array_task_id_start>-<array_task_id_end>]
# See (`Job Array Support<https://slurm.schedmd.com/job_array.html`__)
_state_patt = r'\d+(?:_\d+|_\[\d+-\d+\])?'
# (https://slurm.schedmd.com/job_array.html)
_jobid_patt = r'\d+(?:\+\d+|_\d+|_\[\d+-\d+\])?'

def __init__(self):
self._prefix = '#SBATCH'
Expand Down Expand Up @@ -406,7 +410,7 @@ def poll(self, *jobs):

# We need the match objects, so we have to use finditer()
state_match = list(re.finditer(
fr'^(?P<jobid>{self._state_patt})\|(?P<state>\S+)([^\|]*)\|'
fr'^(?P<jobid>{self._jobid_patt})\|(?P<state>\S+)([^\|]*)\|'
fr'(?P<exitcode>\d+)\:(?P<signal>\d+)\|(?P<end>\S+)\|'
fr'(?P<nodespec>.*)', completed.stdout, re.MULTILINE)
)
Expand All @@ -418,7 +422,8 @@ def poll(self, *jobs):

job_info = {}
for s in state_match:
jobid = s.group('jobid').split('_')[0]
# Take into account both job arrays and heterogeneous jobs
jobid = re.split(r'_|\+', s.group('jobid'))[0]
job_info.setdefault(jobid, []).append(s)

for job in jobs:
Expand All @@ -427,7 +432,7 @@ def poll(self, *jobs):
except KeyError:
continue

# Join the states with ',' in case of job arrays
# Join the states with ',' in case of job arrays|heterogeneous jobs
job._state = ','.join(m.group('state') for m in jobarr_info)

if not self._update_state_count % self.SACCT_SQUEUE_RATIO:
Expand Down Expand Up @@ -574,7 +579,7 @@ def poll(self, *jobs):

# We need the match objects, so we have to use finditer()
state_match = list(re.finditer(
fr'^(?P<jobid>{self._state_patt})\|(?P<state>\S+)\|'
fr'^(?P<jobid>{self._jobid_patt})\|(?P<state>\S+)\|'
fr'(?P<nodespec>\S*)\|(?P<reason>.+)',
completed.stdout, re.MULTILINE)
)
Expand Down