Skip to content

Return job steps from slurmdb_jobs.get #167 #168

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Feb 20, 2020
Merged

Return job steps from slurmdb_jobs.get #167 #168

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7120b40
add # cython: language_level=2
vkantchev Feb 13, 2020
0f3073c
Return job steps from slurmdb_jobs.get() #167
vkantchev Feb 14, 2020
9e0b8c4
Merge from upstream `master'
vkantchev Feb 18, 2020
2595291
Add test #167
vkantchev Feb 19, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions examples/listdb_jobs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
List all jobs in Slurm, similar to `sacct`
"""
import time
import datetime

import pyslurm

Expand All @@ -15,11 +16,11 @@ def job_display(job):

if __name__ == "__main__":
try:
end = time.time()
start = end - (30 * 24 * 60 * 60)
print("start={}, end={}".format(start, end))
start = (datetime.datetime.utcnow() - datetime.timedelta(days=1)).strftime("%Y-%m-%dT00:00:00")
end = (datetime.datetime.utcnow() + datetime.timedelta(days=1)).strftime("%Y-%m-%dT00:00:00")

jobs = pyslurm.slurmdb_jobs()
jobs_dict = jobs.get(starttime=start, endtime=end)
jobs_dict = jobs.get(starttime=start.encode('utf-8'), endtime=end.encode('utf-8'))
if jobs_dict:
for key, value in jobs_dict.items():
print("{} Job: {}".format("{", key))
Expand Down
106 changes: 99 additions & 7 deletions pyslurm/pyslurm.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# cython: embedsignature=True
# cython: profile=False
# cython: language_level=2
import os
import re
import sys
Expand Down Expand Up @@ -5434,8 +5435,8 @@ cdef class slurmdb_jobs:
if job is not NULL:
jobid = job.jobid
JOBS_info[u'account'] = slurm.stringOrNone(job.account, '')
JOBS_info[u'allocated_gres'] = slurm.stringOrNone(job.alloc_gres, '')
JOBS_info[u'allocated_nodes'] = job.alloc_nodes
JOBS_info[u'alloc_gres'] = slurm.stringOrNone(job.alloc_gres, '')
JOBS_info[u'alloc_nodes'] = job.alloc_nodes
JOBS_info[u'array_job_id'] = job.array_job_id
JOBS_info[u'array_max_tasks'] = job.array_max_tasks
JOBS_info[u'array_task_id'] = job.array_task_id
Expand All @@ -5448,7 +5449,7 @@ cdef class slurmdb_jobs:
JOBS_info[u'elapsed'] = job.elapsed
JOBS_info[u'eligible'] = job.eligible
JOBS_info[u'end'] = job.end
JOBS_info[u'exit_code'] = job.exitcode
JOBS_info[u'exitcode'] = job.exitcode
JOBS_info[u'gid'] = job.gid
JOBS_info[u'jobid'] = job.jobid
JOBS_info[u'jobname'] = slurm.stringOrNone(job.jobname, '')
Expand All @@ -5466,11 +5467,101 @@ cdef class slurmdb_jobs:
JOBS_info[u'show_full'] = job.show_full
JOBS_info[u'start'] = job.start
JOBS_info[u'state'] = job.state
JOBS_info[u'state_str'] = slurm.slurm_job_state_string(job.state)
JOBS_info[u'state_str'] = slurm.stringOrNone(slurm.slurm_job_state_string(job.state), '')
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does slurm_job_state_string ever return null?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It returns byte array, e.g. b'123' which breaks things like json.dumps.


# TRES are reported as strings in the format `TRESID=value` where TRESID is one of:
# TRES_CPU=1, TRES_MEM=2, TRES_ENERGY=3, TRES_NODE=4, TRES_BILLING=5, TRES_FS_DISK=6, TRES_VMEM=7, TRES_PAGES=8
# Example: '1=0,2=745472,3=0,6=1949,7=7966720,8=0'
JOBS_info[u'stats'] = {}
stats = JOBS_info[u'stats']
stats[u'act_cpufreq'] = job.stats.act_cpufreq
stats[u'consumed_energy'] = job.stats.consumed_energy
stats[u'tres_usage_in_max'] = slurm.stringOrNone(job.stats.tres_usage_in_max, '')
stats[u'tres_usage_in_max_nodeid'] = slurm.stringOrNone(job.stats.tres_usage_in_max_nodeid, '')
stats[u'tres_usage_in_max_taskid'] = slurm.stringOrNone(job.stats.tres_usage_in_max_taskid, '')
stats[u'tres_usage_in_min'] = slurm.stringOrNone(job.stats.tres_usage_in_min, '')
stats[u'tres_usage_in_min_nodeid'] = slurm.stringOrNone(job.stats.tres_usage_in_min_nodeid, '')
stats[u'tres_usage_in_min_taskid'] = slurm.stringOrNone(job.stats.tres_usage_in_min_taskid, '')
stats[u'tres_usage_in_tot'] = slurm.stringOrNone(job.stats.tres_usage_in_tot, '')
stats[u'tres_usage_out_ave'] = slurm.stringOrNone(job.stats.tres_usage_out_ave, '')
stats[u'tres_usage_out_max'] = slurm.stringOrNone(job.stats.tres_usage_out_max, '')
stats[u'tres_usage_out_max_nodeid'] = slurm.stringOrNone(job.stats.tres_usage_out_max_nodeid, '')
stats[u'tres_usage_out_max_taskid'] = slurm.stringOrNone(job.stats.tres_usage_out_max_taskid, '')
stats[u'tres_usage_out_min'] = slurm.stringOrNone(job.stats.tres_usage_out_min, '')
stats[u'tres_usage_out_min_nodeid'] = slurm.stringOrNone(job.stats.tres_usage_out_min_nodeid, '')
stats[u'tres_usage_out_min_taskid'] = slurm.stringOrNone(job.stats.tres_usage_out_min_taskid, '')
stats[u'tres_usage_out_tot'] = slurm.stringOrNone(job.stats.tres_usage_out_tot, '')

# add job steps
JOBS_info[u'steps'] = {}
step_dict = JOBS_info[u'steps']

stepsNum = slurm.slurm_list_count(job.steps)
stepsIter = slurm.slurm_list_iterator_create(job.steps)
for i in range(stepsNum):
step = <slurm.slurmdb_step_rec_t *>slurm.slurm_list_next(stepsIter)
step_info = {}
if step is not NULL:
step_id = step.stepid

step_info[u'elapsed'] = step.elapsed
step_info[u'end'] = step.end
step_info[u'exitcode'] = step.exitcode

# Don't add this unless you want to create an endless recursive structure
# step_info[u'job_ptr'] = JOBS_Info # job's record

step_info[u'nnodes'] = step.nnodes
step_info[u'nodes'] = slurm.stringOrNone(step.nodes, '')
step_info[u'ntasks'] = step.ntasks
step_info[u'pid_str'] = slurm.stringOrNone(step.pid_str, '')
step_info[u'req_cpufreq_min'] = step.req_cpufreq_min
step_info[u'req_cpufreq_max'] = step.req_cpufreq_max
step_info[u'req_cpufreq_gov'] = step.req_cpufreq_gov
step_info[u'requid'] = step.requid
step_info[u'start'] = step.start
step_info[u'state'] = step.state
step_info[u'state_str'] = slurm.stringOrNone(slurm.slurm_job_state_string(step.state), '')

# TRES are reported as strings in the format `TRESID=value` where TRESID is one of:
# TRES_CPU=1, TRES_MEM=2, TRES_ENERGY=3, TRES_NODE=4, TRES_BILLING=5, TRES_FS_DISK=6, TRES_VMEM=7, TRES_PAGES=8
# Example: '1=0,2=745472,3=0,6=1949,7=7966720,8=0'
step_info[u'stats'] = {}
stats = step_info[u'stats']
stats[u'act_cpufreq'] = step.stats.act_cpufreq
stats[u'consumed_energy'] = step.stats.consumed_energy
stats[u'tres_usage_in_max'] = slurm.stringOrNone(step.stats.tres_usage_in_max, '')
stats[u'tres_usage_in_max_nodeid'] = slurm.stringOrNone(step.stats.tres_usage_in_max_nodeid, '')
stats[u'tres_usage_in_max_taskid'] = slurm.stringOrNone(step.stats.tres_usage_in_max_taskid, '')
stats[u'tres_usage_in_min'] = slurm.stringOrNone(step.stats.tres_usage_in_min, '')
stats[u'tres_usage_in_min_nodeid'] = slurm.stringOrNone(step.stats.tres_usage_in_min_nodeid, '')
stats[u'tres_usage_in_min_taskid'] = slurm.stringOrNone(step.stats.tres_usage_in_min_taskid, '')
stats[u'tres_usage_in_tot'] = slurm.stringOrNone(step.stats.tres_usage_in_tot, '')
stats[u'tres_usage_out_ave'] = slurm.stringOrNone(step.stats.tres_usage_out_ave, '')
stats[u'tres_usage_out_max'] = slurm.stringOrNone(step.stats.tres_usage_out_max, '')
stats[u'tres_usage_out_max_nodeid'] = slurm.stringOrNone(step.stats.tres_usage_out_max_nodeid, '')
stats[u'tres_usage_out_max_taskid'] = slurm.stringOrNone(step.stats.tres_usage_out_max_taskid, '')
stats[u'tres_usage_out_min'] = slurm.stringOrNone(step.stats.tres_usage_out_min, '')
stats[u'tres_usage_out_min_nodeid'] = slurm.stringOrNone(step.stats.tres_usage_out_min_nodeid, '')
stats[u'tres_usage_out_min_taskid'] = slurm.stringOrNone(step.stats.tres_usage_out_min_taskid, '')
stats[u'tres_usage_out_tot'] = slurm.stringOrNone(step.stats.tres_usage_out_tot, '')

step_info[u'stepid'] = step_id
step_info[u'stepname'] = slurm.stringOrNone(step.stepname, '')
step_info[u'suspended'] = step.suspended
step_info[u'sys_cpu_sec'] = step.sys_cpu_sec
step_info[u'sys_cpu_usec'] = step.sys_cpu_usec
step_info[u'task_dist'] = step.task_dist
step_info[u'tot_cpu_sec'] = step.tot_cpu_sec
step_info[u'tot_cpu_usec'] = step.tot_cpu_usec
step_info[u'tres_alloc_str'] = slurm.stringOrNone(step.tres_alloc_str, '')
step_info[u'user_cpu_sec'] = step.user_cpu_sec
step_info[u'user_cpu_usec'] = step.user_cpu_usec

step_dict[step_id] = step_info

slurm.slurm_list_iterator_destroy(stepsIter)

JOBS_info[u'stat_actual_cpufreq'] = job.stats.act_cpufreq

JOBS_info[u'steps'] = "Not filled, string should be handled"
JOBS_info[u'submit'] = job.submit
JOBS_info[u'suspended'] = job.suspended
JOBS_info[u'sys_cpu_sec'] = job.sys_cpu_sec
Expand All @@ -5488,6 +5579,7 @@ cdef class slurmdb_jobs:
JOBS_info[u'user_cpu_sec'] = job.user_cpu_usec
JOBS_info[u'wckey'] = slurm.stringOrNone(job.wckey, '')
JOBS_info[u'wckeyid'] = job.wckeyid
JOBS_info[u'work_dir'] = slurm.stringOrNone(job.work_dir, '')
J_dict[jobid] = JOBS_info

slurm.slurm_list_iterator_destroy(iters)
Expand Down
29 changes: 28 additions & 1 deletion pyslurm/slurm.pxd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2781,7 +2781,34 @@ cdef extern from 'slurm/slurmdb.h' nogil:
List tres_list
List user_list

# ctypedef struct slurmdb_step_rec_t
ctypedef struct slurmdb_step_rec_t:
uint32_t elapsed
time_t end
int32_t exitcode
slurmdb_job_rec_t *job_ptr # job's record
uint32_t nnodes
char *nodes
uint32_t ntasks
char *pid_str
uint32_t req_cpufreq_min
uint32_t req_cpufreq_max
uint32_t req_cpufreq_gov
uint32_t requid
time_t start
uint32_t state
slurmdb_stats_t stats
uint32_t stepid # job's step number
char *stepname
uint32_t suspended
uint32_t sys_cpu_sec
uint32_t sys_cpu_usec
uint32_t task_dist
uint32_t tot_cpu_sec
uint32_t tot_cpu_usec
char *tres_alloc_str
uint32_t user_cpu_sec
uint32_t user_cpu_usec

# ctypedef struct slurmdb_res_cond_t
# ctypedef struct slurmdb_res_rec_t
# ctypedef struct slurmdb_txn_cond_t
Expand Down
39 changes: 38 additions & 1 deletion tests/test-slurmdb.py → tests/test_slurmdb.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import datetime
import pwd
import subprocess
import datetime
import time
import json

from nose.tools import assert_equals

Expand Down Expand Up @@ -62,6 +64,41 @@ def test_slurmdb_jobs_get():
njobs_sacct = njobs_sacct_jobs(starttime, endtime)
assert_equals(njobs_pyslurm, njobs_sacct)

def test_slurmdb_jobs_get_steps():
"""
Slurmdb: Get jobs with steps for all users
"""
job = {
"wrap": """
srun hostname
srun sleep 1
""",
"job_name": "pyslurm_test_job_steps",
"ntasks": 1,
"cpus_per_task": 1,
}
job_id = pyslurm.job().submit_batch_job(job)

# wait for job to finish
time.sleep(10)

# get `sacct` jobs
start = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime("%Y-%m-%dT00:00:00")
end = (datetime.datetime.now() + datetime.timedelta(days=1)).strftime("%Y-%m-%dT00:00:00")
jobs = pyslurm.slurmdb_jobs().get(starttime=start.encode('utf-8'), endtime=end.encode('utf-8'))

# make sure results are valid json
assert json.dumps(jobs, sort_keys=True, indent=4)

# we should get our job in the results
assert jobs.get(job_id, None)

# and it should have steps
assert jobs[job_id]["steps"]

# and 3 steps, 1 batch + 2 srun
assert 3 == len(jobs[job_id]["steps"])


def test_slurmdb_jobs_get_byuser():
"""
Expand Down