Skip to content

Add exec_prefix option #73

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 29, 2018
Merged

Add exec_prefix option #73

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 27 additions & 20 deletions batchspawner/batchspawner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,10 @@ class BatchSpawnerBase(Spawner):
# override default server ip since batch jobs normally running remotely
ip = Unicode("0.0.0.0", help="Address for singleuser server to listen at").tag(config=True)

exec_prefix = Unicode('sudo -E -u {username}', \
help="Standard executon prefix (e.g. the default sudo -E -u {username})"
).tag(config=True)

# all these req_foo traits will be available as substvars for templated strings
req_queue = Unicode('', \
help="Queue name to submit job to resource manager"
Expand Down Expand Up @@ -177,7 +181,8 @@ def cmd_formatted_for_batch(self):
@gen.coroutine
def submit_batch_script(self):
subvars = self.get_req_subvars()
cmd = self.batch_submit_cmd.format(**subvars)
cmd = self.exec_prefix + ' ' + self.batch_submit_cmd
cmd = cmd.format(**subvars)
subvars['cmd'] = self.cmd_formatted_for_batch()
if hasattr(self, 'user_options'):
subvars.update(self.user_options)
Expand Down Expand Up @@ -207,7 +212,8 @@ def read_job_state(self):
return self.job_status
subvars = self.get_req_subvars()
subvars['job_id'] = self.job_id
cmd = self.batch_query_cmd.format(**subvars)
cmd = self.exec_prefix + ' ' + self.batch_query_cmd
cmd = cmd.format(**subvars)
self.log.debug('Spawner querying job: ' + cmd)
try:
out = yield run_command(cmd)
Expand All @@ -226,7 +232,8 @@ def read_job_state(self):
def cancel_batch_job(self):
subvars = self.get_req_subvars()
subvars['job_id'] = self.job_id
cmd = self.batch_cancel_cmd.format(**subvars)
cmd = self.exec_prefix + ' ' + self.batch_cancel_cmd
cmd = cmd.format(**subvars)
self.log.info('Cancelling job ' + self.job_id + ': ' + cmd)
yield run_command(cmd)

Expand Down Expand Up @@ -420,21 +427,21 @@ class TorqueSpawner(BatchSpawnerRegexStates):
""").tag(config=True)

# outputs job id string
batch_submit_cmd = Unicode('sudo -E -u {username} qsub').tag(config=True)
batch_submit_cmd = Unicode('qsub').tag(config=True)
# outputs job data XML string
batch_query_cmd = Unicode('sudo -E -u {username} qstat -x {job_id}').tag(config=True)
batch_cancel_cmd = Unicode('sudo -E -u {username} qdel {job_id}').tag(config=True)
batch_query_cmd = Unicode('qstat -x {job_id}').tag(config=True)
batch_cancel_cmd = Unicode('qdel {job_id}').tag(config=True)
# search XML string for job_state - [QH] = pending, R = running, [CE] = done
state_pending_re = Unicode(r'<job_state>[QH]</job_state>').tag(config=True)
state_running_re = Unicode(r'<job_state>R</job_state>').tag(config=True)
state_exechost_re = Unicode(r'<exec_host>((?:[\w_-]+\.?)+)/\d+').tag(config=True)

class MoabSpawner(TorqueSpawner):
# outputs job id string
batch_submit_cmd = Unicode('sudo -E -u {username} msub').tag(config=True)
batch_submit_cmd = Unicode('msub').tag(config=True)
# outputs job data XML string
batch_query_cmd = Unicode('sudo -E -u {username} mdiag -j {job_id} --xml').tag(config=True)
batch_cancel_cmd = Unicode('sudo -E -u {username} mjobctl -c {job_id}').tag(config=True)
batch_query_cmd = Unicode('mdiag -j {job_id} --xml').tag(config=True)
batch_cancel_cmd = Unicode('mjobctl -c {job_id}').tag(config=True)
state_pending_re = Unicode(r'State="Idle"').tag(config=True)
state_running_re = Unicode(r'State="Running"').tag(config=True)
state_exechost_re = Unicode(r'AllocNodeList="([^\r\n\t\f :"]*)').tag(config=True)
Expand Down Expand Up @@ -487,10 +494,10 @@ class SlurmSpawner(UserEnvMixin,BatchSpawnerRegexStates):
{cmd}
""").tag(config=True)
# outputs line like "Submitted batch job 209"
batch_submit_cmd = Unicode('sudo -E -u {username} sbatch --parsable').tag(config=True)
batch_submit_cmd = Unicode('sbatch --parsable').tag(config=True)
# outputs status and exec node like "RUNNING hostname"
batch_query_cmd = Unicode("sudo -E -u {username} squeue -h -j {job_id} -o '%T %B'").tag(config=True) #
batch_cancel_cmd = Unicode('sudo -E -u {username} scancel {job_id}').tag(config=True)
batch_query_cmd = Unicode("squeue -h -j {job_id} -o '%T %B'").tag(config=True) #
batch_cancel_cmd = Unicode('scancel {job_id}').tag(config=True)
# use long-form states: PENDING, CONFIGURING = pending
# RUNNING, COMPLETING = running
state_pending_re = Unicode(r'^(?:PENDING|CONFIGURING)').tag(config=True)
Expand Down Expand Up @@ -532,10 +539,10 @@ class GridengineSpawner(BatchSpawnerBase):
""").tag(config=True)

# outputs job id string
batch_submit_cmd = Unicode('sudo -E -u {username} qsub').tag(config=True)
batch_submit_cmd = Unicode('qsub').tag(config=True)
# outputs job data XML string
batch_query_cmd = Unicode('sudo -E -u {username} qstat -xml').tag(config=True)
batch_cancel_cmd = Unicode('sudo -E -u {username} qdel {job_id}').tag(config=True)
batch_query_cmd = Unicode('qstat -xml').tag(config=True)
batch_cancel_cmd = Unicode('qdel {job_id}').tag(config=True)

def parse_job_id(self, output):
return output.split(' ')[2]
Expand Down Expand Up @@ -582,10 +589,10 @@ class CondorSpawner(UserEnvMixin,BatchSpawnerRegexStates):
""").tag(config=True)

# outputs job id string
batch_submit_cmd = Unicode('sudo -E -u {username} condor_submit').tag(config=True)
batch_submit_cmd = Unicode('condor_submit').tag(config=True)
# outputs job data XML string
batch_query_cmd = Unicode('condor_q {job_id} -format "%s, " JobStatus -format "%s" RemoteHost -format "\n" True').tag(config=True)
batch_cancel_cmd = Unicode('sudo -E -u {username} condor_rm {job_id}').tag(config=True)
batch_cancel_cmd = Unicode('condor_rm {job_id}').tag(config=True)
# job status: 1 = pending, 2 = running
state_pending_re = Unicode(r'^1,').tag(config=True)
state_running_re = Unicode(r'^2,').tag(config=True)
Expand Down Expand Up @@ -618,9 +625,9 @@ class LsfSpawner(BatchSpawnerBase):
''').tag(config=True)


batch_submit_cmd = Unicode('sudo -E -u {username} bsub').tag(config=True)
batch_query_cmd = Unicode('sudo -E -u {username} bjobs -a -noheader -o "STAT EXEC_HOST" {job_id}').tag(config=True)
batch_cancel_cmd = Unicode('sudo -E -u {username} bkill {job_id}').tag(config=True)
batch_submit_cmd = Unicode('bsub').tag(config=True)
batch_query_cmd = Unicode('bjobs -a -noheader -o "STAT EXEC_HOST" {job_id}').tag(config=True)
batch_cancel_cmd = Unicode('bkill {job_id}').tag(config=True)

def get_env(self):
env = super().get_env()
Expand Down
1 change: 1 addition & 0 deletions batchspawner/tests/test_spawners.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
testjob = "12345"

class BatchDummy(BatchSpawnerRegexStates):
exec_prefix = ''
batch_submit_cmd = Unicode('cat > /dev/null; echo '+testjob)
batch_query_cmd = Unicode('echo RUN '+testhost)
batch_cancel_cmd = Unicode('echo STOP')
Expand Down