Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

sge_queuer - improve the way jobs are run

Ignore-this: cf717a9c7fd571bb724a6642725ce6cc

darcs-hash:20130718192534-3a4db-66b2ffa09ab91fe14593c5a914063a8d984f7c4d
  • Loading branch information...
commit 4e213e044654478637e84ec3cd941eecb1176822 1 parent 915e090
@MattShannon authored
Showing with 25 additions and 15 deletions.
  1. +25 −15 bisque/sge_queuer.py
View
40 bisque/sge_queuer.py
@@ -21,7 +21,6 @@
import os
import sys
from bisque import subprocesshelp
-import inspect
@codeDeps(qr.Queuer)
class SgeQueuer(qr.Queuer):
@@ -42,11 +41,15 @@ def submitOne(self, job, live, verbosity):
@codeDeps(SgeQueuer, mock_sge.getMockSge, persist.secHashObject, sge_runner)
class MockSgeQueuer(SgeQueuer):
- def __init__(self, buildRepo, jointLog = False, pythonExec = '/usr/bin/python'):
+ def __init__(self, buildRepo, jointLog = False, pythonExec = None):
self.buildRepo = buildRepo
self.jointLog = jointLog
self.pythonExec = pythonExec
+ if self.pythonExec is None:
+ self.pythonExec = sys.executable
+ assert self.pythonExec
+
def __enter__(self):
self.submitServer, self.runServerProcess = mock_sge.getMockSge()
self.secHashUid = persist.secHashObject((
@@ -60,7 +63,7 @@ def __exit__(self, exc_type, exc_value, traceback):
def qsub(self, jobName, liveJobDir, parentJids, verbosity):
args = [
- inspect.getsourcefile(sge_runner),
+ '-m', 'bisque.sge_runner',
liveJobDir
]
@@ -90,7 +93,8 @@ def qsub(self, jobName, liveJobDir, parentJids, verbosity):
subprocesshelp.check_output
)
class QsubSgeQueuer(SgeQueuer):
- def __init__(self, buildRepo, project, email = None, emailOpts = 'n', jointLog = False, pythonExec = '/usr/bin/python'):
+ def __init__(self, buildRepo, project, email = None, emailOpts = 'n',
+ jointLog = False, pythonExec = None):
self.buildRepo = buildRepo
self.project = project
self.email = email
@@ -98,6 +102,10 @@ def __init__(self, buildRepo, project, email = None, emailOpts = 'n', jointLog =
self.jointLog = jointLog
self.pythonExec = pythonExec
+ if self.pythonExec is None:
+ self.pythonExec = sys.executable
+ assert self.pythonExec
+
# attempt to find name of cell as a way of identifying this grid
self.cell = os.environ.get('GE_CELL', os.environ.get('SGE_CELL', 'default'))
self.secHashUid = persist.secHashObject((self.buildRepo, self.cell, self.project))
@@ -109,13 +117,11 @@ def __init__(self, buildRepo, project, email = None, emailOpts = 'n', jointLog =
raise RuntimeError('grid not found (running qstat failed)')
def qsub(self, jobName, liveJobDir, parentJids, verbosity):
- # N.B. SGE copies sge_runner.py to a spool directory. This means
- # that if we somehow manage to call this module without PYTHONPATH
- # being set (e.g. if we call it from a module defined in the parent
- # directory of 'armspeech') then we still need to set PYTHONPATH when
- # running the job (and sys.path[0] is just a decent heuristic guess).
- # FIXME : does sys.path[0] ever do anything here? Should just fail instead?
- envPythonPath = os.environ['PYTHONPATH'] if 'PYTHONPATH' in os.environ else sys.path[0]
+ env = dict()
+ if 'PYTHONPATH' in os.environ:
+ env['PYTHONPATH'] = os.environ['PYTHONPATH']
+ env['PYTHONUNBUFFERED'] = 'yes'
+
args = [
'qsub',
'-N', jobName,
@@ -124,17 +130,21 @@ def qsub(self, jobName, liveJobDir, parentJids, verbosity):
] + (
[] if self.email is None else ['-M', self.email]
) + [
- '-S', self.pythonExec,
- '-v', 'PYTHONPATH='+envPythonPath,
- '-v', 'PYTHONUNBUFFERED=yes',
+ # specify that command to run is a binary executable, not a script
+ '-b', 'y',
'-cwd'
+ ] + [
+ token
+ for key, value in env.items()
+ for token in ['-v', '%s=%s' % (key, value)]
] + (
['-hold_jid', ','.join(parentJids)] if parentJids else []
) + [
'-j', 'y' if self.jointLog else 'n',
'-o', liveJobDir,
'-e', liveJobDir,
- inspect.getsourcefile(sge_runner),
+ self.pythonExec,
+ '-m', 'bisque.sge_runner',
liveJobDir
]
Please sign in to comment.
Something went wrong with that request. Please try again.