Skip to content

Commit

Permalink
optimizing openlava / minor other fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@mfiers
mfiers committed Jul 18, 2012
1 parent 3ca63e3 commit 927a447
Show file tree
Hide file tree
Showing 7 changed files with 172 additions and 103 deletions.
50 changes: 22 additions & 28 deletions lib/python/moa/plugin/job/metavar.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,44 +198,38 @@ def hook_pre_filesets(job):

#print i, clean_p, p, cp
if not p: break
#job.conf.setPrivateVar('dir%d' % i, p)
job.conf.setPrivateVar('_%d' % i, p)
job.conf.setPrivateVar('_%s' % clean_p, cp)
job.conf.setPrivateVar('__%s' % clean_p, cp)

#job.conf.setPrivateVar('_%d' % i, p)
#job.conf.setPrivateVar('_%s' % clean_p, cp)
#job.conf.setPrivateVar('__%s' % clean_p, cp)

if i <= 3:
job.conf.setPrivateVar('_' * i, p)

if i > 1 and i <= 3:
thisdirlist = [x for x in os.listdir(cp) if not x[0] == '.']
thisdirlist.sort()

#print i
#print cp
#print p
#print lastp
#print thisdirlist
# if i > 1 and i <= 3:
# thisdirlist = [x for x in os.listdir(cp) if not x[0] == '.']
# thisdirlist.sort()

iofp = thisdirlist.index(lastp)
# iofp = thisdirlist.index(lastp)

job.conf.setPrivateVar('_' + ('f' * (i-1)), thisdirlist[0])
job.conf.setPrivateVar('__' + ('f' * (i-1)),
os.path.join(cp, thisdirlist[0]))
# job.conf.setPrivateVar('_' + ('f' * (i-1)), thisdirlist[0])
# job.conf.setPrivateVar('__' + ('f' * (i-1)),
# os.path.join(cp, thisdirlist[0]))

job.conf.setPrivateVar('_' +('l' * (i-1)), thisdirlist[-1])
job.conf.setPrivateVar('__' +('l' * (i-1)),
os.path.join(cp, thisdirlist[-1]))
# job.conf.setPrivateVar('_' +('l' * (i-1)), thisdirlist[-1])
# job.conf.setPrivateVar('__' +('l' * (i-1)),
# os.path.join(cp, thisdirlist[-1]))

if iofp > 0:
job.conf.setPrivateVar('_' +('p' * (i-1)), thisdirlist[iofp-1])
job.conf.setPrivateVar('__' +('p' * (i-1)),
os.path.join(cp, thisdirlist[iofp-1]))
# if iofp > 0:
# job.conf.setPrivateVar('_' +('p' * (i-1)), thisdirlist[iofp-1])
# job.conf.setPrivateVar('__' +('p' * (i-1)),
# os.path.join(cp, thisdirlist[iofp-1]))


if iofp < (len(thisdirlist)-1):
job.conf.setPrivateVar('_' +('n' * (i-1)), thisdirlist[iofp+1])
job.conf.setPrivateVar('__' +('n' * (i-1)),
os.path.join(cp, thisdirlist[iofp+1]))
# if iofp < (len(thisdirlist)-1):
# job.conf.setPrivateVar('_' +('n' * (i-1)), thisdirlist[iofp+1])
# job.conf.setPrivateVar('__' +('n' * (i-1)),
# os.path.join(cp, thisdirlist[iofp+1]))


lastp = p
Expand Down
115 changes: 66 additions & 49 deletions lib/python/moa/plugin/system/configure.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,65 +58,80 @@ def show(job, args):
isPrivate = False

if p[:4] == 'moa_':
if not args.showAll:
continue
isPrivate = True

if p[0] == '_':
if not args.showPrivate:
continue
isPrivate = True

isPrivate = job.conf.isPrivate(p)
if p[0] == '_': isPrivate = True
if p[:4] == 'moa_': isPrivate = True

#see if this private in the template defintion
if job.conf.isPrivate(p):
if args.showPrivate:
isPrivate = True
else:
continue
isOptional = job.template.parameters[p].get('optional', True)
isLocal = job.conf.is_local(p)
isDefined = len(str(job.conf[p])) > 0

# args.showPrivate:
# args.showAll
# args.showRecursive

if not args.showAll:
if job.template.parameters[p].optional and \
(not job.conf.setInJobConf(p)):
#do not show undefined optional parameters unless -a
#is defined on the command line
continue
if isPrivate and not args.showPrivate:
continue

if args.showRecursive:
#also - no recursively defined stuff - unless it is relevant
#to the current job
if not job.conf.is_local(p):
if not args.showAll:
if not isLocal and isOptional:
continue



outkeys.append(p)

#print '%s "%s" %s' % (p, job.conf[p], isDefined)

key = ''
if not isDefined:
key += '{{gray}}.{{reset}}'
outvals.append('')
elif isLocal:
outvals.append(str(job.conf[p]))
key += '{{bold}}{{green}}l{{reset}}'
else:
outvals.append(str(job.conf[p]))
key += '{{magenta}}r{{reset}}'

#is this variable defined?
if isPrivate:
outflags.append('{{red}}p{{reset}}')
outvals.append(job.conf[p])
elif job.conf.setInJobConf(p):
#yes: locally?
if job.conf.is_local(p):
outflags.append('{{green}}L{{reset}}')
else:
outflags.append('{{magenta}}R{{reset}}')
outvals.append(job.conf[p])
key += 'p'
else:
#not defined - does it need to be??
if job.template.parameters[p].optional:
outflags.append('{{blue}}o{{reset}}')
val = job.conf[p]
if val != None:
outvals.append(
moa.ui.fformat('{{gray}}%s{{reset}}' % val, f='j'))
else:
outvals.append(
moa.ui.fformat(
'{{gray}}(undefined){{reset}}', f='j'))
else:
#not optional
outflags.append('{{bold}}{{red}}E{{reset}}')
outvals.append(
moa.ui.fformat(
'{{red}}{{bold}}(undefined){{reset}}', f='j'))
key += '{{gray}}.{{reset}}'

if isOptional:
key += '{{gray}}o{{reset}}'
else:
key += '{{green}}{{bold}}M{{reset}}'

outflags.append(key)

# #yes: locally?
# if job.conf.is_local(p):
# outflags.append('{{green}}L{{reset}}')
# else:
# outflags.append('{{magenta}}R{{reset}}')
# outvals.append(job.conf[p])
# else:
# #not defined - does it need to be??
# if job.template.parameters[p].optional:
# val = job.conf[p]
# if val != None:
# outvals.append(
# moa.ui.fformat('{{gray}}%s{{reset}}' % val, f='j'))
# else:
# outvals.append(
# moa.ui.fformat(
# '{{gray}}(undefined){{reset}}', f='j'))
# else:
# #not optional
# outflags.append('{{bold}}{{red}}E{{reset}}')
# outvals.append(
# moa.ui.fformat(
# '{{red}}{{bold}}(undefined){{reset}}', f='j'))

maxKeylen = max([len(x) for x in outkeys]) + 1

Expand All @@ -137,6 +152,7 @@ def show(job, args):
key, val, flag = zippy
if not args.showAll:
if not val: continue

moa.ui.fprint(("%%-%ds" % maxKeylen) % key, f='jinja', newline=False)
moa.ui.fprint(" " + flag + " ", f='jinja', newline=False)
if len(str(val)) == 0:
Expand All @@ -160,6 +176,7 @@ def show(job, args):
moa.ui.fprint(ll, f=None)
else:
moa.ui.fprint(spacer + ll, f=None)

if args.showUnrendered and rendered[key] and rendered[key] != val:
for j, ll in enumerate(textwrap.wrap(str(rendered[key]), wrapInit)):
moa.ui.fprint(spacerR + ll + closeR)
Expand Down
74 changes: 56 additions & 18 deletions lib/python/moa/plugin/system/openLavaActor.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ def hook_defineCommandOptions(job, parser):
parser.add_argument( '--oln', default=1, type=int, dest='openlavaSlots',
help='The number of cores the jobs requires')

parser.add_argument( '--oldummy', default=False, dest='openlavaDummy',
action='store_true', help='Do not execute - just create a script to run')

parser.add_argument( '--olm', default=1, dest='openlavaHost',
help='The host to use for openlava')

Expand All @@ -55,9 +58,15 @@ def openlavaRunner(wd, cl, conf={}, **kwargs):
l.critical("runner should be called with a command")
sys.exit(-1)

l.debug("starting openlava actor for %s" % command)
l.debug("starting openlava actor for %s" % command)

# this is a trick to get the real path of the log dir - but not of
# any underlying directory - in case paths are mounted differently
# on different hosts
outDir = os.path.abspath(os.path.join(wd, '.moa', 'log.latest'))
outDir = outDir.rsplit('.moa', 1)[0] + '.moa' + \
os.path.realpath(outDir).rsplit('.moa', 1)[1]

if not os.path.exists(outDir):
try:
os.makedirs(outDir)
Expand All @@ -75,21 +84,26 @@ def s(*cl):
sc.append(" ".join(map(str, cl)))

s("#!/bin/bash")
s("cd", wd)
bsub_cl.extend(["-o", outfile])
bsub_cl.extend(["-e", errfile])
bsub_cl.extend(["-q", sysConf.args.openlavaQueue])
s("#BSUB -o %s" % outfile)
s("#BSUB -e %s" % errfile)
s("#BSUB -q %s" % sysConf.args.openlavaQueue)

#bsub_cl.extend(["-o", outfile])
#bsub_cl.extend(["-e", errfile])
#bsub_cl.extend(["-q", sysConf.args.openlavaQueue])


if '--oln' in sys.argv:
slots = sysConf.args.openlavaSlots
else:
slots = sysConf.job.conf.get('threads', sysConf.args.openlavaSlots)

bsub_cl.extend(["-n", slots])
s("#BSUB -n %d" % slots)
#bsub_cl.extend(["-n", slots])

if '--olm' in sys.argv:
bsub_cl.extend(["-m", sysConf.args.openlavaHost])
s("#BSUB -m %s" % sysConf.args.openlavaHost)
#bsub_cl.extend(["-m", sysConf.args.openlavaHost])

lastJids = []

Expand All @@ -101,30 +115,38 @@ def s(*cl):
#hold until the 'prepare' jobs are done
#l.critical("Prepare jids - wait for these! %s" % prep_jids)
for j in prep_jids:
bsub_cl.extend(["-w", "'done(%d)'" % j])
s("#BSUB -w 'done(%d)'" % j)
#bsub_cl.extend(["-w", "'done(%d)'" % j])
elif command == 'finish':
run_jids = sysConf.job.data.openlava.jids.get('run', [])
#hold until the 'prepare' jobs are done
for j in run_jids:
bsub_cl.extend(["-w", "'done(%d)'" % j])
s("#BSUB -w 'done(%d)'" % j)
#bsub_cl.extend(["-w", "'done(%d)'" % j])

#give it a reasonable name
jobname = "%s/%s" % (str(conf['runid']), command)
if conf.get('runid', None):
bsub_cl.extend(["-J", jobname])
#s("#BSUB-J %s" % jobname)
jobname = ("moa %s in %s" % (command, wd)).replace("'", '"')
#bsub_cl.extend(["-J", jobname])
s("#BSUB -J '%s'" % jobname)

# make sure the environment is copied
#qcl.append('-V')
#qcl.extend(cl)

#print " ".join(qcl)
#dump the configuration in the environment

s("")
s("## ensure we're in the correct directory")
s("cd", wd)


s("")
s("## Defining moa specific environment variables")
s("")

for k in conf:

confkeys = sorted(conf.keys())
for k in confkeys:
# to prevent collusion, prepend all env variables
# with 'moa_'
if k[0] == '_' or k[:3] == 'moa':
Expand All @@ -150,6 +172,21 @@ def s(*cl):

s(*cl)


if sysConf.args.openlavaDummy:
# Dummy mode - do not execute - just write the script.
ii = 0
while True:
outFile = os.path.join(wd, 'openlava.%s.%d.bash' % (command, ii))
if not os.path.exists(outFile): break
ii += 1
with open(outFile, 'w') as F:
F.write("\n".join(sc))
moa.ui.message("Created openlava submit script: %s" % outFile.rsplit('/',1)[1])
moa.ui.message("now run:")
moa.ui.message(" %s < %s" % ((" ".join(map(str, bsub_cl))), outFile.rsplit('/',1)[1]))
return 0

#save the file
tmpdir = os.path.join(wd, '.moa', 'tmp')
if not os.path.exists(tmpdir):
Expand All @@ -164,11 +201,12 @@ def s(*cl):

l.debug("executing bsub")
moa.ui.message("Submitting job to openlava")
bsub_cl.append(tmpfilename)
#bsub_cl.append("<")
#bsub_cl.append(tmpfilename)
moa.ui.message("Executing")
moa.ui.message(" ".join(map(str, bsub_cl)))
p = sp.Popen(map(str, bsub_cl), cwd = wd, stdout=sp.PIPE)
o,e = p.communicate()
p = sp.Popen(map(str, bsub_cl), cwd = wd, stdout=sp.PIPE, stdin = sp.PIPE)
o,e = p.communicate("\n".join(sc))

jid = int(o.split("<")[1].split(">")[0])

Expand Down
9 changes: 7 additions & 2 deletions lib/python/moa/plugin/system/parameterCheck.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,13 @@ def test(job, args):
elif pt.type == 'file' \
and job.conf[p] \
and not os.path.isfile(rconf[p]):
messages.append(("Not a file",
"%s=%s " % (p, rconf[p])))
if pt.get('optional', True):
#ignore this
pass
else:
print
messages.append(("Not a file",
"%s=%s " % (p, rconf[p])))
elif pt.type == 'directory' \
and job.conf[p] \
and not os.path.isdir(rconf[p]):
Expand Down
5 changes: 4 additions & 1 deletion lib/python/moa/ui.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,10 @@ def fformat(message, f='text', newline = True, ansi = None):
if not f:
rt += message
elif f[0].lower() == 't':
rt += message % codes
try:
rt += message % codes
except:
rt += message
elif f[0].lower() == 'j':
try:
template = jenv.from_string(message)
Expand Down

0 comments on commit 927a447

Please sign in to comment.