Skip to content

Commit

Permalink
Merge branch 'moa.0.11' of ssh.github.com:mfiers/Moa into moa.0.11
Browse files Browse the repository at this point in the history
  • Loading branch information
Mark Fiers committed Jul 18, 2012
2 parents 75e1011 + 927a447 commit fcd7ca8
Show file tree
Hide file tree
Showing 17 changed files with 238 additions and 133 deletions.
3 changes: 2 additions & 1 deletion bin/moa
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,8 @@ def run_2(force_silent=False):

sysConf.pluginHandler.run('prepare_recursion')

wd = os.getcwd()
wd = moa.utils.getCwd()
l.debug("starting run_2 in %s" % wd)
if not '-r' in sys.argv:
#not recursive - go directly to stage 3
try:
Expand Down
14 changes: 10 additions & 4 deletions bin/moar
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,11 @@ base = os.path.abspath(os.getcwd())
parser = argparse.ArgumentParser(description='run recursively')
parser.add_argument('--bg', dest='background', action='store_true',
default=False,
help='run in the background')
help='run each job in the background')
parser.add_argument('-d', dest='depth', default=0, help='depth: 1 means run only in first ' +
'level subdirs, 2 only in second level, and so on', type=int)
parser.add_argument('-t', dest='test', default=False,
help='test run', action='store_true')
parser.add_argument('command', nargs='+')
args = parser.parse_args()

Expand All @@ -31,7 +33,11 @@ for path, dirs, files in os.walk(base):

if args.depth and args.depth != level:
continue
print 'executing in %s' % localpath

P = sp.Popen(cl, cwd=path, shell=True)
P.wait()
if args.test:
print 'would executed in', localpath
print ' ', cl
else:
print 'executing in %s' % localpath
P = sp.Popen(cl, cwd=path, shell=True)
P.wait()
4 changes: 2 additions & 2 deletions lib/python/moa/backend/ruff/executor.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ def ruffusExecutor(input, output, script, jobData):
sysConf.actor.files_processed.append((input, output))

wd = jobData['wd']
tmpdir = os.path.realpath(os.path.abspath(
os.path.join(wd, '.moa', 'tmp')))
tmpdir = os.path.abspath(
os.path.join(wd, '.moa', 'tmp'))
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)

Expand Down
4 changes: 3 additions & 1 deletion lib/python/moa/job.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,6 @@ class Job(object):
def __init__(self, wd):
"""
:param wd: The directory containing the job
:param register: Register this job in the sysConf? Usually this should be yes
"""

#prepare the plugins
Expand All @@ -108,6 +107,9 @@ def __init__(self, wd):
wd = wd[:-1]
self.wd = wd


l.debug('Creating job with wd %s' % self.wd)

self.confDir = os.path.join(self.wd, '.moa')
self.templateFile = os.path.join(self.confDir, 'template')
self.templateMetaFile = os.path.join(self.confDir, 'template.d', 'meta')
Expand Down
6 changes: 3 additions & 3 deletions lib/python/moa/jobConf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ def _checkJobId(self):

jobid = self['jobid']
if jobid != 'unset': return
name = os.path.basename(os.getcwd())
name = os.path.basename(moa.utils.getCwd())
name = re.sub("^[0-9]+\.+", "", name)
l.debug("Setting job id to '%s'" % name)
self['jobid'] = name
Expand Down Expand Up @@ -287,8 +287,8 @@ def load(self, confFile, delta=None):

normdelta = os.path.normpath(delta)

if y.has_key('jobid'):
self.setPrivateVar('_%s' % y['jobid'], normdelta)
#if y.has_key('jobid'):
# self.setPrivateVar('_%s' % y['jobid'], normdelta)

#print self.job.template.parameters.keys()
#find relative links & see if they need to be adjusted
Expand Down
53 changes: 24 additions & 29 deletions lib/python/moa/plugin/job/metavar.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@

from moa.sysConf import sysConf
import moa.logger
import moa.utils

l = moa.logger.getLogger(__name__)
#l.setLevel(moa.logger.DEBUG)
Expand All @@ -131,7 +132,7 @@ class MoaPathParser(Extension):
tags = set(['mp'])

def processDirString(self, s):
cwd = os.getcwd()
cwd = moa.utils.getCwd()
return frappant.frp(cwd, s)
# l.critical(cwd)

Expand Down Expand Up @@ -197,44 +198,38 @@ def hook_pre_filesets(job):

#print i, clean_p, p, cp
if not p: break
#job.conf.setPrivateVar('dir%d' % i, p)
job.conf.setPrivateVar('_%d' % i, p)
job.conf.setPrivateVar('_%s' % clean_p, cp)
job.conf.setPrivateVar('__%s' % clean_p, cp)

#job.conf.setPrivateVar('_%d' % i, p)
#job.conf.setPrivateVar('_%s' % clean_p, cp)
#job.conf.setPrivateVar('__%s' % clean_p, cp)

if i <= 3:
job.conf.setPrivateVar('_' * i, p)

if i > 1 and i <= 3:
thisdirlist = [x for x in os.listdir(cp) if not x[0] == '.']
thisdirlist.sort()

#print i
#print cp
#print p
#print lastp
#print thisdirlist
# if i > 1 and i <= 3:
# thisdirlist = [x for x in os.listdir(cp) if not x[0] == '.']
# thisdirlist.sort()

iofp = thisdirlist.index(lastp)
# iofp = thisdirlist.index(lastp)

job.conf.setPrivateVar('_' + ('f' * (i-1)), thisdirlist[0])
job.conf.setPrivateVar('__' + ('f' * (i-1)),
os.path.join(cp, thisdirlist[0]))
# job.conf.setPrivateVar('_' + ('f' * (i-1)), thisdirlist[0])
# job.conf.setPrivateVar('__' + ('f' * (i-1)),
# os.path.join(cp, thisdirlist[0]))

job.conf.setPrivateVar('_' +('l' * (i-1)), thisdirlist[-1])
job.conf.setPrivateVar('__' +('l' * (i-1)),
os.path.join(cp, thisdirlist[-1]))
# job.conf.setPrivateVar('_' +('l' * (i-1)), thisdirlist[-1])
# job.conf.setPrivateVar('__' +('l' * (i-1)),
# os.path.join(cp, thisdirlist[-1]))

if iofp > 0:
job.conf.setPrivateVar('_' +('p' * (i-1)), thisdirlist[iofp-1])
job.conf.setPrivateVar('__' +('p' * (i-1)),
os.path.join(cp, thisdirlist[iofp-1]))
# if iofp > 0:
# job.conf.setPrivateVar('_' +('p' * (i-1)), thisdirlist[iofp-1])
# job.conf.setPrivateVar('__' +('p' * (i-1)),
# os.path.join(cp, thisdirlist[iofp-1]))


if iofp < (len(thisdirlist)-1):
job.conf.setPrivateVar('_' +('n' * (i-1)), thisdirlist[iofp+1])
job.conf.setPrivateVar('__' +('n' * (i-1)),
os.path.join(cp, thisdirlist[iofp+1]))
# if iofp < (len(thisdirlist)-1):
# job.conf.setPrivateVar('_' +('n' * (i-1)), thisdirlist[iofp+1])
# job.conf.setPrivateVar('__' +('n' * (i-1)),
# os.path.join(cp, thisdirlist[iofp+1]))


lastp = p
Expand Down
119 changes: 68 additions & 51 deletions lib/python/moa/plugin/system/configure.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,65 +58,80 @@ def show(job, args):
isPrivate = False

if p[:4] == 'moa_':
if not args.showAll:
continue
isPrivate = True

if p[0] == '_':
if not args.showPrivate:
continue
isPrivate = True

isPrivate = job.conf.isPrivate(p)
if p[0] == '_': isPrivate = True
if p[:4] == 'moa_': isPrivate = True

#see if this private in the template defintion
if job.conf.isPrivate(p):
if args.showPrivate:
isPrivate = True
else:
continue
isOptional = job.template.parameters[p].get('optional', True)
isLocal = job.conf.is_local(p)
isDefined = len(str(job.conf[p])) > 0

# args.showPrivate:
# args.showAll
# args.showRecursive

if not args.showAll:
if job.template.parameters[p].optional and \
(not job.conf.setInJobConf(p)):
#do not show undefined optional parameters unless -a
#is defined on the command line
continue
if isPrivate and not args.showPrivate:
continue

if args.showRecursive:
#also - no recursively defined stuff - unless it is relevant
#to the current job
if not job.conf.is_local(p):
if not args.showAll:
if not isLocal and isOptional:
continue



outkeys.append(p)

#print '%s "%s" %s' % (p, job.conf[p], isDefined)

key = ''
if not isDefined:
key += '{{gray}}.{{reset}}'
outvals.append('')
elif isLocal:
outvals.append(str(job.conf[p]))
key += '{{bold}}{{green}}l{{reset}}'
else:
outvals.append(str(job.conf[p]))
key += '{{magenta}}r{{reset}}'

#is this variable defined?
if isPrivate:
outflags.append('{{red}}p{{reset}}')
outvals.append(job.conf[p])
elif job.conf.setInJobConf(p):
#yes: locally?
if job.conf.is_local(p):
outflags.append('{{green}}L{{reset}}')
else:
outflags.append('{{magenta}}R{{reset}}')
outvals.append(job.conf[p])
key += 'p'
else:
#not defined - does it need to be??
if job.template.parameters[p].optional:
outflags.append('{{blue}}o{{reset}}')
val = job.conf[p]
if val != None:
outvals.append(
moa.ui.fformat('{{gray}}%s{{reset}}' % val, f='j'))
else:
outvals.append(
moa.ui.fformat(
'{{gray}}(undefined){{reset}}', f='j'))
else:
#not optional
outflags.append('{{bold}}{{red}}E{{reset}}')
outvals.append(
moa.ui.fformat(
'{{red}}{{bold}}(undefined){{reset}}', f='j'))
key += '{{gray}}.{{reset}}'

if isOptional:
key += '{{gray}}o{{reset}}'
else:
key += '{{green}}{{bold}}M{{reset}}'

outflags.append(key)

# #yes: locally?
# if job.conf.is_local(p):
# outflags.append('{{green}}L{{reset}}')
# else:
# outflags.append('{{magenta}}R{{reset}}')
# outvals.append(job.conf[p])
# else:
# #not defined - does it need to be??
# if job.template.parameters[p].optional:
# val = job.conf[p]
# if val != None:
# outvals.append(
# moa.ui.fformat('{{gray}}%s{{reset}}' % val, f='j'))
# else:
# outvals.append(
# moa.ui.fformat(
# '{{gray}}(undefined){{reset}}', f='j'))
# else:
# #not optional
# outflags.append('{{bold}}{{red}}E{{reset}}')
# outvals.append(
# moa.ui.fformat(
# '{{red}}{{bold}}(undefined){{reset}}', f='j'))

maxKeylen = max([len(x) for x in outkeys]) + 1

Expand All @@ -137,6 +152,7 @@ def show(job, args):
key, val, flag = zippy
if not args.showAll:
if not val: continue

moa.ui.fprint(("%%-%ds" % maxKeylen) % key, f='jinja', newline=False)
moa.ui.fprint(" " + flag + " ", f='jinja', newline=False)
if len(str(val)) == 0:
Expand All @@ -160,6 +176,7 @@ def show(job, args):
moa.ui.fprint(ll, f=None)
else:
moa.ui.fprint(spacer + ll, f=None)

if args.showUnrendered and rendered[key] and rendered[key] != val:
for j, ll in enumerate(textwrap.wrap(str(rendered[key]), wrapInit)):
moa.ui.fprint(spacerR + ll + closeR)
Expand Down Expand Up @@ -219,12 +236,12 @@ def set(job, args):
val = moa.ui.askUser("%s:\n> " % a, old)
job.conf[a] = val
new_pars.append((a,val))
moa.ui.message('set "%s" to "%s"' % (a, " ".join(val.split())[:80]))
moa.ui.message('set "%s" to "%s"' % (a, " ".join(val.split())))
else:
key,val = a.split('=',1)
job.conf[key] = val
new_pars.append((key,val))
moa.ui.message('set "%s" to "%s"' % (key, " ".join(val.split())[:80]))
moa.ui.message('set "%s" to "%s"' % (a, " ".join(val.split())))

job.conf.save()

Expand Down
3 changes: 2 additions & 1 deletion lib/python/moa/plugin/system/metavar.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,12 +110,13 @@
from moa.sysConf import sysConf
import moa.logger as l
import moa.ui
import moa.utils

class MoaPathParser(Extension):
tags = set(['mp'])

def processDirString(self, s):
cwd = os.getcwd()
cwd = moa.utils.getCwd()
return frappant.frp(cwd, s)
# l.critical(cwd)

Expand Down

0 comments on commit fcd7ca8

Please sign in to comment.