Merge branch 'moa.0.11' of ssh.github.com:mfiers/Moa into moa.0.11

moa/backend/ruff/executor.py

@@ -1,6 +1,7 @@
 import os
 import stat
 import tempfile
+import ruffus 
 
 import moa.actor
 

moa/backend/ruff/map.py

@@ -141,13 +141,12 @@ def generate_data_map():
             #caught here.
             l.debug("Caught a Ruffus error!")
             l.debug(str(e))
-            startOfError = "{{gray}}" + re.sub(r'\s+', " ", str(e))[:72].strip() + "...{{reset}}"
-            endOfError = "{{gray}}..." + re.sub(r'\s+', " ", str(e)).strip()[-72:] + \
-                         "{{reset}}"
-            moa.ui.error("Caught a Ruffus error")
-            moa.ui.error(startOfError)
-            print endOfError
-            moa.ui.error(endOfError.replace('%', '%%'))
+            #startOfError = "{{gray}}" + re.sub(r'\s+', " ", str(e))[:72].strip() + "...{{reset}}"
+            #endOfError = "{{gray}}..." + re.sub(r'\s+', " ", str(e)).strip()[-72:] + \
+            #             "{{reset}}"
+            moa.ui.error("Caught an execution error")
+            #moa.ui.error(startOfError)
+            #moa.ui.error(endOfError.replace('%', '%%'))
             try:
                 #try to get some structured info & output that.
                 einfo = e[0][1].split('->')[0].split('=')[1].strip()

moa/cli/setstatus.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import argparse
+import subprocess as sp
+
+parser = argparse.ArgumentParser(description='set moa status, needs to ' + 
+                                 'be executed in a moa job directory')
+
+parser.add_argument('status', help='status to set')
+args = parser.parse_args()
+
+
+def setstatus():
+
+    if not os.path.exists('.moa') and os.path.isdir('.moa'):
+        sys.stderr.write("must be executed in a moa job directory\n")
+        sys.exit(-1)
+
+    statusFile = os.path.join('.moa', 'status')
+    pidFile = os.path.join('.moa', 'pid')
+
+    status = args.status.lower()
+
+    if not status in 'waiting running success error interrupted'.split():
+        sys.stderr.write('invalid status: %s\n' % status)
+        sys.exit(-1)
+
+    if status != 'run':
+        if os.path.exists(pidFile):
+            os.unlink(pidFile)
+
+    with open(statusFile, 'w') as F:
+        F.write(status)
+
+
+if __name__ == '__main__':
+    setstatus()

moa/plugin/job/task.py

@@ -47,7 +47,7 @@ def _getTaskId(job):
 
     pidlist = out.strip().split()
     if len(pidlist) > 1:
-        moa.ui.warn("multiple jobs with the warn")
+        #moa.ui.warn("multiple jobs with this job's uid - not good")
         return pidlist[-1]
 
     if len(pidlist) == 1:
@@ -75,7 +75,7 @@ def _getTaskId(job):
 def hook_pre_run(job):
 
     taskid = _getTaskId(job)
-    moa.ui.message("TaskId %s" % taskid)
+    #moa.ui.message("TaskId %s" % taskid)
     hostname = socket.gethostname()
 
     runTask("task %s modify +moa +%s" % (taskid, hostname))

moa/plugin/system/openLavaActor.py

@@ -16,6 +16,8 @@
 import sys
 import tempfile
 
+
+import jinja2
 import moa.logger
 import moa.ui
 from moa.sysConf import sysConf
@@ -35,7 +37,7 @@ def hook_defineCommandOptions(job, parser):
     parser.add_argument('--olx', default='', dest='openlavaExtra',
                         help='Extra arguments for bsub')
 
-    parser.add_argument('--oln', default=1, type=int, dest='openlavaSlots',
+    parser.add_argument('--olC', default=1, type=int, dest='openlavaCores',
                         help='The number of cores the jobs requires')
 
     parser.add_argument('--oldummy', default=False, dest='openlavaDummy',
@@ -94,17 +96,12 @@ def s(*cl):
     s("#BSUB -e %s" % errfile)
     s("#BSUB -q %s" % sysConf.args.openlavaQueue)
 
-    #bsub_cl.extend(["-o", outfile])
-    #bsub_cl.extend(["-e", errfile])
-    #bsub_cl.extend(["-q", sysConf.args.openlavaQueue])
-
     if '--oln' in sys.argv:
-        slots = sysConf.args.openlavaSlots
+        cores = sysConf.args.openlavaCores
     else:
-        slots = sysConf.job.conf.get('threads', sysConf.args.openlavaSlots)
+        cores = sysConf.job.conf.get('threads', sysConf.args.openlavaCores)
 
-    s("#BSUB -n %d" % slots)
-    #bsub_cl.extend(["-n", slots])
+    s("#BSUB -C %d" % cores)
 
     if sysConf.args.openlavaExtra.strip():
         s("#BSUB %s" % sysConf.args.openlavaExtra)
@@ -201,15 +198,14 @@ def s(*cl):
     os.chmod(tmpfile.name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
 
     l.debug("executing bsub")
-    moa.ui.message("Submitting job to openlava")
-    moa.ui.message("Executing")
-    moa.ui.message(" ".join(map(str, bsub_cl)))
 
     p = sp.Popen(map(str, bsub_cl), cwd=wd, stdout=sp.PIPE, stdin=sp.PIPE)
     o, e = p.communicate("\n".join(sc))
 
     jid = int(o.split("<")[1].split(">")[0])
 
+    moa.ui.message("Submitted a job to openlava with id %d" % jid)
+
     if not sysConf.job.data.openlava.jids.get(command):
         sysConf.job.data.openlava.jids[command] = []
 
@@ -225,6 +221,16 @@ def s(*cl):
     l.debug("jids stored %s" % str(sysConf.job.data.openlava.jids))
     return p.returncode
 
+OnSuccessScript = """
+#BSUB -w '({%- for j in jids -%}
+{%- if loop.index0 > 0 %}&&{% endif -%}
+done({{j}})
+{%- endfor -%})'
+"""
+
+OnErrorScript = """
+
+"""
 
 def hook_postRun():
     """
@@ -234,6 +240,7 @@ def hook_postRun():
         with open('jidlist', 'w') as F:
             F.write("\n".join(
                 map(str, sysConf.job.data.openlava.get('alljids'))))
+
 
 #register this actor globally
 sysConf.actor.actors['openlava'] = openlavaRunner

moa/plugin/system/project.py

@@ -104,7 +104,7 @@ def hook_prepare_3():
     while True:
 
         if lookat == '/':
-            moa.ui.warn("No project found")
+            #moa.ui.warn("No project found")
             break
 
         templateFile = os.path.join(lookat, '.moa', 'template')

moa/sysConf.py

@@ -14,7 +14,13 @@
 """
 
 import os
-import pkg_resources
+import warnings
+
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore")
+    import pkg_resources
+
+
 import Yaco
 
 import moa.logger

setup.py

@@ -9,6 +9,7 @@
     'console_scripts': [
         'moa = moa.cli.main:dispatch',
         'moaprompt = moa.cli.moaprompt:moaprompt',
+        'moasetstatus = moa.cli.setstatus:setstatus',
         'moar = moa.cli.moar:moar',
         'moainit = moa.cli.moainit:moainit',
     ]}

util/test_init

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-MB=/home/mf/project/moa
-
+MB=$1
+echo "base $1"
 export PATH=${MB}/bin:$PATH
 export PYTHONPATH=${MB}:$PYTHONPATH