minor cleanups

bin/moa

@@ -56,6 +56,7 @@ if sys.version_info < (2, 6):
 
 from moa.sysConf import sysConf
 
+
 # moa specific libs
 import moa.ui
 import moa.job

lib/python/moa/plugin/job/versioning.py

@@ -58,8 +58,6 @@ def _get_specific_versioninfo(run_id):
                 l.warning(line.strip())
                 continue
             vd[ls[0]] = ls[1].strip()
-    from pprint import pprint
-    pprint(vd)
     return vd
 
 

lib/python/moa/plugin/system/doc/pelican_util.py

@@ -1,4 +1,3 @@
-
 """
 Functions to support pelican
 """

lib/python/moa/plugin/system/logger.py

@@ -1,10 +1,10 @@
 # Copyright 2009-2011 Mark Fiers
 # The New Zealand Institute for Plant & Food Research
-# 
+#
 # This file is part of Moa - http://github.com/mfiers/Moa
-# 
+#
 # Licensed under the GPL license (see 'COPYING')
-# 
+#
 """
 **logger** - Log Moa activity
 -----------------------------
@@ -22,16 +22,15 @@
 import moa.plugin
 from moa.sysConf import sysConf
 
-def hook_prepareCommand():
-    moa.ui.message('Start "%s"' % sysConf.originalCommand)
-
+
 def hook_prepare_3():
     sysConf.logger.start_time = datetime.today()
 
+
 def niceRunTime(d):
     """
     Nice representation of the run time
-    d is time duration string    
+    d is time duration string
     """
     d = str(d)
     if ',' in d:
@@ -45,13 +44,13 @@ def niceRunTime(d):
     seconds, miliseconds = seconds.split('.')
     seconds = int(seconds)
     miliseconds = int(miliseconds)
-    
+
     if days > 0:
-        if days == 1:            
+        if days == 1:
             return "1 day, %d hrs" % hours
         else:
             return "%d days, %d hrs" % (days, hours)
-        
+
     if hours == 0 and minutes == 0 and seconds == 0:
         return "<1 sec"
     if hours > 0:
@@ -61,55 +60,64 @@ def niceRunTime(d):
     else:
         return "%d sec" % seconds
 
+
 def hook_post_interrupt():
     _writeLog('interrupted')
 
+
 def hook_post_error():
-    # if sysConf.logger.run_time:
-    #     moa.ui.message("{{red}}{{bold}}Error{{reset}} running %s (%s)" % (
-    #         sysConf.originalCommand,
-    #         niceRunTime(str(sysConf.logger.run_time))))
-    # else:
-    #     moa.ui.message("{{red}}{{bold}}Error{{reset}} running %s" % (
-    #         sysConf.originalCommand))
     _writeLog('error')
 
+
 def hook_postRun():
     _writeLog('ok')
 
+
 def hook_postNew():
     _writeLog('new')
 
+
 def hook_postCp():
     _writeLog('cp')
 
+
 def hook_postMv():
     _writeLog('mv')
 
+
 def hook_postLock():
     _writeLog('lock')
 
+
 def hook_postUnlock():
     _writeLog('unlock')
 
+
 def hook_preRun():
     _writeLog('start')
 
+
 def _writeLog(status):
-    
+
     #only save logs for moa jobs
-    if (not sysConf) or (not sysConf.has_key('job')) or (not sysConf.job.isMoa()):
+    if (not sysConf) or \
+       (not 'job' in sysConf) or \
+       (not sysConf.job.isMoa()):
         return
 
     sysConf.logger.end_time = datetime.today()
     if sysConf.logger.start_time:
-        sysConf.logger.run_time = sysConf.logger.end_time - sysConf.logger.start_time
+        sysConf.logger.run_time = \
+            sysConf.logger.end_time - sysConf.logger.start_time
     else:
         sysConf.logger.start_time = 0
-    runtime = sysConf.logger.end_time - sysConf.logger.start_time    
+
+    runtime = sysConf.logger.end_time - sysConf.logger.start_time
     sysConf.logger.niceRunTime = niceRunTime(runtime)
 
     logFile = os.path.join(sysConf.job.confDir, 'log')
+    #logFile2 = os.path.join(sysConf.job.confDir, 'log.d',
+    #                        'log.%d' % sysConf.runId)
 
     commandInfo = {}
     logLevel = logging.INFO
@@ -124,23 +132,23 @@ def _writeLog(status):
     else:
         runtime = 0
 
-
     sysConf.logger.logLevel = logLevel
     sysConf.logger.status = status
     sysConf.logger.full_command = command
     sysConf.logger.moa_command = sysConf.originalCommand
     sysConf.pluginHandler.run('logMessage')
-
+
+    #sysConf.logger.save(logFile2)
     with open(logFile, 'a') as F:
         F.write("%s\n" % "\t".join([
             status, str(sysConf.originalCommand),
             str(logLevel),
             sysConf.logger.start_time.strftime("%Y-%m-%dT%H:%M:%S.%f"),
             sysConf.logger.end_time.strftime("%Y-%m-%dT%H:%M:%S.%f"),
             str(runtime), command
-            ]))
+        ]))
+
 
-#@moa.utils.printstack
 def hook_finish():
     #and - probably not the location to do this, but print something to screen
     #as well
@@ -153,23 +161,49 @@ def hook_finish():
                 niceRunTime(str(sysConf.logger.run_time))))
 
 
+def hook_pelican():
+    l.critical("logger pelican hook")
+    return
+    l.debug("pelican versioning output")
+    nov = 10
+    last_run_id, vinfo = _get_last_versioninfo(nov)
+    vrange = list(reversed(sorted(range(last_run_id, last_run_id - nov, -1))))
+    if last_run_id == 0 or vinfo == {}:
+        l.warning("cannot create version info page")
+        return
+
+    allkeys = set()
+    for rid in vinfo:
+        allkeys.update(set(vinfo[rid]))
+    allkeys = sorted(list(allkeys))
+
+    jenv = sysConf.plugins.pelican.jenv
+    jtemplate = jenv.select_template(['versioning.page.jinja2'])
+    with open('./doc/pages/version.md', 'w') as F:
+        F.write(jtemplate.render({
+            'last_run_id': last_run_id,
+            'vrange': vrange,
+            'allkeys': allkeys,
+            'vinfo': vinfo}))
+
+
 @moa.args.needsJob
 @moa.args.command
 def log(job, args):
     """
     Show activity log
 
     Shows a log of moa commands executed. Only commands with an impact
-    on the pipeline are logged, such as `moa run` & `moa set`. 
+    on the pipeline are logged, such as `moa run` & `moa set`.
     """
-    
+
     noLines = 10
     logFile = os.path.join(job.confDir, 'log')
 
     moa.utils.moaDirOrExit(job)
     if not os.path.exists(logFile):
         moa.ui.exit("No logs found")
-   
+
     with open(logFile) as F:
         #read the last 2k - prevent reading the whole file
         try:
@@ -185,23 +219,27 @@ def _isInteger(_s):
                 try:
                     int(_s)
                     return True
-                except: return False
-
-            if (len(ls) != 7) or (_isInteger(ls[0])) or \
-                   (not _isInteger(ls[2])):
+                except:
+                    return False
+
+            if (len(ls) != 7) or \
+               (_isInteger(ls[0])) or \
+               (not _isInteger(ls[2])):
+
                 continue
-            
+
             status, command, logLevel, start, stop, delta, command = \
-                    line.split("\t")
+                line.split("\t")
+
             logLevel = int(logLevel)
             if status == 'ok':
                 lc = '{{bold}}{{green}}Success {{reset}}'
             elif status == 'error':
                 lc = "{{bold}}{{red}}Error   {{reset}}"
             else:
                 lc = "{{blue}}%-8s{{reset}}" % status[:7].capitalize()
-                
-            lc += "%s " % start.rsplit(':',1)[0]
+
+            lc += "%s " % start.rsplit(':', 1)[0]
             lc += "%10s " % niceRunTime(delta)
             lc += command
             moa.ui.fprint(lc, f='jinja')

lib/python/moa/sysConf.py

@@ -30,14 +30,22 @@ class SysConf(Yaco.Yaco):
 
 
     def __init__(self):
-
-
+
         super(SysConf, self).__init__(moa.resources.getResource('etc/config'))
 
         l.debug("Loading system config: %s" % USERCONFIGFILE)
         if os.path.exists(USERCONFIGFILE):
             self.load(USERCONFIGFILE)
-
+
+        #assign a runId
+        runid = '.moa/last_run_id'
+        if os.path.exists(runid):
+            lri = open(runid).read().strip()
+        else:
+            lri = 1
+
+
+
     def getVersion(self):
         """
         Return the version number of this Moa instance

template2/ncbi.moa

@@ -1,8 +1,7 @@
 author: Mark Fiers
 backend: ruff
 prerequisites:
-  xml_grep: 
-    version: 'xml_grep --version'
+  xml_grep: 'xml_grep --version 2>&1'
 commands:
   clean:
     help: Remove all job data, not the Moa job itself, note that this must be implemented