sysConf is now used to store all (or almost all) invocation and syste…

…m related vars - leaving only job related info in the job object -> this is preparing for recursive invocation

bin/moa

@@ -39,7 +39,6 @@ import moa.commands
 
 from moa.sysConf import sysConf
 
-
 ## Initialize the logger
 import moa.logger as l
 
@@ -48,7 +47,7 @@ if '-v' in sys.argv:
     moa.logger.setVerbose()
 
 ## Initialze the plugins
-plugins = moa.plugin.PluginHandler(sysConf.getPlugins())
+plugins = moa.plugin.PluginHandler(sysConf)
 
 
 ### Determine project root (if there is one)
@@ -205,7 +204,7 @@ if __name__ == "__main__":
         plugins.run('defineOptions')
 
         job = moa.job.Job(wd)
-
+        job.sysConf = sysConf
         ## Aks the job & backend if they want to add to the options
         job.defineOptions(parser)
         plugins.register(job = job)
@@ -215,8 +214,9 @@ if __name__ == "__main__":
 
         ## make sure that the options are accessible to the plugins
         plugins.register(options = options, args = args)
-        job.options = options
-        job.args = args
+
+        #job.options = options
+        #job.args = args
 
         ## Proper setting of verbosity - after parsing of the command line
         if options.verbose:
@@ -231,7 +231,7 @@ if __name__ == "__main__":
             newargs
 
         plugins.register(newargs = newargs)
-        job.args = newargs
+        #job.args = newargs
 
         plugins.register(originalCommand=command)
 

lib/python/moa/backend/gnumake.py

@@ -13,6 +13,7 @@
 import moa.backend
 import moa.sysConf
 import moa.logger as l
+from moa.sysConf import sysConf
 
 class Gnumake(moa.backend.BaseBackend):
 
@@ -24,14 +25,6 @@ def prepare(self):
         self.job.makeArgs = getattr(self.job, 'makeArgs', [])
         self.job.env = getattr(self.job, 'env', {})
 
-        ## Define extra parameters to use with Make
-        if getattr(self.job.options, 'remake', False):
-            self.job.makeArgs.append('-B')
-        if getattr(self.job.options, 'makedebug', False):
-            self.job.makeArgs.append('-d')
-
-        self.job.makeArgs.extend(self.job.args)
-
     def hasCommand(self, command):
         """
         No way of finding out if this self.job exists (well, we might
@@ -43,6 +36,12 @@ def execute(self, command, **options):
         """
         Execute!
         """
+        ## Define extra parameters to use with Make
+        if getattr(sysConf.options, 'remake', False):
+            self.job.makeArgs.append('-B')
+        if getattr(sysConf.options, 'makedebug', False):
+            self.job.makeArgs.append('-d')
+
         #self.job.plugins.run("readFilesets")
 
         verbose = options.get('verbose', False)
@@ -53,17 +52,17 @@ def execute(self, command, **options):
         ## different from the other parameters. multi-threaded
         ## operation is only allowed in the second phase of execution.
 
-        if getattr(self.job.options, 'threads', False):
-            self.job.env['MOA_THREADS'] = "%s" % self.job.options.threads
+        if getattr(sysConf.options, 'threads', False):
+            self.job.env['MOA_THREADS'] = "%s" % sysConf.options.threads
         else:
             self.job.env['MOA_THREADS'] = "1"
 
         self.job.env['MOA_TEMPLATE'] = "%s" % self.job.template.name
         self.job.env['moa_plugins'] = "%s" % " ".join(
-            moa.sysConf.getPlugins())
+            sysConf.getPlugins())
 
         #if moa is silent, make should be silent
-        if not self.job.options.verbose:
+        if not sysConf.options.verbose:
             self.job.makeArgs.append('-s')
 
         self.job.makeArgs.append('-f')

lib/python/moa/backend/ruff.py

@@ -19,8 +19,8 @@
 import moa.template
 import moa.actor
 import moa.backend
-import moa.sysConf
 import moa.logger as l
+from moa.sysConf import sysConf
 
 import Yaco
 
@@ -166,15 +166,19 @@ def generate_data_map():
         rc = 0
         if cmode == 'map':
             #late decoration - see if that works :/
+            #for x in generate_data_map():
+            #    print 'xxxx', x
             executor2 = ruffus.files(generate_data_map)(executor)
-            l.info("Start run (with %d thread(s))" % self.job.options.threads)
+            l.info("Start run (with %d thread(s))" %
+                   sysConf.options.threads)
             ruffus.pipeline_run(
                 [executor2],
-                verbose = self.job.options.verbose,
+                verbose = sysConf.options.verbose,
                 one_second_per_job=False,
-                multiprocess= self.job.options.threads,
+                multiprocess= sysConf.options.threads,
                 )
-            l.info("Finished running (with %d thread(s))" % self.job.options.threads)
+            l.info("Finished running (with %d thread(s))" %
+                   sysConf.options.threads)
             rc = 0
         elif cmode == 'reduce':
             pass

lib/python/moa/job.py

@@ -138,27 +138,27 @@ def hasCommand(self, command):
         else:
             return True
 
-    def checkCommands(self, commands):
+    def checkCommands(self, command):
         """
-        Check commands, and rearrange if there are delegates.
+        Check command, and rearrange if there are delegates.
         
         >>> job = newTestJob('unittest')
         >>> assert(job.template.commands.run.delegate == ['prepare', 'run2'])
-        >>> assert(job.checkCommands(['run2']) == ['run2'])
-        >>> assert(job.checkCommands(['run']) == ['prepare', 'run2'])
-        >>> assert(job.checkCommands(['prepare', 'run']) == ['prepare', 'prepare', 'run2'])
+        >>> assert(job.checkCommands('run2') == ['run2'])
+        >>> assert(job.checkCommands('run') == ['prepare', 'run2'])
+        >>> assert(job.checkCommands('prepare') == ['prepare'])
         
         :param commands: The list of commands to check
         :type commands: list of strings
         :returns: The checked list of commands
         :rtype: list of strings
         """        
         rv = []
-        for command in commands:
-            if self.template.commands.get(command, {}).has_key('delegate'):
-                rv.extend(self.template.commands[command].delegate)
-            else:
-                rv.append(command)
+        if self.template.commands.get(command, {}).has_key('delegate'):
+            rv.extend(self.template.commands[command].delegate)
+        else:
+            rv.append(command)
+
         return rv
 
     def checkConfDir(self):

lib/python/moa/plugin/__init__.py

@@ -5,6 +5,7 @@
 # 
 # Licensed under the GPL license (see 'COPYING')
 # 
+
 """
 Handle Moa commands (i.e. anything that you can run as `moa COMMAND` on the
 commandline
@@ -18,12 +19,19 @@
 ## Load & handle plugins
 class PluginHandler(UserDict.DictMixin):
 
-    def __init__(self, plugins):
-
+    def __init__(self, sysConf):
+        """
+        Must be called with a global 'system configuration' object
+        (Yaco)
+        
+        """
         ## Determine what plugins are loaded
+        self.sysConf = sysConf
+        plugins = sysConf.getPlugins()
         self.plugins = {}
-        self.data = Yaco.Yaco({'plugins' : self})
+        self.sysConf.plugins = self
         self.allPlugins = plugins
+
         l.debug("Plugins %s" % ", ".join(self.allPlugins))
         ## load the plugins as seperate modules. A plugin does not need to
         self.initialize()
@@ -51,30 +59,29 @@ def initialize(self):
         newOrder.sort()
         self.allPlugins = [x[1] for x in newOrder]
 
-
     def register(self, **kwargs):
         """
         Keep track of a dictionary of data that might be used
         by any of the plugins - this seems cleaner than relying
         on using globals
         """
         for k in kwargs:
-            self.data[k] = kwargs[k]
+            self.sysConf[k] = kwargs[k]
 
     def run(self, command):
         rv = {}
         for p in self.allPlugins:
             if not command in dir(self[p]):
                 continue
             l.debug("plugin executing hook %s for %s" % (command, p))
-            rv['p'] = getattr(self[p], command)(self.data)
+            rv['p'] = getattr(self[p], command)(self.sysConf)
         return rv
 
     def runCallback(self, command):
         """
         Run a plugin callback 
         """
-        command['call'](self.data)
+        command['call'](self.sysConf)
 
     def getAttr(self, attribute):
         """

lib/python/moa/plugin/adhoc.py

@@ -386,7 +386,12 @@ def createAdhoc(data):
 [[ "$output" =~ "out.01" ]] || (echo "invalid output 1" && false )
 output=`moa run 2>&1`
 # out.01 should not be in the output (it was already processed last time)
-[[ ! "$output" =~ "out.01" ]] || (echo "invalid output 2" && false )
+[[ ! "$output" =~ "out.01" ]] || ( \
+    echo "invalid output 2";
+    echo $output
+    ls
+    false
+)
 touch test.01
 output=`moa run 2>&1`
 # out.01 should be in the output (it was touched since the last process)

lib/python/moa/plugin/logger.py

@@ -63,7 +63,7 @@ def niceRunTime(d):
 def postInterrupt(data):
     return postCommand(data)
 
-def postCommand(data):
+def finish(data):
     data.logger.end_time = datetime.today()
     data.logger.run_time = data.logger.end_time - data.logger.start_time
     runtime = data.logger.end_time - data.logger.start_time

lib/python/moa/sysConf.py

@@ -55,6 +55,7 @@ def getPlugins(self):
             rv.add(p)
         return list(rv)
 
+#This function is not necessary??? or is it??
 def getPlugins():
     return sysConf.getPlugins()
 

template2/adhoc.mk

@@ -129,7 +129,8 @@ adhoc_unittest:
 	moa set input=10.input/test.*.input
 	moa set process='cat $$< >> test.2'
 	moa run
-	[[ "`cat test.2`" == "2341" ]]
+	cat test.2
+	[[ "`cat test.2`" == "1234" ]]
 	rm test.2
 	moa set name_sed='s/input/output/'
 	moa set process='cat $$< > $$t'

template2/blast.jinja2

@@ -25,11 +25,15 @@ mkdir gff || true 2> /dev/null
 	cd 10.blastdb
 	moa new blastdb -t 'test blast database'
 	moa set fasta_file=$MOABASE/share/test/test.fasta
+    echo "running blastdb"
 	moa run
 	cd ../
 	mkdir 20.blast
     cd 20.blast
 	moa new blast -t 'test blast'
     moa set db=../10.blastdb/blastdb
 	moa set input=$MOABASE/share/test/*.fasta -j 12
-	moa run
+    echo "running blast"
+	moa run
+    echo "finished running blast"        
+    ls -l

template2/map.jinja2

@@ -4,6 +4,13 @@
 
 ### unittest
 
-moa set process=ls
+for x in `seq -w 1 20`; do 
+   touch test.${x}.in
+done
+
+moa set process='cp {{ input }} {{ output }}'
+moa set input='test.*.in'
+moa set output='test.*.out'
 moa run
 
+