Merge pull request #360 from yarikoptic/enh-schedule-api

NF: --pbs-runner  to schedule any given command to be ran via PBS (condor ATM)

datalad/cmdline/common_args.py

@@ -40,3 +40,10 @@
          help="""level of verbosity. Integers provide even more debugging information""")
 )
 
+pbs_runner = (
+    'pbs-runner', ('-p', '--pbs-runner'),
+    dict(choices=['condor'],
+         default=None,
+         help="""execute command by scheduling it via available PBS.  For settings config fill be consulted""")
+)
+

datalad/cmdline/helpers.py

@@ -16,8 +16,14 @@
 import re
 import sys
 
+from tempfile import NamedTemporaryFile
+
+from ..cmd import Runner
 from ..log import is_interactive
+from ..utils import getpwd
 
+from logging import getLogger
+lgr = getLogger('datalad.cmdline')
 
 class HelpAction(argparse.Action):
     def __call__(self, parser, namespace, values, option_string=None):
@@ -87,6 +93,13 @@ def __call__(self, parser, namespace, values, option_string=None):
         from ..log import LoggerHelper
         LoggerHelper().set_level(level=values)
 
+class PBSAction(argparse.Action):
+    """Action to schedule actual command execution via PBS (e.g. Condor)"""
+    def __call__(self, parser, namespace, values, option_string=None):
+        pbs = values[0]
+        import pdb; pdb.set_trace()
+        i = 1
+
 
 def parser_add_common_args(parser, pos=None, opt=None, **kwargs):
     from . import common_args
@@ -114,6 +127,55 @@ def parser_add_common_opt(parser, opt, names=None, **kwargs):
         parser.add_argument(*names, **opt_kwargs)
 
 
+def strip_arg_from_argv(args, value, opt_names):
+    """Strip an originally listed option (with its value) from the list cmdline args
+    """
+    # Yarik doesn't know better
+    args = args or sys.argv
+    # remove present pbs-runner option
+    args_clean = []
+    skip = 0
+    for i, arg in enumerate(args):
+        if skip:
+            # we skip only one as instructed
+            skip -= 1
+            continue
+        if not (arg in opt_names and i < len(args)-1 and args[i + 1] == value):
+            args_clean.append(arg)
+        else:
+            # we need to skip this one and next one
+            skip = 1
+    return args_clean
+
+
+def run_via_pbs(args, pbs):
+    assert(pbs in ('condor',))  # for now
+
+    # TODO: RF to support multiple backends, parameters, etc, for now -- just condor, no options
+    f = NamedTemporaryFile('w', prefix='datalad-%s-' % pbs, suffix='.submit', delete=False)
+    try:
+        pwd = getpwd()
+        logs = f.name.replace('.submit', '.log')
+        exe = args[0]
+        # TODO: we might need better way to join them, escaping spaces etc.  There must be a stock helper
+        #exe_args = ' '.join(map(repr, args[1:])) if len(args) > 1 else ''
+        exe_args = ' '.join(args[1:]) if len(args) > 1 else ''
+        f.write("""\
+Executable = %(exe)s
+Initialdir = %(pwd)s
+Output = %(logs)s
+Error = %(logs)s
+getenv = True
+
+arguments = %(exe_args)s
+queue
+""" % locals())
+        f.close()
+        Runner().run(['condor_submit', f.name])
+        lgr.info("Scheduled execution via %s.  Logs will be stored under %s" % (pbs, logs))
+    finally:
+        os.unlink(f.name)
+
 class RegexpType(object):
     """Factory for creating regular expression types for argparse
 

datalad/cmdline/main.py

@@ -66,6 +66,7 @@ def setup_parser():
     # common options
     helpers.parser_add_common_opt(parser, 'help')
     helpers.parser_add_common_opt(parser, 'log_level')
+    helpers.parser_add_common_opt(parser, 'pbs_runner')
     helpers.parser_add_common_opt(
         parser,
         'version',
@@ -129,6 +130,7 @@ def setup_parser():
             # our own custom help for all commands
             helpers.parser_add_common_opt(subparser, 'help')
             helpers.parser_add_common_opt(subparser, 'log_level')
+            helpers.parser_add_common_opt(subparser, 'pbs_runner')
             # let module configure the parser
             _intf.setup_parser(subparser)
             # logger for command
@@ -181,21 +183,29 @@ def setup_parser():
 #     return cmdlineargs, functor, args, kwargs
 
 
-def main(cmdlineargs=None):
+def main(args=None):
     # PYTHON_ARGCOMPLETE_OK
     parser = setup_parser()
     try:
         import argcomplete
         argcomplete.autocomplete(parser)
     except ImportError:
         pass
+
     # parse cmd args
-    cmdlineargs = parser.parse_args(cmdlineargs)
+    cmdlineargs = parser.parse_args(args)
     if not cmdlineargs.change_path is None:
         for path in cmdlineargs.change_path:
             chpwd(path)
-    # run the function associated with the selected command
-    if cmdlineargs.common_debug:
+
+    if cmdlineargs.pbs_runner:
+        from .helpers import run_via_pbs
+        from .helpers import strip_arg_from_argv
+        from .common_args import pbs_runner as pbs_runner_opt
+        args_ = strip_arg_from_argv(args or sys.argv, cmdlineargs.pbs_runner, pbs_runner_opt[1])
+        # run the function associated with the selected command
+        run_via_pbs(args_, cmdlineargs.pbs_runner)
+    elif cmdlineargs.common_debug:
         # So we could see/stop clearly at the point of failure
         setup_exceptionhook()
         cmdlineargs.func(cmdlineargs)
@@ -207,3 +217,5 @@ def main(cmdlineargs=None):
         except Exception as exc:
             lgr.error('%s (%s)' % (exc_str(exc), exc.__class__.__name__))
             sys.exit(1)
+
+

datalad/cmdline/tests/test_helpers.py

@@ -16,6 +16,7 @@
 from os import mkdir
 from os.path import join as opj, exists, realpath
 from ..helpers import get_datalad_master, get_repo_instance
+from ..helpers import strip_arg_from_argv
 
 from ...tests.utils import ok_, eq_, assert_cwd_unchanged, ok_clean_git, \
     with_tempfile, SkipTest, with_testrepos
@@ -181,3 +182,11 @@ def test_get_repo_instance_collection(path):
     eq_(realpath(repo.path), realpath(path))
 
     chpwd(old_pwd)
+
+
+def test_strip_arg_from_argv():
+    eq_(strip_arg_from_argv(['-s', 'value'], 'value', ('-s',)), [])
+    eq_(strip_arg_from_argv(['-s', 'value'], 'value', ('-s', '--long-s')), [])
+    eq_(strip_arg_from_argv(
+            ['cmd', '-s', 'value', '--more'], 'value', ('-s', '--long-s')),
+            ['cmd',                '--more'])