Implement dynamic command resolution and update help messages and usages

indexfile/cli/__init__.py

@@ -4,37 +4,20 @@
 import sys
 import os
 import csv
+import imp
 import yaml
+import glob
 import indexfile
 import simplejson as json
 from os import environ as env
 from schema import SchemaError
 from indexfile.index import Index
 
-COMMANDS = json.load(open(os.path.join(os.path.dirname(os.path.realpath(__file__)),'commands.json')))
-
-#COMMANDS = {
-#    'show': {
-#        'desc': 'Show the index',
-#        'aliases': []
-#    },
-#    'add': {
-#        'desc': 'Add file contents to the index',
-#        'aliases': []
-#    },
-#    'remove': {
-#        'desc': 'Remove files from the index',
-#        'aliases': ['rm']
-#    },
-#    'help': {
-#        'desc': 'Show the help',
-#    }
-#}
-
 DEFAULT_CONFIG_FILE = '.indexfile.yml'
 DEFAULT_ENV_INDEX = 'IDX_FILE'
 DEFAULT_ENV_FORMAT = 'IDX_FORMAT'
 
+IGNORE_COMMANDS = ['__init__', 'indexfile_main']
 
 def walk_up(bottom):
     """
@@ -70,20 +53,51 @@ def walk_up(bottom):
         yield x
 
 
-def get_command_aliases():
+def load_commands():
+    """Load commands from files"""
+    basedir = os.path.dirname(__file__)
+    cmds = glob.glob('{0}/*.py'.format(basedir))
+    d = {}
+
+    for cmd in cmds:
+        mod = os.path.basename(cmd).replace('.py', '')
+        if mod in IGNORE_COMMANDS:
+            continue
+        info = imp.find_module(mod, [basedir])
+        m = imp.load_module(mod, *info)
+        d[m.name] = {'desc': m.desc, 'aliases': m.aliases}
+
+    d['help'] = {'desc': "Show the help"}
+
+    return d
+
+
+def get_command_aliases(cmds):
     """Get all command aliases"""
-    return [alias for command in COMMANDS.values()
+    return [alias for command in cmds.values()
             for alias in command.get('aliases', [])]
 
 
-def get_command(alias):
+def get_command(alias, cmds):
     """Get command from command string or alias"""
-    if alias in COMMANDS:
+    if alias in cmds:
         return alias
-    return [k for k, v in COMMANDS.iteritems()
+    return [k for k, v in cmds.iteritems()
             if alias in v.get('aliases', [])][0]
 
 
+def get_commands_help(cmds):
+    """Get list of commands and descriptions for the help message"""
+    s = []
+    m = 0
+    for k, v in cmds.iteritems():
+        if v.get('aliases'):
+            k = k + " ({0})".format(', '.join(v.get('aliases')))
+        m = max(m, len(k))
+        s.append([k, v.get('desc', "")])
+    return '\n'.join(['  {0}\t{1}'.format(name.ljust(m), desc) for name, desc in s])
+
+
 def default_config():
     """Return the default configuration"""
     config = {}
@@ -143,14 +157,15 @@ def open_index(config):
 # validation objects
 class Command(object):
 
-    def __init__(self, error=None):
+    def __init__(self, error=None, commands=None):
         self._error = error
+        self._commands = commands
 
     def validate(self, data):
         """Return valid command string or SchemaException in case of error"""
         if not data:
             data = 'help'
-        if data in COMMANDS.keys() + get_command_aliases():
+        if data in self._commands.keys() + get_command_aliases(self._commands):
             return data
         else:
             raise SchemaError('Invalid command %r' %

indexfile/cli/indexfile_add.py

@@ -2,7 +2,7 @@
 Add new file to the index using the provided file metadata
 (eg. path, type, size, md5...)
 
-Usage: %s_add [options] [<metadata>...]
+Usage: %s [options] [<metadata>...]
 
 Options:
   -a --attributes <attributes>  List of attribute name referring to the file
@@ -18,13 +18,17 @@
 from schema import Schema, Use, Optional, Or, And
 from docopt import docopt
 
+# set command info
+name = __name__.replace('indexfile_','')
+desc = "Add file contents to the index"
+aliases = []
 
 def run(index):
     """Add files to the index"""
     log = indexfile.getLogger(__name__)
 
     # parser args and remove dashes
-    args = docopt(__doc__)
+    args = docopt(__doc__ % command)
     args = dict([(k.replace('-', ''), v) for k, v in args.iteritems()])
 
     # create validation schema

indexfile/cli/indexfile_main.py

@@ -14,11 +14,6 @@
 
 The main commands are:
 
-  help       Show this help message and exit
-  show       Show the index
-  add        Add file contents to the index
-  remove     Remove files from the index
-
 """
 import sys
 import os
@@ -27,7 +22,7 @@
 import indexfile
 from docopt import docopt
 from schema import Schema, And, Or, Use, Optional
-from indexfile.cli import open_index, load_config, Command, get_command
+from indexfile.cli import open_index, load_config, Command, get_command, load_commands, get_commands_help
 
 
 def main():
@@ -42,6 +37,10 @@ def main():
         # local variables
         index = None
 
+        # load commands
+        commands = load_commands()
+        helpstr = __doc__ % (name, name) + get_commands_help(commands)
+
         # create validation schema
         sch = Schema({
             'index': Or(None,
@@ -55,36 +54,34 @@ def main():
                                          'warn',
                                          'info',
                                          'debug')),
-            '<command>': Command(),
+            '<command>': Command(commands=commands),
             str: object
         })
 
         # parse args and remove dashes
-        args = docopt(__doc__ % (name, name), version="%s v%s" % (name, version),
-                      options_first=True)
+        args = docopt(helpstr, version="%s v%s" % (name, version), options_first=True)
         args = dict([(k.replace('-', ''), v) for k, v in args.iteritems()])
 
         # validate args
         args = sch.validate(args)
 
         # deal with 'help' command
         if args.get('<command>') == 'help':
-            docopt(__doc__ % (name, name),
-                   version="%s v%s" % (name, version), argv=['--help'])
+            docopt(helpstr, version="%s v%s" % (name, version), argv=['--help'])
 
         # load the index and delegate command
         config = load_config(os.getcwd(), args)
 
         indexfile.setLogLevel(config.get('loglevel'))
         index = open_index(config)
 
-        command_ = get_command(args.get('<command>'))
-        argv = [command_] + args['<args>']
+        command_ = get_command(args.get('<command>'), commands)
+        argv = [name, command_] + args['<args>']
         sys.argv = argv
         module_ = "indexfile.cli.indexfile_%s" % command_
         runpy.run_module(module_,
                          run_name="__main__",
-                         init_globals={'index': index})
+                         init_globals={'index': index, 'command': '{0} {1}'.format(name, command_)})
 
     except KeyboardInterrupt, e:
         sys.exit(1)

indexfile/cli/indexfile_remove.py

@@ -1,20 +1,28 @@
 """
-Usage: %s_remove [options] <path>... [--clear]
+Remove files and/or datasets from the index. Query can be a file path or a string
+like 'id=ID001' or 'type=bam'.
+
+Usage: %s [options] <query>...
 
 Options:
 
-  --clear                   Clear all metadata when <file_path> is the last file of the dataset [default: false]
+  -c, --clear  Remove a dataset entry in the index if it does not contain any
+               more files [default: false]
 
 """
 from docopt import docopt
 from schema import Schema, Use, Optional
 
+# set command info
+name = __name__.replace('indexfile_','')
+desc = "Remove files from the index"
+aliases = ['rm']
 
 def run(index):
     """Remove files and/or datasets from the index"""
 
     # parser args and remove dashes
-    args = docopt(__doc__)
+    args = docopt(__doc__ % command)
     args = dict([(k.replace('-', ''), v) for k, v in args.iteritems()])
 
     # create validation schema
@@ -25,7 +33,7 @@ def run(index):
     args = sch.validate(args)
 
     index.lock()
-    paths = args.get('<path>')
+    paths = args.get('<query>')
     for path in paths:
         if '=' in path:
             kwargs = dict([path.split('=')])

indexfile/cli/indexfile_show.py

@@ -1,9 +1,9 @@
 """
-Usage: indexfile_show [options] [<query>]...
-
 Select datasets using query strings. Examples of valid strings are: 'sex=M' and 'lab=CRG'.
 Multiple fields in a query are joind with an 'AND'.
 
+Usage: %s [options] [<query>]...
+
 Options:
 
   -a, --absolute-path    Specify if absolute path should be returned
@@ -25,12 +25,18 @@
 from docopt import docopt
 from indexfile.index import Index
 
+# set command info
+name = __name__.replace('indexfile_','')
+desc = "Show the index"
+aliases = []
+
 def run(index):
     """Show index contents and filter based on query terms"""
     export_type = 'index'
 
     # parser args and remove dashes
-    args = docopt(__doc__)
+
+    args = docopt(__doc__ % command)
     args = dict([(k.replace('-', ''), v) for k, v in args.iteritems()])
 
     # create validation schema