Merge pull request #2728 from kyleam/doc-api-commands

DOC: Add command summaries to datalad.api.__doc__

datalad/__init__.py

@@ -7,7 +7,13 @@
 #
 # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
 """DataLad aims to expose (scientific) data available online as a unified data
-distribution with the convenience of git-annex repositories as a backend."""
+distribution with the convenience of git-annex repositories as a backend.
+
+Commands are exposed through both a command-line interface and a Python API. On
+the command line, run 'datalad --help' for a summary of the available commands.
+From an interactive Python session, import `datalad.api` and inspect its
+documentation with `help`.
+"""
 
 # For reproducible demos/tests
 import os

datalad/api.py

@@ -11,6 +11,33 @@
 from datalad.coreapi import *
 
 
+def _command_summary():
+    # Import here to avoid polluting the datalad.api namespace.
+    from collections import defaultdict
+    from datalad.interface.base import alter_interface_docs_for_api
+    from datalad.interface.base import get_api_name
+    from datalad.interface.base import get_cmd_doc
+    from datalad.interface.base import get_cmd_summaries
+    from datalad.interface.base import get_interface_groups
+    from datalad.interface.base import load_interface
+
+    groups = get_interface_groups(include_plugins=True)
+    grp_short_descriptions = defaultdict(list)
+    for group, _, specs in sorted(groups, key=lambda x: x[1]):
+        for spec in specs:
+            intf = load_interface(spec)
+            if intf is None:
+                continue
+            sdescr = getattr(intf, "short_description", None) or \
+                alter_interface_docs_for_api(get_cmd_doc(intf)).split("\n")[0]
+            grp_short_descriptions[group].append(
+                (get_api_name(spec), sdescr))
+    return "\n".join(get_cmd_summaries(grp_short_descriptions, groups))
+
+
+__doc__ += "\n\n{}".format(_command_summary())
+
+
 def _load_plugins():
     from datalad.plugin import _get_plugins
     from datalad.plugin import _load_plugin
@@ -68,3 +95,4 @@ def _generate_extension_api():
 # Be nice and clean up the namespace properly
 del _load_plugins
 del _generate_extension_api
+del _command_summary

datalad/cmdline/main.py

@@ -17,18 +17,16 @@
 lgr.log(5, "Importing cmdline.main")
 
 import argparse
+from collections import defaultdict
 import sys
 import textwrap
-from importlib import import_module
 import os
 
 from six import text_type
 
 import datalad
 
 from datalad.cmdline import helpers
-from datalad.plugin import _get_plugins
-from datalad.plugin import _load_plugin
 from datalad.support.exceptions import InsufficientArgumentsError
 from datalad.support.exceptions import IncompleteResultsError
 from datalad.support.exceptions import CommandError
@@ -90,7 +88,8 @@ def setup_parser(
     lgr.log(5, "Starting to setup_parser")
     # delay since it can be a heavy import
     from ..interface.base import dedent_docstring, get_interface_groups, \
-        get_cmdline_command_name, alter_interface_docs_for_cmdline
+        get_cmdline_command_name, alter_interface_docs_for_cmdline, \
+        load_interface, get_cmd_doc
     # setup cmdline args parser
     parts = {}
     # main parser
@@ -226,12 +225,7 @@ def setup_parser(
         need_single_subparser = False
         unparsed_args = cmdlineargs[1:]  # referenced before assignment otherwise
 
-    interface_groups = get_interface_groups()
-    # TODO: see if we could retain "generator" for plugins
-    # ATM we need to make it explicit so we could check the command(s) below
-    # It could at least follow the same destiny as extensions so we would
-    # just do more iterative "load ups"
-    interface_groups.append(('plugins', 'Plugins', list(_get_plugins())))
+    interface_groups = get_interface_groups(include_plugins=True)
 
     # First unparsed could be either unknown option to top level "datalad"
     # or a command. Among unknown could be --help/--help-np which would
@@ -280,45 +274,27 @@ def setup_parser(
     # --help output before we setup --help for each command
     helpers.parser_add_common_opt(parser, 'help')
 
-    grp_short_descriptions = []
+    grp_short_descriptions = defaultdict(list)
     # create subparser, use module suffix as cmd name
     subparsers = parser.add_subparsers()
-    for _, _, _interfaces \
+    for group_name, _, _interfaces \
             in sorted(interface_groups, key=lambda x: x[1]):
-        # for all subcommand modules it can find
-        cmd_short_descriptions = []
-
         for _intfspec in _interfaces:
             cmd_name = get_cmdline_command_name(_intfspec)
             if need_single_subparser and cmd_name != need_single_subparser:
                 continue
-            if isinstance(_intfspec[1], dict):
-                # plugin
-                _intf = _load_plugin(_intfspec[1]['file'], fail=False)
-                if _intf is None:
-                    # TODO:  add doc why we could skip this one... makes this
-                    # loop harder to extract into a dedicated function
-                    continue
-            else:
-                # turn the interface spec into an instance
-                lgr.log(5, "Importing module %s " % _intfspec[0])
-                try:
-                    _mod = import_module(_intfspec[0], package='datalad')
-                except Exception as e:
-                    lgr.error("Internal error, cannot import interface '%s': %s",
-                              _intfspec[0], exc_str(e))
-                    continue
-                _intf = getattr(_mod, _intfspec[1])
+            _intf = load_interface(_intfspec)
+            if _intf is None:
+                # TODO(yoh):  add doc why we could skip this one... makes this
+                # loop harder to extract into a dedicated function
+                continue
             # deal with optional parser args
             if hasattr(_intf, 'parser_args'):
                 parser_args = _intf.parser_args
             else:
                 parser_args = dict(formatter_class=formatter_class)
                 # use class description, if no explicit description is available
-                intf_doc = '' if _intf.__doc__ is None else _intf.__doc__.strip()
-                if hasattr(_intf, '_docs_'):
-                    # expand docs
-                    intf_doc = intf_doc.format(**_intf._docs_)
+                intf_doc = get_cmd_doc(_intf)
                 parser_args['description'] = alter_interface_docs_for_cmdline(
                     intf_doc)
             subparser = subparsers.add_parser(cmd_name, add_help=False, **parser_args)
@@ -340,8 +316,7 @@ def setup_parser(
             # store short description for later
             sdescr = getattr(_intf, 'short_description',
                              parser_args['description'].split('\n')[0])
-            cmd_short_descriptions.append((cmd_name, sdescr))
-        grp_short_descriptions.append(cmd_short_descriptions)
+            grp_short_descriptions[group_name].append((cmd_name, sdescr))
 
     # create command summary
     if '--help' in cmdlineargs or '--help-np' in cmdlineargs:
@@ -399,23 +374,12 @@ def fail_with_short_help(parser=None,
 def get_description_with_cmd_summary(grp_short_descriptions, interface_groups,
                                      parser_description):
     from ..interface.base import dedent_docstring
+    from ..interface.base import get_cmd_summaries
     lgr.debug("Generating detailed description for the parser")
-    cmd_summary = []
+
     console_width = get_console_width()
-    for i, grp in enumerate(
-            sorted(interface_groups, key=lambda x: x[1])):
-        grp_descr = grp[1]
-        grp_cmds = grp_short_descriptions[i]
-
-        cmd_summary.append('\n*%s*\n' % (grp_descr,))
-        for cd in grp_cmds:
-            cmd_summary.append('  %s\n%s'
-                               % ((cd[0],
-                                   textwrap.fill(
-                                       cd[1].rstrip(' .'),
-                                       console_width - 5,
-                                       initial_indent=' ' * 6,
-                                       subsequent_indent=' ' * 6))))
+    cmd_summary = get_cmd_summaries(grp_short_descriptions, interface_groups,
+                                    width=console_width)
     # we need one last formal section to not have the trailed be
     # confused with the last command group
     cmd_summary.append('\n*General information*\n')

datalad/interface/base.py

@@ -18,6 +18,7 @@
 import sys
 import re
 import textwrap
+from importlib import import_module
 import inspect
 from collections import OrderedDict
 
@@ -29,6 +30,8 @@
 from datalad.support.constraints import EnsureKeyChoice
 from datalad.distribution.dataset import Dataset
 from datalad.distribution.dataset import resolve_path
+from datalad.plugin import _get_plugins
+from datalad.plugin import _load_plugin
 
 
 default_logchannels = {
@@ -58,7 +61,19 @@ def get_cmdline_command_name(intfspec):
     return name
 
 
-def get_interface_groups():
+def get_interface_groups(include_plugins=False):
+    """Return a list of command groups.
+
+    Parameters
+    ----------
+    include_plugins : bool, optional
+        Whether to include a group named 'plugins' that has a list of
+        discovered plugin commands.
+
+    Returns
+    -------
+    A list of tuples with the form (GROUP_NAME, GROUP_DESCRIPTION, COMMANDS).
+    """
     from .. import interface as _interfaces
 
     grps = []
@@ -70,9 +85,94 @@ def get_interface_groups():
         grp_name = _item[7:]
         grp = getattr(_interfaces, _item)
         grps.append((grp_name,) + grp)
+    # TODO(yoh): see if we could retain "generator" for plugins
+    # ATM we need to make it explicit so we could check the command(s) below
+    # It could at least follow the same destiny as extensions so we would
+    # just do more iterative "load ups"
+
+    if include_plugins:
+        grps.append(('plugins', 'Plugins', list(_get_plugins())))
     return grps
 
 
+def get_cmd_summaries(descriptions, groups, width=79):
+    """Return summaries for the commands in `groups`.
+
+    Parameters
+    ----------
+    descriptions : dict
+        A map of group names to summaries.
+    groups : list of tuples
+        A list of groups and commands in the form described by
+        `get_interface_groups`.
+    width : int, optional
+        The maximum width of each line in the summary text.
+
+    Returns
+    -------
+    A list with a formatted entry for each command. The first command of each
+    group is preceded by an entry describing the group.
+    """
+    cmd_summary = []
+    for grp in sorted(groups, key=lambda x: x[1]):
+        grp_descr = grp[1]
+        grp_cmds = descriptions[grp[0]]
+
+        cmd_summary.append('\n*%s*\n' % (grp_descr,))
+        for cd in grp_cmds:
+            cmd_summary.append('  %s\n%s'
+                               % ((cd[0],
+                                   textwrap.fill(
+                                       cd[1].rstrip(' .'),
+                                       width - 5,
+                                       initial_indent=' ' * 6,
+                                       subsequent_indent=' ' * 6))))
+    return cmd_summary
+
+
+def load_interface(spec):
+    """Load and return the class for `spec`.
+
+    Parameters
+    ----------
+    spec : tuple
+        For a standard interface, the first item is the datalad source module
+        and the second object name for the interface. For a plugin, the second
+        item should be a dictionary that maps 'file' to the path the of module.
+
+    Returns
+    -------
+    The interface class or, if importing the module fails, None.
+    """
+    if isinstance(spec[1], dict):
+        intf = _load_plugin(spec[1]['file'], fail=False)
+    else:
+        lgr.log(5, "Importing module %s " % spec[0])
+        try:
+            mod = import_module(spec[0], package='datalad')
+        except Exception as e:
+            lgr.error("Internal error, cannot import interface '%s': %s",
+                      spec[0], exc_str(e))
+            intf = None
+        else:
+            intf = getattr(mod, spec[1])
+    return intf
+
+
+def get_cmd_doc(interface):
+    """Return the documentation for the command defined by `interface`.
+
+    Parameters
+    ----------
+    interface : subclass of Interface
+    """
+    intf_doc = '' if interface.__doc__ is None else interface.__doc__.strip()
+    if hasattr(interface, '_docs_'):
+        # expand docs
+        intf_doc = intf_doc.format(**interface._docs_)
+    return intf_doc
+
+
 def dedent_docstring(text):
     """Remove uniform indentation from a multiline docstring"""
     # Problem is that first line might often have no offset, so might