NF: Result summary renderer (more or less useful demo in get)

datalad/cmdline/main.py

@@ -21,7 +21,6 @@
 import shutil
 from importlib import import_module
 import os
-import inspect
 
 import datalad
 
@@ -314,15 +313,11 @@ def main(args=None):
             # so we could see/stop clearly at the point of failure
             setup_exceptionhook(ipython=cmdlineargs.common_idebug)
             ret = cmdlineargs.func(cmdlineargs)
-            if inspect.isgenerator(ret):
-                ret = list(ret)
         else:
             # otherwise - guard and only log the summary. Postmortem is not
             # as convenient if being caught in this ultimate except
             try:
                 ret = cmdlineargs.func(cmdlineargs)
-                if inspect.isgenerator(ret):
-                    ret = list(ret)
             except InsufficientArgumentsError as exc:
                 # if the func reports inappropriate usage, give help output
                 lgr.error('%s (%s)' % (exc_str(exc), exc.__class__.__name__))

datalad/distribution/get.py

@@ -25,6 +25,7 @@
 from datalad.interface.results import results_from_paths
 from datalad.interface.results import YieldDatasets
 from datalad.interface.results import annexjson2result
+from datalad.interface.results import count_results
 from datalad.interface.common_opts import recursion_flag
 # from datalad.interface.common_opts import git_opts
 # from datalad.interface.common_opts import annex_opts
@@ -427,39 +428,31 @@ def __call__(
                                        refds=refds_path)
                 yield res
 
-
-    # TODO generator
-    # RF for new interface
     @staticmethod
-    def result_renderer_cmdline(res, args):
+    def custom_result_summary_renderer(res):
         from datalad.ui import ui
         from os import linesep
-        if res is None:
-            res = []
-        if not isinstance(res, list):
-            res = [res]
         if not len(res):
             ui.message("Got nothing new")
             return
 
-        # provide summary
-        nsuccess = sum(item.get('success', False) if isinstance(item, dict) else True
-                       for item in res)
-        nfailure = len(res) - nsuccess
-        msg = "Tried to get %d %s." % (
-            len(res), single_or_plural("file", "files", len(res)))
-        if nsuccess:
-            msg += " Got %d. " % nsuccess
+        nfiles = count_results(res, type='file')
+        nsuccess_file = count_results(res, type='file', status='ok')
+        nfailure = nfiles - nsuccess_file
+        msg = "Tried to get %d %s that had no content yet." % (
+            nfiles, single_or_plural("file", "files", nfiles))
+        if nsuccess_file:
+            msg += " Successfully obtained %d. " % nsuccess_file
         if nfailure:
-            msg += " Failed to get %d." % (nfailure,)
+            msg += " %d (failed)." % (nfailure,)
         ui.message(msg)
 
         # if just a few or less than initially explicitly requested
-        if len(res) < 10 or args.verbose:
+        if len(res) < 10:
             msg = linesep.join([
-                "{path} ... {suc}".format(
-                    suc="ok." if isinstance(item, Dataset) or item.get('success', False)
-                        else "failed. (%s)" % item.get('note', 'unknown reason'),
-                    path=item.get('file') if isinstance(item, dict) else item.path)
+                "{path}{type} ... {suc}".format(
+                    suc=item.get('status'),
+                    path=item.get('path'),
+                    type=' [{}]'.format(item['type']) if 'type' in item else '')
                 for item in res])
             ui.message(msg)

datalad/interface/base.py

@@ -19,6 +19,7 @@
 import re
 import textwrap
 from os.path import curdir
+import inspect
 
 from ..ui import ui
 from ..dochelpers import exc_str
@@ -334,7 +335,13 @@ def call_from_parser(cls, args):
                 result_filter = result_filter & tfilt if result_filter else tfilt
             kwargs['result_filter'] = result_filter
         try:
-            return cls.__call__(**kwargs)
+            ret = cls.__call__(**kwargs)
+            if inspect.isgenerator(ret):
+                ret = list(ret)
+            if args.common_output_format == 'tailored' and \
+                    hasattr(cls, 'custom_result_summary_renderer'):
+                cls.custom_result_summary_renderer(ret)
+            return ret
         except KeyboardInterrupt as exc:
             ui.error("\nInterrupted by user while doing magic: %s" % exc_str(exc))
             sys.exit(1)

datalad/interface/results.py

@@ -103,3 +103,9 @@ def annexjson2result(d, ds, **kwargs):
     res['action'] = d['command']
     res['annexkey'] = d['key']
     return res
+
+
+def count_results(res, **kwargs):
+    """Return number if results that match all property values in kwargs"""
+    return sum(
+        all(k in r and r[k] == v for k, v in kwargs.items()) for r in res)

datalad/interface/utils.py

@@ -850,7 +850,7 @@ def filter_unmodified(content_by_ds, refds, since):
         constraints=EnsureChoice(*list(known_result_xfms.keys())) | EnsureCallable()),
     result_renderer=Parameter(
         doc="""format of return value rendering on stdout""",
-        constraints=EnsureChoice('json', 'simple') | EnsureNone()),
+        constraints=EnsureChoice('json', 'simple', 'tailored') | EnsureNone()),
     on_failure=Parameter(
         doc="""behavior to perform on failure: 'ignore' any failure is reported,
         but does not cause an exception; 'continue' if any failure occurs an
@@ -953,6 +953,7 @@ def eval_func(wrapped, instance, args, kwargs):
                 p_name,
                 getattr(_func_class, p_name, eval_defaults[p_name]))
             for p_name in eval_params}
+        result_renderer = common_params['result_renderer']
 
         def generator_func(*_args, **_kwargs):
             # obtain results
@@ -961,8 +962,8 @@ def generator_func(*_args, **_kwargs):
             # TODO actually compose a meaningful exception
             incomplete_results = []
             # inspect and render
-            result_renderer = common_params['result_renderer']
             result_filter = common_params['result_filter']
+            result_renderer = common_params['result_renderer']
             result_xfm = common_params['result_xfm']
             if result_xfm in known_result_xfms:
                 result_xfm = known_result_xfms[result_xfm]
@@ -1036,6 +1037,11 @@ def return_func(wrapped_, instance_, args_, kwargs_):
                 results = wrapped_(*args_, **kwargs_)
                 if inspect.isgenerator(results):
                     results = list(results)
+                # render summaries
+                if not common_params['result_xfm'] and result_renderer == 'tailored':
+                    # cannot render transformed results
+                    if hasattr(_func_class, 'custom_result_summary_renderer'):
+                        _func_class.custom_result_summary_renderer(results)
                 if common_params['return_type'] == 'item-or-list' and \
                         len(results) == 1:
                     return results[0]