[Prototype] Generalize magic machinery to allow completion.

In particular useful for multi language integration. One of defect is
that you can't really complete. But what you can do is actually
dispatch the completion to the Magic Class (if you really interested in
integration anyway you go with a magic Class. And dispatch to the class.

So far it completes only for cell magics. I need to poke at
InputSpliter to know if we can figure out we're completing a line magic
which is not at the beginning of the line.

There are a few question remaining:
  - How to tell IPython to give-up (or not) on completing using jedi and
  Python completion. Indeed for %%timeit you just want to __extend__
  completion with your own. For %%sql, you don't want to include Python
  completions.
  - What API should we provide ?
    - here is line/cell/cursor figure things out with a single function
    - Allow user to register a separate completer for the first line,
    and the core of the cell independently.

I'd like to have a prototype that can forward the completion to another
kernel (hook into the python2 magic and have a Python 3 kernel that
control a Python2 kernel ?

IPython/core/completer.py

@@ -1799,6 +1799,15 @@ def _completions(self, full_text: str, offset: int, *, _timeout)->Iterator[Compl
         have lots of processing to do.
 
         """
+        if full_text.startswith('%%') and '\n' in full_text:
+            line, *rest = full_text.split('\n')
+            magic = line.split(' ')[0][2:]
+            magic_completer = self.shell.magics_manager.magics['completer'].get(magic, None)
+            if magic_completer:
+                for c in magic_completer(line, '\n'.join(rest), offset):
+                    assert c.start <= offset
+                    yield c
+            return
         deadline = time.monotonic() + _timeout
 
 

IPython/core/magic.py

@@ -17,16 +17,22 @@
 from getopt import getopt, GetoptError
 
 from traitlets.config.configurable import Configurable
+
 from IPython.core import oinspect
 from IPython.core.error import UsageError
 from IPython.core.inputsplitter import ESC_MAGIC, ESC_MAGIC2
+
 from decorator import decorator
+
 from IPython.utils.ipstruct import Struct
 from IPython.utils.process import arg_split
 from IPython.utils.text import dedent
+
 from traitlets import Bool, Dict, Instance, observe
+
 from logging import error
 
+
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
@@ -37,7 +43,7 @@
 # access to the class when they run.  See for more details:
 # http://stackoverflow.com/questions/2366713/can-a-python-decorator-of-an-instance-method-access-the-class
 
-magics = dict(line={}, cell={})
+magics = dict(line={}, cell={}, completer={})
 
 magic_kinds = ('line', 'cell')
 magic_spec = ('line', 'cell', 'line_cell')
@@ -103,9 +109,11 @@ def magics_class(cls):
     """
     cls.registered = True
     cls.magics = dict(line = magics['line'],
-                      cell = magics['cell'])
+                      cell = magics['cell'], 
+                      completer = magics['completer'])
     magics['line'] = {}
     magics['cell'] = {}
+    magics['completer'] = {}
     return cls
 
 
@@ -282,6 +290,14 @@ def mark(func, *a, **kw):
 # Core Magic classes
 #-----------------------------------------------------------------------------
 
+def completer_for(magic_name):
+    def decorator(method):
+        s = str(method.__name__)
+        magics['completer'][magic_name] = s
+        return method
+    return decorator
+
+
 class MagicsManager(Configurable):
     """Object that handles all magic-related functionality for IPython.
     """
@@ -314,7 +330,7 @@ def __init__(self, shell=None, config=None, user_magics=None, **traits):
 
         super(MagicsManager, self).__init__(shell=shell, config=config,
                                            user_magics=user_magics, **traits)
-        self.magics = dict(line={}, cell={})
+        self.magics = dict(line={}, cell={}, completer={})
         # Let's add the user_magics to the registry for uniformity, so *all*
         # registered magic containers can be found there.
         self.registry[user_magics.__class__.__name__] = user_magics
@@ -388,7 +404,7 @@ def register(self, *magic_objects):
             # Now that we have an instance, we can register it and update the
             # table of callables
             self.registry[m.__class__.__name__] = m
-            for mtype in magic_kinds:
+            for mtype in m.magics.keys():
                 self.magics[mtype].update(m.magics[mtype])
 
     def register_function(self, func, magic_kind='line', magic_name=None):
@@ -507,16 +523,16 @@ def __init__(self, shell=None, **kwargs):
         # But we mustn't clobber the *class* mapping, in case of multiple instances.
         class_magics = self.magics
         self.magics = {}
-        for mtype in magic_kinds:
-            tab = self.magics[mtype] = {}
+        for mtype in (*magic_kinds, 'completer'):
+            self.magics[mtype] = {}
             cls_tab = class_magics[mtype]
             for magic_name, meth_name in cls_tab.items():
                 if isinstance(meth_name, str):
                     # it's a method name, grab it
-                    tab[magic_name] = getattr(self, meth_name)
+                    self.magics[mtype][magic_name] = getattr(self, meth_name)
                 else:
                     # it's the real thing
-                    tab[magic_name] = meth_name
+                    self.magics[mtype][magic_name] = meth_name
         # Configurable **needs** to be initiated at the end or the config
         # magics get screwed up.
         super(Magics, self).__init__(**kwargs)

IPython/core/magics/display.py

@@ -14,14 +14,15 @@
 # Our own packages
 from IPython.core.display import display, Javascript, Latex, SVG, HTML, Markdown
 from IPython.core.magic import  (
-    Magics, magics_class, cell_magic
+    Magics, magics_class, cell_magic, completer_for
 )
 
 #-----------------------------------------------------------------------------
 # Magic implementation classes
 #-----------------------------------------------------------------------------
 
 
+
 @magics_class
 class DisplayMagics(Magics):
     """Magics for displaying various output types with literals

IPython/terminal/ptutils.py

@@ -8,6 +8,7 @@
 # Distributed under the terms of the Modified BSD License.
 
 import unicodedata
+
 from wcwidth import wcwidth
 
 from IPython.core.completer import (