[Prototype] Generalize magic machinery to allow completion.

In particular useful for multi language integration. One of defect is
that you can't really complete. But what you can do is actually
dispatch the completion to the Magic Class (if you really interested in
integration anyway you go with a magic Class. And dispatch to the class.

So far it completes only for cell magics. I need to poke at
InputSpliter to know if we can figure out we're completing a line magic
which is not at the beginning of the line.

There are a few question remaining:
  - How to tell IPython to give-up (or not) on completing using jedi and
  Python completion. Indeed for %%timeit you just want to __extend__
  completion with your own. For %%sql, you don't want to include Python
  completions.
  - What API should we provide ?
    - here is line/cell/cursor figure things out with a single function
    - Allow user to register a separate completer for the first line,
    and the core of the cell independently.

I'd like to have a prototype that can forward the completion to another
kernel (hook into the python2 magic and have a Python 3 kernel that
control a Python2 kernel ?

IPython/core/completer.py

@@ -1802,6 +1802,15 @@ def _completions(self, full_text: str, offset: int, *, _timeout)->Iterator[Compl
         have lots of processing to do.
 
         """
+        if full_text.startswith('%%') and '\n' in full_text:
+            line, *rest = full_text.split('\n')
+            magic = line.split(' ')[0][2:]
+            magic_completer = self.shell.magics_manager.magics['completer'].get(magic, None)
+            if magic_completer:
+                for c in magic_completer(line, '\n'.join(rest), offset):
+                    assert c.start <= offset
+                    yield c
+            return
         deadline = time.monotonic() + _timeout
 
 

IPython/core/magic.py

@@ -17,16 +17,22 @@
 from getopt import getopt, GetoptError
 
 from traitlets.config.configurable import Configurable
+
 from IPython.core import oinspect
 from IPython.core.error import UsageError
 from IPython.core.inputsplitter import ESC_MAGIC, ESC_MAGIC2
+
 from decorator import decorator
+
 from IPython.utils.ipstruct import Struct
 from IPython.utils.process import arg_split
 from IPython.utils.text import dedent
+
 from traitlets import Bool, Dict, Instance, observe
+
 from logging import error
 
+
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
@@ -37,7 +43,7 @@
 # access to the class when they run.  See for more details:
 # http://stackoverflow.com/questions/2366713/can-a-python-decorator-of-an-instance-method-access-the-class
 
-magics = dict(line={}, cell={})
+magics = dict(line={}, cell={}, completer={})
 
 magic_kinds = ('line', 'cell')
 magic_spec = ('line', 'cell', 'line_cell')
@@ -103,9 +109,11 @@ def magics_class(cls):
     """
     cls.registered = True
     cls.magics = dict(line = magics['line'],
-                      cell = magics['cell'])
+                      cell = magics['cell'], 
+                      completer = magics['completer'])
     magics['line'] = {}
     magics['cell'] = {}
+    magics['completer'] = {}
     return cls
 
 
@@ -282,6 +290,24 @@ def mark(func, *a, **kw):
 # Core Magic classes
 #-----------------------------------------------------------------------------
 
+def completer_for(magic_name):
+    """
+    Decorator to mark a function as being the completer for a given magic.
+
+    Example::
+        
+        @completer_for(magic_name)
+        def function(line, cell, offset):
+            yield 'Example'
+
+    """
+    def decorator(method):
+        s = str(method.__name__)
+        magics['completer'][magic_name] = s
+        return method
+    return decorator
+
+
 class MagicsManager(Configurable):
     """Object that handles all magic-related functionality for IPython.
     """
@@ -314,7 +340,7 @@ def __init__(self, shell=None, config=None, user_magics=None, **traits):
 
         super(MagicsManager, self).__init__(shell=shell, config=config,
                                            user_magics=user_magics, **traits)
-        self.magics = dict(line={}, cell={})
+        self.magics = dict(line={}, cell={}, completer={})
         # Let's add the user_magics to the registry for uniformity, so *all*
         # registered magic containers can be found there.
         self.registry[user_magics.__class__.__name__] = user_magics
@@ -388,7 +414,7 @@ def register(self, *magic_objects):
             # Now that we have an instance, we can register it and update the
             # table of callables
             self.registry[m.__class__.__name__] = m
-            for mtype in magic_kinds:
+            for mtype in m.magics.keys():
                 self.magics[mtype].update(m.magics[mtype])
 
     def register_function(self, func, magic_kind='line', magic_name=None):
@@ -507,16 +533,16 @@ def __init__(self, shell=None, **kwargs):
         # But we mustn't clobber the *class* mapping, in case of multiple instances.
         class_magics = self.magics
         self.magics = {}
-        for mtype in magic_kinds:
-            tab = self.magics[mtype] = {}
+        for mtype in list(magic_kinds)+['completer']:
+            self.magics[mtype] = {}
             cls_tab = class_magics[mtype]
             for magic_name, meth_name in cls_tab.items():
                 if isinstance(meth_name, str):
                     # it's a method name, grab it
-                    tab[magic_name] = getattr(self, meth_name)
+                    self.magics[mtype][magic_name] = getattr(self, meth_name)
                 else:
                     # it's the real thing
-                    tab[magic_name] = meth_name
+                    self.magics[mtype][magic_name] = meth_name
         # Configurable **needs** to be initiated at the end or the config
         # magics get screwed up.
         super(Magics, self).__init__(**kwargs)

IPython/core/magics/display.py

@@ -14,14 +14,15 @@
 # Our own packages
 from IPython.core.display import display, Javascript, Latex, SVG, HTML, Markdown
 from IPython.core.magic import  (
-    Magics, magics_class, cell_magic
+    Magics, magics_class, cell_magic, completer_for
 )
 
 #-----------------------------------------------------------------------------
 # Magic implementation classes
 #-----------------------------------------------------------------------------
 
 
+
 @magics_class
 class DisplayMagics(Magics):
     """Magics for displaying various output types with literals

IPython/core/tests/test_magic.py

@@ -1028,7 +1028,7 @@ def test_ls_magic():
     lsmagic = ip.magic('lsmagic')
     with warnings.catch_warnings(record=True) as w:
         j = json_formatter(lsmagic)
-    nt.assert_equal(sorted(j), ['cell', 'line'])
+    nt.assert_equal(sorted(j), ['cell', 'completer', 'line'])
     nt.assert_equal(w, []) # no warnings
 
 def test_strip_initial_indent():

IPython/terminal/ptutils.py

@@ -8,6 +8,7 @@
 # Distributed under the terms of the Modified BSD License.
 
 import unicodedata
+
 from wcwidth import wcwidth
 
 from IPython.core.completer import (

docs/source/config/custommagics.rst

@@ -177,3 +177,75 @@ setuptools, distutils, or any other distribution tools like `flit
         def cadabra(self, line, cell):
             return line, cell
 
+Defining completer for cell magics
+----------------------------------
+
+A number of magics allow to embed non-python code in Python documents. It can
+thus be useful to define custom completer which provide completion for the user.
+Since IPython 6.2 this is possible when Defining custom magic classes, and using
+the ``@complete_for`` decorator.
+
+Let's extend our above magic with completions:
+
+.. sourcecode::
+
+    from IPython.core.magic import (Magics, magics_class, line_magic, cell_magic, completer_for)
+    from IPython.core.completer import Completion
+
+    words = ['Supercalifragilisticexpialidocious', 'Alakazam', 'Shazam']
+
+    @magics_class
+    class Abracadabra(Magics):
+
+        @line_magic
+        def abra(self, line):
+            return line
+
+        @cell_magic
+        def cadabra(self, line, cell):
+            return line, cell
+
+        @completer_for('cadabra')
+        def complete(self, line:str, cell:str, offset:int):
+            """
+            `line` will be the first line of the cell, `cell`
+            the rest of the body starting at the second line,
+            `offset` the position of the cursor in the cell, starting
+            at the beginning of `line` (incuding the % or %%) character
+            in unicode codepoints.
+
+            The end of line ned tobe explicitly take care of.
+
+            This function should `yield` a set of `IPython.core.completer.Completions(start, end, text)`
+            telling IPython to replace the text between `start` and `end` by `text`.
+            """
+
+            # get the full body and text until the cursor
+            # there can be some text after the cusrsor we should
+            take care of that but keep the example simple.
+            full_body = line + '\n'+cell
+            text_until_cursor = full_body[:offset]
+
+            # split on whitespace and get last token:
+            token_before_cursor = text_until_cursor.split()[-1].lower()
+
+            for w in words:
+                if w.lower().startswith(token_before_cursor):
+                    # We'll replace the current token so replace
+                    # from the position of the cursor back len of token.
+                    start = offset-len(token_before_cursor)
+                    end = offset
+                    yield Completion(start, end, w)
+
+We  can optionally register them live in an IPython shell of notebook::
+
+    ip = get_ipython()
+    ip.register_magics(Abracadabra)
+
+Now try the following::
+
+    %%cadabra
+    s<tab>
+
+You should get ``Shazam`` and ``Supercalifragilisticexpialidocious`` as
+potential completions.