Merge PR #847 (connection files)

* JSON connection files are now used to connect files
* HMAC message signing is now on by default in all IPython apps
* Adds IPython.lib.kernel, which contains utility functions for connecting
  clients. These were mostly copied out of qtconsoleapp in order to be
  more general.
* Adds %connection_info and %qtconsole magics to zmqshell

closes gh-688
closes gh-806
closes gh-691

IPython/frontend/html/notebook/handlers.py

@@ -173,7 +173,13 @@ def _on_zmq_reply(self, msg_list):
 class AuthenticatedZMQStreamHandler(ZMQStreamHandler):
     def open(self, kernel_id):
         self.kernel_id = kernel_id.decode('ascii')
-        self.session = Session()
+        try:
+            cfg = self.application.ipython_app.config
+        except AttributeError:
+            # protect from the case where this is run from something other than
+            # the notebook app:
+            cfg = None
+        self.session = Session(config=cfg)
         self.save_on_message = self.on_message
         self.on_message = self.on_first_message
 

IPython/frontend/html/notebook/kernelmanager.py

@@ -16,6 +16,7 @@
 # Imports
 #-----------------------------------------------------------------------------
 
+import os
 import signal
 import sys
 import uuid
@@ -27,6 +28,7 @@
 
 from IPython.config.configurable import LoggingConfigurable
 from IPython.zmq.ipkernel import launch_kernel
+from IPython.zmq.kernelmanager import KernelManager
 from IPython.utils.traitlets import Instance, Dict, List, Unicode, Float, Int
 
 #-----------------------------------------------------------------------------
@@ -37,12 +39,14 @@ class DuplicateKernelError(Exception):
     pass
 
 
-class KernelManager(LoggingConfigurable):
+class MultiKernelManager(LoggingConfigurable):
     """A class for managing multiple kernels."""
 
     context = Instance('zmq.Context')
     def _context_default(self):
         return zmq.Context.instance()
+
+    connection_dir = Unicode('')
 
     _kernels = Dict()
 
@@ -64,18 +68,13 @@ def __contains__(self, kernel_id):
     def start_kernel(self, **kwargs):
         """Start a new kernel."""
         kernel_id = unicode(uuid.uuid4())
-        (process, shell_port, iopub_port, stdin_port, hb_port) = launch_kernel(**kwargs)
-        # Store the information for contacting the kernel. This assumes the kernel is
-        # running on localhost.
-        d = dict(
-            process = process,
-            stdin_port = stdin_port,
-            iopub_port = iopub_port,
-            shell_port = shell_port,
-            hb_port = hb_port,
-            ip = '127.0.0.1'
+        # use base KernelManager for each Kernel
+        km = KernelManager(connection_file=os.path.join(
+                    self.connection_dir, "kernel-%s.json" % kernel_id),
+                    config=self.config,
         )
-        self._kernels[kernel_id] = d
+        km.start_kernel(**kwargs)
+        self._kernels[kernel_id] = km
         return kernel_id
 
     def kill_kernel(self, kernel_id):
@@ -86,17 +85,8 @@ def kill_kernel(self, kernel_id):
         kernel_id : uuid
             The id of the kernel to kill.
         """
-        kernel_process = self.get_kernel_process(kernel_id)
-        if kernel_process is not None:
-            # Attempt to kill the kernel.
-            try:
-                kernel_process.kill()
-            except OSError, e:
-                # In Windows, we will get an Access Denied error if the process
-                # has already terminated. Ignore it.
-                if not (sys.platform == 'win32' and e.winerror == 5):
-                    raise
-            del self._kernels[kernel_id]
+        self.get_kernel(kernel_id).kill_kernel()
+        del self._kernels[kernel_id]
 
     def interrupt_kernel(self, kernel_id):
         """Interrupt (SIGINT) the kernel by its uuid.
@@ -106,14 +96,7 @@ def interrupt_kernel(self, kernel_id):
         kernel_id : uuid
             The id of the kernel to interrupt.
         """
-        kernel_process = self.get_kernel_process(kernel_id)
-        if kernel_process is not None:
-            if sys.platform == 'win32':
-                from parentpoller import ParentPollerWindows as Poller
-                Poller.send_interrupt(kernel_process.win32_interrupt_event)
-            else:
-                kernel_process.send_signal(signal.SIGINT)
-
+        return self.get_kernel(kernel_id).interrupt_kernel()
 
     def signal_kernel(self, kernel_id, signum):
         """ Sends a signal to the kernel by its uuid.
@@ -126,21 +109,19 @@ def signal_kernel(self, kernel_id, signum):
         kernel_id : uuid
             The id of the kernel to signal.
         """
-        kernel_process = self.get_kernel_process(kernel_id)
-        if kernel_process is not None:
-            kernel_process.send_signal(signum)
+        return self.get_kernel(kernel_id).signal_kernel(signum)
 
-    def get_kernel_process(self, kernel_id):
-        """Get the process object for a kernel by its uuid.
+    def get_kernel(self, kernel_id):
+        """Get the single KernelManager object for a kernel by its uuid.
 
         Parameters
         ==========
         kernel_id : uuid
             The id of the kernel.
         """
-        d = self._kernels.get(kernel_id)
-        if d is not None:
-            return d['process']
+        km = self._kernels.get(kernel_id)
+        if km is not None:
+            return km
         else:
             raise KeyError("Kernel with id not found: %s" % kernel_id)
 
@@ -159,14 +140,13 @@ def get_kernel_ports(self, kernel_id):
             (stdin_port,iopub_port,shell_port) and the values are the
             integer port numbers for those channels.
         """
-        d = self._kernels.get(kernel_id)
-        if d is not None:
-            dcopy = d.copy()
-            dcopy.pop('process')
-            dcopy.pop('ip')
-            return dcopy
-        else:
-            raise KeyError("Kernel with id not found: %s" % kernel_id)
+        # this will raise a KeyError if not found:
+        km = self.get_kernel(kernel_id)
+        return dict(shell_port=km.shell_port,
+                    iopub_port=km.iopub_port,
+                    stdin_port=km.stdin_port,
+                    hb_port=km.hb_port,
+        )
 
     def get_kernel_ip(self, kernel_id):
         """Return ip address for a kernel.
@@ -181,11 +161,7 @@ def get_kernel_ip(self, kernel_id):
         ip : str
             The ip address of the kernel.
         """
-        d = self._kernels.get(kernel_id)
-        if d is not None:
-            return d['ip']
-        else:
-            raise KeyError("Kernel with id not found: %s" % kernel_id)
+        return self.get_kernel(kernel_id).ip
 
     def create_connected_stream(self, ip, port, socket_type):
         sock = self.context.socket(socket_type)
@@ -214,7 +190,7 @@ def create_hb_stream(self, kernel_id):
         return hb_stream
 
 
-class MappingKernelManager(KernelManager):
+class MappingKernelManager(MultiKernelManager):
     """A KernelManager that handles notebok mapping and HTTP error handling"""
 
     kernel_argv = List(Unicode)
@@ -292,6 +268,13 @@ def interrupt_kernel(self, kernel_id):
     def restart_kernel(self, kernel_id):
         """Restart a kernel while keeping clients connected."""
         self._check_kernel_id(kernel_id)
+        km = self.get_kernel(kernel_id)
+        km.restart_kernel(now=True)
+        self.log.info("Kernel restarted: %s" % kernel_id)
+        return kernel_id
+
+        # the following remains, in case the KM restart machinery is
+        # somehow unacceptable
         # Get the notebook_id to preserve the kernel/notebook association.
         notebook_id = self.notebook_for_kernel(kernel_id)
         # Create the new kernel first so we can move the clients over.

IPython/frontend/html/notebook/notebookapp.py

@@ -44,7 +44,7 @@
 
 from IPython.core.application import BaseIPythonApplication
 from IPython.core.profiledir import ProfileDir
-from IPython.zmq.session import Session
+from IPython.zmq.session import Session, default_secure
 from IPython.zmq.zmqshell import ZMQInteractiveShell
 from IPython.zmq.ipkernel import (
     flags as ipkernel_flags,
@@ -128,6 +128,10 @@ def __init__(self, ipython_app, kernel_manager, notebook_manager, log):
     'notebook-dir': 'NotebookManager.notebook_dir',
 })
 
+# remove ipkernel flags that are singletons, and don't make sense in
+# multi-kernel evironment:
+aliases.pop('f', None)
+
 notebook_aliases = [u'port', u'ip', u'keyfile', u'certfile', u'ws-hostname',
                     u'notebook-dir']
 
@@ -212,19 +216,31 @@ def parse_command_line(self, argv=None):
         for a in argv:
             if a.startswith('-') and a.lstrip('-') in notebook_flags:
                 self.kernel_argv.remove(a)
+        swallow_next = False
         for a in argv:
+            if swallow_next:
+                self.kernel_argv.remove(a)
+                swallow_next = False
+                continue
             if a.startswith('-'):
-                alias = a.lstrip('-').split('=')[0]
+                split = a.lstrip('-').split('=')[0]
+                alias = split[0]
                 if alias in notebook_aliases:
                     self.kernel_argv.remove(a)
+                    if len(split) == 1:
+                        # alias passed with arg via space
+                        swallow_next = True
 
     def init_configurables(self):
         # Don't let Qt or ZMQ swallow KeyboardInterupts.
         signal.signal(signal.SIGINT, signal.SIG_DFL)
 
+        # force Session default to be secure
+        default_secure(self.config)
         # Create a KernelManager and start a kernel.
         self.kernel_manager = MappingKernelManager(
-            config=self.config, log=self.log, kernel_argv=self.kernel_argv
+            config=self.config, log=self.log, kernel_argv=self.kernel_argv,
+            connection_dir = self.profile_dir.security_dir,
         )
         self.notebook_manager = NotebookManager(config=self.config, log=self.log)
         self.notebook_manager.list_notebooks()

IPython/frontend/html/notebook/tests/test_kernelsession.py

@@ -2,12 +2,12 @@
 
 from unittest import TestCase
 
-from IPython.frontend.html.notebook.kernelmanager import KernelManager
+from IPython.frontend.html.notebook.kernelmanager import MultiKernelManager
 
 class TestKernelManager(TestCase):
 
     def test_km_lifecycle(self):
-        km = KernelManager()
+        km = MultiKernelManager()
         kid = km.start_kernel()
         self.assert_(kid in km)
         self.assertEquals(len(km),1)
@@ -21,6 +21,6 @@ def test_km_lifecycle(self):
         self.assert_('iopub_port' in port_dict)
         self.assert_('shell_port' in port_dict)
         self.assert_('hb_port' in port_dict)
-        km.get_kernel_process(kid)
+        km.get_kernel(kid)