Skip to content
Browse files

Merge PR #847 (connection files)

* JSON connection files are now used to connect files
* HMAC message signing is now on by default in all IPython apps
* Adds IPython.lib.kernel, which contains utility functions for connecting
  clients. These were mostly copied out of qtconsoleapp in order to be
  more general.
* Adds %connection_info and %qtconsole magics to zmqshell

closes gh-688
closes gh-806
closes gh-691
  • Loading branch information...
2 parents a924f50 + 48450ef commit f93a8ca2e129b46ec7b896e1c4658cbd676e3399 @minrk minrk committed Oct 13, 2011
View
8 IPython/frontend/html/notebook/handlers.py
@@ -173,7 +173,13 @@ def _on_zmq_reply(self, msg_list):
class AuthenticatedZMQStreamHandler(ZMQStreamHandler):
def open(self, kernel_id):
self.kernel_id = kernel_id.decode('ascii')
- self.session = Session()
+ try:
+ cfg = self.application.ipython_app.config
+ except AttributeError:
+ # protect from the case where this is run from something other than
+ # the notebook app:
+ cfg = None
+ self.session = Session(config=cfg)
self.save_on_message = self.on_message
self.on_message = self.on_first_message
View
89 IPython/frontend/html/notebook/kernelmanager.py
@@ -16,6 +16,7 @@
# Imports
#-----------------------------------------------------------------------------
+import os
import signal
import sys
import uuid
@@ -27,6 +28,7 @@
from IPython.config.configurable import LoggingConfigurable
from IPython.zmq.ipkernel import launch_kernel
+from IPython.zmq.kernelmanager import KernelManager
from IPython.utils.traitlets import Instance, Dict, List, Unicode, Float, Int
#-----------------------------------------------------------------------------
@@ -37,12 +39,14 @@ class DuplicateKernelError(Exception):
pass
-class KernelManager(LoggingConfigurable):
+class MultiKernelManager(LoggingConfigurable):
"""A class for managing multiple kernels."""
context = Instance('zmq.Context')
def _context_default(self):
return zmq.Context.instance()
+
+ connection_dir = Unicode('')
_kernels = Dict()
@@ -64,18 +68,13 @@ def __contains__(self, kernel_id):
def start_kernel(self, **kwargs):
"""Start a new kernel."""
kernel_id = unicode(uuid.uuid4())
- (process, shell_port, iopub_port, stdin_port, hb_port) = launch_kernel(**kwargs)
- # Store the information for contacting the kernel. This assumes the kernel is
- # running on localhost.
- d = dict(
- process = process,
- stdin_port = stdin_port,
- iopub_port = iopub_port,
- shell_port = shell_port,
- hb_port = hb_port,
- ip = '127.0.0.1'
+ # use base KernelManager for each Kernel
+ km = KernelManager(connection_file=os.path.join(
+ self.connection_dir, "kernel-%s.json" % kernel_id),
+ config=self.config,
)
- self._kernels[kernel_id] = d
+ km.start_kernel(**kwargs)
+ self._kernels[kernel_id] = km
return kernel_id
def kill_kernel(self, kernel_id):
@@ -86,17 +85,8 @@ def kill_kernel(self, kernel_id):
kernel_id : uuid
The id of the kernel to kill.
"""
- kernel_process = self.get_kernel_process(kernel_id)
- if kernel_process is not None:
- # Attempt to kill the kernel.
- try:
- kernel_process.kill()
- except OSError, e:
- # In Windows, we will get an Access Denied error if the process
- # has already terminated. Ignore it.
- if not (sys.platform == 'win32' and e.winerror == 5):
- raise
- del self._kernels[kernel_id]
+ self.get_kernel(kernel_id).kill_kernel()
+ del self._kernels[kernel_id]
def interrupt_kernel(self, kernel_id):
"""Interrupt (SIGINT) the kernel by its uuid.
@@ -106,14 +96,7 @@ def interrupt_kernel(self, kernel_id):
kernel_id : uuid
The id of the kernel to interrupt.
"""
- kernel_process = self.get_kernel_process(kernel_id)
- if kernel_process is not None:
- if sys.platform == 'win32':
- from parentpoller import ParentPollerWindows as Poller
- Poller.send_interrupt(kernel_process.win32_interrupt_event)
- else:
- kernel_process.send_signal(signal.SIGINT)
-
+ return self.get_kernel(kernel_id).interrupt_kernel()
def signal_kernel(self, kernel_id, signum):
""" Sends a signal to the kernel by its uuid.
@@ -126,21 +109,19 @@ def signal_kernel(self, kernel_id, signum):
kernel_id : uuid
The id of the kernel to signal.
"""
- kernel_process = self.get_kernel_process(kernel_id)
- if kernel_process is not None:
- kernel_process.send_signal(signum)
+ return self.get_kernel(kernel_id).signal_kernel(signum)
- def get_kernel_process(self, kernel_id):
- """Get the process object for a kernel by its uuid.
+ def get_kernel(self, kernel_id):
+ """Get the single KernelManager object for a kernel by its uuid.
Parameters
==========
kernel_id : uuid
The id of the kernel.
"""
- d = self._kernels.get(kernel_id)
- if d is not None:
- return d['process']
+ km = self._kernels.get(kernel_id)
+ if km is not None:
+ return km
else:
raise KeyError("Kernel with id not found: %s" % kernel_id)
@@ -159,14 +140,13 @@ def get_kernel_ports(self, kernel_id):
(stdin_port,iopub_port,shell_port) and the values are the
integer port numbers for those channels.
"""
- d = self._kernels.get(kernel_id)
- if d is not None:
- dcopy = d.copy()
- dcopy.pop('process')
- dcopy.pop('ip')
- return dcopy
- else:
- raise KeyError("Kernel with id not found: %s" % kernel_id)
+ # this will raise a KeyError if not found:
+ km = self.get_kernel(kernel_id)
+ return dict(shell_port=km.shell_port,
+ iopub_port=km.iopub_port,
+ stdin_port=km.stdin_port,
+ hb_port=km.hb_port,
+ )
def get_kernel_ip(self, kernel_id):
"""Return ip address for a kernel.
@@ -181,11 +161,7 @@ def get_kernel_ip(self, kernel_id):
ip : str
The ip address of the kernel.
"""
- d = self._kernels.get(kernel_id)
- if d is not None:
- return d['ip']
- else:
- raise KeyError("Kernel with id not found: %s" % kernel_id)
+ return self.get_kernel(kernel_id).ip
def create_connected_stream(self, ip, port, socket_type):
sock = self.context.socket(socket_type)
@@ -214,7 +190,7 @@ def create_hb_stream(self, kernel_id):
return hb_stream
-class MappingKernelManager(KernelManager):
+class MappingKernelManager(MultiKernelManager):
"""A KernelManager that handles notebok mapping and HTTP error handling"""
kernel_argv = List(Unicode)
@@ -292,6 +268,13 @@ def interrupt_kernel(self, kernel_id):
def restart_kernel(self, kernel_id):
"""Restart a kernel while keeping clients connected."""
self._check_kernel_id(kernel_id)
+ km = self.get_kernel(kernel_id)
+ km.restart_kernel(now=True)
+ self.log.info("Kernel restarted: %s" % kernel_id)
+ return kernel_id
+
+ # the following remains, in case the KM restart machinery is
+ # somehow unacceptable
# Get the notebook_id to preserve the kernel/notebook association.
notebook_id = self.notebook_for_kernel(kernel_id)
# Create the new kernel first so we can move the clients over.
View
22 IPython/frontend/html/notebook/notebookapp.py
@@ -44,7 +44,7 @@
from IPython.core.application import BaseIPythonApplication
from IPython.core.profiledir import ProfileDir
-from IPython.zmq.session import Session
+from IPython.zmq.session import Session, default_secure
from IPython.zmq.zmqshell import ZMQInteractiveShell
from IPython.zmq.ipkernel import (
flags as ipkernel_flags,
@@ -128,6 +128,10 @@ def __init__(self, ipython_app, kernel_manager, notebook_manager, log):
'notebook-dir': 'NotebookManager.notebook_dir',
})
+# remove ipkernel flags that are singletons, and don't make sense in
+# multi-kernel evironment:
+aliases.pop('f', None)
+
notebook_aliases = [u'port', u'ip', u'keyfile', u'certfile', u'ws-hostname',
u'notebook-dir']
@@ -212,19 +216,31 @@ def parse_command_line(self, argv=None):
for a in argv:
if a.startswith('-') and a.lstrip('-') in notebook_flags:
self.kernel_argv.remove(a)
+ swallow_next = False
for a in argv:
+ if swallow_next:
+ self.kernel_argv.remove(a)
+ swallow_next = False
+ continue
if a.startswith('-'):
- alias = a.lstrip('-').split('=')[0]
+ split = a.lstrip('-').split('=')[0]
+ alias = split[0]
if alias in notebook_aliases:
self.kernel_argv.remove(a)
+ if len(split) == 1:
+ # alias passed with arg via space
+ swallow_next = True
def init_configurables(self):
# Don't let Qt or ZMQ swallow KeyboardInterupts.
signal.signal(signal.SIGINT, signal.SIG_DFL)
+ # force Session default to be secure
+ default_secure(self.config)
# Create a KernelManager and start a kernel.
self.kernel_manager = MappingKernelManager(
- config=self.config, log=self.log, kernel_argv=self.kernel_argv
+ config=self.config, log=self.log, kernel_argv=self.kernel_argv,
+ connection_dir = self.profile_dir.security_dir,
)
self.notebook_manager = NotebookManager(config=self.config, log=self.log)
self.notebook_manager.list_notebooks()
View
6 IPython/frontend/html/notebook/tests/test_kernelsession.py
@@ -2,12 +2,12 @@
from unittest import TestCase
-from IPython.frontend.html.notebook.kernelmanager import KernelManager
+from IPython.frontend.html.notebook.kernelmanager import MultiKernelManager
class TestKernelManager(TestCase):
def test_km_lifecycle(self):
- km = KernelManager()
+ km = MultiKernelManager()
kid = km.start_kernel()
self.assert_(kid in km)
self.assertEquals(len(km),1)
@@ -21,6 +21,6 @@ def test_km_lifecycle(self):
self.assert_('iopub_port' in port_dict)
self.assert_('shell_port' in port_dict)
self.assert_('hb_port' in port_dict)
- km.get_kernel_process(kid)
+ km.get_kernel(kid)
View
176 IPython/frontend/qt/console/qtconsoleapp.py
@@ -20,25 +20,24 @@
import os
import signal
import sys
-from getpass import getpass
# System library imports
from IPython.external.qt import QtGui
from pygments.styles import get_all_styles
-
-# external imports
-from IPython.external.ssh import tunnel
+from zmq.utils import jsonapi as json
# Local imports
from IPython.config.application import boolean_flag
from IPython.core.application import BaseIPythonApplication
from IPython.core.profiledir import ProfileDir
+from IPython.lib.kernel import tunnel_to_kernel, find_connection_file
from IPython.frontend.qt.console.frontend_widget import FrontendWidget
from IPython.frontend.qt.console.ipython_widget import IPythonWidget
from IPython.frontend.qt.console.rich_ipython_widget import RichIPythonWidget
from IPython.frontend.qt.console import styles
from IPython.frontend.qt.kernelmanager import QtKernelManager
-from IPython.parallel.util import select_random_ports
+from IPython.utils.path import filefind
+from IPython.utils.py3compat import str_to_bytes
from IPython.utils.traitlets import (
Dict, List, Unicode, Int, CaselessStrEnum, CBool, Any
)
@@ -47,7 +46,7 @@
aliases as ipkernel_aliases,
IPKernelApp
)
-from IPython.zmq.session import Session
+from IPython.zmq.session import Session, default_secure
from IPython.zmq.zmqshell import ZMQInteractiveShell
@@ -182,8 +181,8 @@ def closeEvent(self, event):
flags = dict(ipkernel_flags)
qt_flags = {
- 'existing' : ({'IPythonQtConsoleApp' : {'existing' : True}},
- "Connect to an existing kernel."),
+ 'existing' : ({'IPythonQtConsoleApp' : {'existing' : 'kernel*.json'}},
+ "Connect to an existing kernel. If no argument specified, guess most recent"),
'pure' : ({'IPythonQtConsoleApp' : {'pure' : True}},
"Use a pure Python kernel instead of an IPython kernel."),
'plain' : ({'ConsoleWidget' : {'kind' : 'plain'}},
@@ -204,10 +203,6 @@ def closeEvent(self, event):
"""
))
flags.update(qt_flags)
-# the flags that are specific to the frontend
-# these must be scrubbed before being passed to the kernel,
-# or it will raise an error on unrecognized flags
-qt_flags = qt_flags.keys()
aliases = dict(ipkernel_aliases)
@@ -217,6 +212,8 @@ def closeEvent(self, event):
iopub = 'IPythonQtConsoleApp.iopub_port',
stdin = 'IPythonQtConsoleApp.stdin_port',
ip = 'IPythonQtConsoleApp.ip',
+ existing = 'IPythonQtConsoleApp.existing',
+ f = 'IPythonQtConsoleApp.connection_file',
style = 'IPythonWidget.syntax_style',
stylesheet = 'IPythonQtConsoleApp.stylesheet',
@@ -227,8 +224,6 @@ def closeEvent(self, event):
ssh = 'IPythonQtConsoleApp.sshserver',
)
aliases.update(qt_aliases)
-# also scrub aliases from the frontend
-qt_flags.extend(qt_aliases.keys())
#-----------------------------------------------------------------------------
@@ -286,9 +281,18 @@ class IPythonQtConsoleApp(BaseIPythonApplication):
help="set the iopub (PUB) port [default: random]")
stdin_port = Int(0, config=True,
help="set the stdin (XREQ) port [default: random]")
+ connection_file = Unicode('', config=True,
+ help="""JSON file in which to store connection info [default: kernel-<pid>.json]
+
+ This file will contain the IP, ports, and authentication key needed to connect
+ clients to this kernel. By default, this file will be created in the security-dir
+ of the current profile, but can be specified by absolute path.
+ """)
+ def _connection_file_default(self):
+ return 'kernel-%i.json' % os.getpid()
- existing = CBool(False, config=True,
- help="Whether to connect to an already running Kernel.")
+ existing = Unicode('', config=True,
+ help="""Connect to an already running kernel""")
stylesheet = Unicode('', config=True,
help="path to a custom CSS stylesheet")
@@ -327,62 +331,144 @@ def parse_command_line(self, argv=None):
self.kernel_argv = list(argv) # copy
# kernel should inherit default config file from frontend
self.kernel_argv.append("--KernelApp.parent_appname='%s'"%self.name)
- # scrub frontend-specific flags
+ # Scrub frontend-specific flags
+ for a in argv:
+ if a.startswith('-') and a.lstrip('-') in qt_flags:
+ self.kernel_argv.remove(a)
+ swallow_next = False
for a in argv:
-
+ if swallow_next:
+ self.kernel_argv.remove(a)
+ swallow_next = False
+ continue
if a.startswith('-'):
- key = a.lstrip('-').split('=')[0]
- if key in qt_flags:
+ split = a.lstrip('-').split('=')[0]
+ alias = split[0]
+ if alias in qt_aliases:
self.kernel_argv.remove(a)
+ if len(split) == 1:
+ # alias passed with arg via space
+ swallow_next = True
+
+ def init_connection_file(self):
+ """find the connection file, and load the info if found.
+
+ The current working directory and the current profile's security
+ directory will be searched for the file if it is not given by
+ absolute path.
+
+ When attempting to connect to an existing kernel and the `--existing`
+ argument does not match an existing file, it will be interpreted as a
+ fileglob, and the matching file in the current profile's security dir
+ with the latest access time will be used.
+ """
+ if self.existing:
+ try:
+ cf = find_connection_file(self.existing)
+ except Exception:
+ self.log.critical("Could not find existing kernel connection file %s", self.existing)
+ self.exit(1)
+ self.log.info("Connecting to existing kernel: %s" % cf)
+ self.connection_file = cf
+ # should load_connection_file only be used for existing?
+ # as it is now, this allows reusing ports if an existing
+ # file is requested
+ self.load_connection_file()
+
+ def load_connection_file(self):
+ """load ip/port/hmac config from JSON connection file"""
+ # this is identical to KernelApp.load_connection_file
+ # perhaps it can be centralized somewhere?
+ try:
+ fname = filefind(self.connection_file, ['.', self.profile_dir.security_dir])
+ except IOError:
+ self.log.debug("Connection File not found: %s", self.connection_file)
+ return
+ self.log.debug(u"Loading connection file %s", fname)
+ with open(fname) as f:
+ s = f.read()
+ cfg = json.loads(s)
+ if self.ip == LOCALHOST and 'ip' in cfg:
+ # not overridden by config or cl_args
+ self.ip = cfg['ip']
+ for channel in ('hb', 'shell', 'iopub', 'stdin'):
+ name = channel + '_port'
+ if getattr(self, name) == 0 and name in cfg:
+ # not overridden by config or cl_args
+ setattr(self, name, cfg[name])
+ if 'key' in cfg:
+ self.config.Session.key = str_to_bytes(cfg['key'])
def init_ssh(self):
"""set up ssh tunnels, if needed."""
if not self.sshserver and not self.sshkey:
return
if self.sshkey and not self.sshserver:
+ # specifying just the key implies that we are connecting directly
self.sshserver = self.ip
- self.ip=LOCALHOST
+ self.ip = LOCALHOST
- lports = select_random_ports(4)
- rports = self.shell_port, self.iopub_port, self.stdin_port, self.hb_port
- self.shell_port, self.iopub_port, self.stdin_port, self.hb_port = lports
+ # build connection dict for tunnels:
+ info = dict(ip=self.ip,
+ shell_port=self.shell_port,
+ iopub_port=self.iopub_port,
+ stdin_port=self.stdin_port,
+ hb_port=self.hb_port
+ )
- remote_ip = self.ip
- self.ip = LOCALHOST
- self.log.info("Forwarding connections to %s via %s"%(remote_ip, self.sshserver))
+ self.log.info("Forwarding connections to %s via %s"%(self.ip, self.sshserver))
- if tunnel.try_passwordless_ssh(self.sshserver, self.sshkey):
- password=False
- else:
- password = getpass("SSH Password for %s: "%self.sshserver)
+ # tunnels return a new set of ports, which will be on localhost:
+ self.ip = LOCALHOST
+ try:
+ newports = tunnel_to_kernel(info, self.sshserver, self.sshkey)
+ except:
+ # even catch KeyboardInterrupt
+ self.log.error("Could not setup tunnels", exc_info=True)
+ self.exit(1)
- for lp,rp in zip(lports, rports):
- tunnel.ssh_tunnel(lp, rp, self.sshserver, remote_ip, self.sshkey, password)
+ self.shell_port, self.iopub_port, self.stdin_port, self.hb_port = newports
- self.log.critical("To connect another client to this tunnel, use:")
- self.log.critical(
- "--existing --shell={0} --iopub={1} --stdin={2} --hb={3}".format(
- self.shell_port, self.iopub_port, self.stdin_port,
- self.hb_port))
+ cf = self.connection_file
+ base,ext = os.path.splitext(cf)
+ base = os.path.basename(base)
+ self.connection_file = os.path.basename(base)+'-ssh'+ext
+ self.log.critical("To connect another client via this tunnel, use:")
+ self.log.critical("--existing %s" % self.connection_file)
def init_kernel_manager(self):
# Don't let Qt or ZMQ swallow KeyboardInterupts.
signal.signal(signal.SIGINT, signal.SIG_DFL)
+ sec = self.profile_dir.security_dir
+ try:
+ cf = filefind(self.connection_file, ['.', sec])
+ except IOError:
+ # file might not exist
+ if self.connection_file == os.path.basename(self.connection_file):
+ # just shortname, put it in security dir
+ cf = os.path.join(sec, self.connection_file)
+ else:
+ cf = self.connection_file
# Create a KernelManager and start a kernel.
self.kernel_manager = QtKernelManager(
- shell_address=(self.ip, self.shell_port),
- sub_address=(self.ip, self.iopub_port),
- stdin_address=(self.ip, self.stdin_port),
- hb_address=(self.ip, self.hb_port),
- config=self.config
+ ip=self.ip,
+ shell_port=self.shell_port,
+ iopub_port=self.iopub_port,
+ stdin_port=self.stdin_port,
+ hb_port=self.hb_port,
+ connection_file=cf,
+ config=self.config,
)
# start the kernel
if not self.existing:
- kwargs = dict(ip=self.ip, ipython=not self.pure)
+ kwargs = dict(ipython=not self.pure)
kwargs['extra_arguments'] = self.kernel_argv
self.kernel_manager.start_kernel(**kwargs)
+ elif self.sshserver:
+ # ssh, write new connection file
+ self.kernel_manager.write_connection_file()
self.kernel_manager.start_channels()
@@ -463,6 +549,8 @@ def init_colors(self):
def initialize(self, argv=None):
super(IPythonQtConsoleApp, self).initialize(argv)
+ self.init_connection_file()
+ default_secure(self.config)
self.init_ssh()
self.init_kernel_manager()
self.init_qt_elements()
View
255 IPython/lib/kernel.py
@@ -0,0 +1,255 @@
+"""Utilities for connecting to kernels
+
+Authors:
+
+* Min Ragan-Kelley
+
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (C) 2011 The IPython Development Team
+#
+# Distributed under the terms of the BSD License. The full license is in
+# the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import glob
+import json
+import os
+import sys
+from getpass import getpass
+from subprocess import Popen, PIPE
+
+# external imports
+from IPython.external.ssh import tunnel
+
+# IPython imports
+from IPython.core.profiledir import ProfileDir
+from IPython.utils.path import filefind, get_ipython_dir
+from IPython.utils.py3compat import str_to_bytes
+
+
+#-----------------------------------------------------------------------------
+# Functions
+#-----------------------------------------------------------------------------
+
+def get_connection_file(app=None):
+ """Return the path to the connection file of an app
+
+ Parameters
+ ----------
+ app : KernelApp instance [optional]
+ If unspecified, the currently running app will be used
+ """
+ if app is None:
+ from IPython.zmq.kernelapp import KernelApp
+ if not KernelApp.initialized():
+ raise RuntimeError("app not specified, and not in a running Kernel")
+
+ app = KernelApp.instance()
+ return filefind(app.connection_file, ['.', app.profile_dir.security_dir])
+
+def find_connection_file(filename, profile=None):
+ """find a connection file, and return its absolute path.
+
+ The current working directory and the profile's security
+ directory will be searched for the file if it is not given by
+ absolute path.
+
+ If profile is unspecified, then the current running application's
+ profile will be used, or 'default', if not run from IPython.
+
+ If the argument does not match an existing file, it will be interpreted as a
+ fileglob, and the matching file in the profile's security dir with
+ the latest access time will be used.
+
+ Parameters
+ ----------
+ filename : str
+ The connection file or fileglob to search for.
+ profile : str [optional]
+ The name of the profile to use when searching for the connection file,
+ if different from the current IPython session or 'default'.
+
+ Returns
+ -------
+ str : The absolute path of the connection file.
+ """
+ from IPython.core.application import BaseIPythonApplication as IPApp
+ try:
+ # quick check for absolute path, before going through logic
+ return filefind(filename)
+ except IOError:
+ pass
+
+ if profile is None:
+ # profile unspecified, check if running from an IPython app
+ if IPApp.initialized():
+ app = IPApp.instance()
+ profile_dir = app.profile_dir
+ else:
+ # not running in IPython, use default profile
+ profile_dir = ProfileDir.find_profile_dir_by_name(get_ipython_dir(), 'default')
+ else:
+ # find profiledir by profile name:
+ profile_dir = ProfileDir.find_profile_dir_by_name(get_ipython_dir(), profile)
+ security_dir = profile_dir.security_dir
+
+ try:
+ # first, try explicit name
+ return filefind(filename, ['.', security_dir])
+ except IOError:
+ pass
+
+ # not found by full name
+
+ if '*' in filename:
+ # given as a glob already
+ pat = filename
+ else:
+ # accept any substring match
+ pat = '*%s*' % filename
+ matches = glob.glob( os.path.join(security_dir, pat) )
+ if not matches:
+ raise IOError("Could not find %r in %r" % (filename, security_dir))
+ elif len(matches) == 1:
+ return matches[0]
+ else:
+ # get most recent match, by access time:
+ return sorted(matches, key=lambda f: os.stat(f).st_atime)[-1]
+
+def get_connection_info(connection_file=None, unpack=False, profile=None):
+ """Return the connection information for the current Kernel.
+
+ Parameters
+ ----------
+ connection_file : str [optional]
+ The connection file to be used. Can be given by absolute path, or
+ IPython will search in the security directory of a given profile.
+ If run from IPython,
+
+ If unspecified, the connection file for the currently running
+ IPython Kernel will be used, which is only allowed from inside a kernel.
+ unpack : bool [default: False]
+ if True, return the unpacked dict, otherwise just the string contents
+ of the file.
+ profile : str [optional]
+ The name of the profile to use when searching for the connection file,
+ if different from the current IPython session or 'default'.
+
+
+ Returns
+ -------
+ The connection dictionary of the current kernel, as string or dict,
+ depending on `unpack`.
+ """
+ if connection_file is None:
+ # get connection file from current kernel
+ cf = get_connection_file()
+ else:
+ # connection file specified, allow shortnames:
+ cf = find_connection_file(connection_file, profile=profile)
+
+ with open(cf) as f:
+ info = f.read()
+
+ if unpack:
+ info = json.loads(info)
+ # ensure key is bytes:
+ info['key'] = str_to_bytes(info.get('key', ''))
+ return info
+
+def connect_qtconsole(connection_file=None, argv=None, profile=None):
+ """Connect a qtconsole to the current kernel.
+
+ This is useful for connecting a second qtconsole to a kernel, or to a
+ local notebook.
+
+ Parameters
+ ----------
+ connection_file : str [optional]
+ The connection file to be used. Can be given by absolute path, or
+ IPython will search in the security directory of a given profile.
+ If run from IPython,
+
+ If unspecified, the connection file for the currently running
+ IPython Kernel will be used, which is only allowed from inside a kernel.
+ argv : list [optional]
+ Any extra args to be passed to the console.
+ profile : str [optional]
+ The name of the profile to use when searching for the connection file,
+ if different from the current IPython session or 'default'.
+
+
+ Returns
+ -------
+ subprocess.Popen instance running the qtconsole frontend
+ """
+ argv = [] if argv is None else argv
+
+ if connection_file is None:
+ # get connection file from current kernel
+ cf = get_connection_file()
+ else:
+ cf = find_connection_file(connection_file, profile=profile)
+
+ cmd = ';'.join([
+ "from IPython.frontend.qt.console import qtconsoleapp",
+ "qtconsoleapp.main()"
+ ])
+
+ return Popen([sys.executable, '-c', cmd, '--existing', cf] + argv, stdout=PIPE, stderr=PIPE)
+
+def tunnel_to_kernel(connection_info, sshserver, sshkey=None):
+ """tunnel connections to a kernel via ssh
+
+ This will open four SSH tunnels from localhost on this machine to the
+ ports associated with the kernel. They can be either direct
+ localhost-localhost tunnels, or if an intermediate server is necessary,
+ the kernel must be listening on a public IP.
+
+ Parameters
+ ----------
+ connection_info : dict or str (path)
+ Either a connection dict, or the path to a JSON connection file
+ sshserver : str
+ The ssh sever to use to tunnel to the kernel. Can be a full
+ `user@server:port` string. ssh config aliases are respected.
+ sshkey : str [optional]
+ Path to file containing ssh key to use for authentication.
+ Only necessary if your ssh config does not already associate
+ a keyfile with the host.
+
+ Returns
+ -------
+
+ (shell, iopub, stdin, hb) : ints
+ The four ports on localhost that have been forwarded to the kernel.
+ """
+ if isinstance(connection_info, basestring):
+ # it's a path, unpack it
+ with open(connection_info) as f:
+ connection_info = json.loads(f.read())
+
+ cf = connection_info
+
+ lports = tunnel.select_random_ports(4)
+ rports = cf['shell_port'], cf['iopub_port'], cf['stdin_port'], cf['hb_port']
+
+ remote_ip = cf['ip']
+
+ if tunnel.try_passwordless_ssh(sshserver, sshkey):
+ password=False
+ else:
+ password = getpass("SSH Password for %s: "%sshserver)
+
+ for lp,rp in zip(lports, rports):
+ tunnel.ssh_tunnel(lp, rp, sshserver, remote_ip, sshkey, password)
+
+ return tuple(lports)
+
+
View
61 IPython/parallel/apps/ipcontrollerapp.py
@@ -27,7 +27,6 @@
import socket
import stat
import sys
-import uuid
from multiprocessing import Process
@@ -36,7 +35,6 @@
from zmq.log.handlers import PUBHandler
from zmq.utils import jsonapi as json
-from IPython.config.application import boolean_flag
from IPython.core.profiledir import ProfileDir
from IPython.parallel.apps.baseapp import (
@@ -47,8 +45,10 @@
from IPython.utils.importstring import import_item
from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict
-# from IPython.parallel.controller.controller import ControllerFactory
-from IPython.zmq.session import Session
+from IPython.zmq.session import (
+ Session, session_aliases, session_flags, default_secure
+)
+
from IPython.parallel.controller.heartmonitor import HeartMonitor
from IPython.parallel.controller.hub import HubFactory
from IPython.parallel.controller.scheduler import TaskScheduler,launch_scheduler
@@ -109,20 +109,13 @@
'reuse existing json connection files')
})
-flags.update(boolean_flag('secure', 'IPControllerApp.secure',
- "Use HMAC digests for authentication of messages.",
- "Don't authenticate messages."
-))
+flags.update(session_flags)
+
aliases = dict(
- secure = 'IPControllerApp.secure',
ssh = 'IPControllerApp.ssh_server',
enginessh = 'IPControllerApp.engine_ssh_server',
location = 'IPControllerApp.location',
- ident = 'Session.session',
- user = 'Session.username',
- keyfile = 'Session.keyfile',
-
url = 'HubFactory.url',
ip = 'HubFactory.ip',
transport = 'HubFactory.transport',
@@ -134,6 +127,7 @@
hwm = 'TaskScheduler.hwm',
)
aliases.update(base_aliases)
+aliases.update(session_aliases)
class IPControllerApp(BaseParallelApplication):
@@ -151,9 +145,6 @@ class IPControllerApp(BaseParallelApplication):
reuse_files = Bool(False, config=True,
help='Whether to reuse existing json connection files.'
)
- secure = Bool(True, config=True,
- help='Whether to use HMAC digests for extra message authentication.'
- )
ssh_server = Unicode(u'', config=True,
help="""ssh url for clients to use when connecting to the Controller
processes. It should be of the form: [user@]server[:port]. The
@@ -225,6 +216,7 @@ def save_connection_dict(self, fname, cdict):
def load_config_from_json(self):
"""load config from existing json connector files."""
c = self.config
+ self.log.debug("loading config from JSON")
# load from engine config
with open(os.path.join(self.profile_dir.security_dir, self.engine_json_file)) as f:
cfg = json.loads(f.read())
@@ -249,28 +241,23 @@ def load_config_from_json(self):
self.ssh_server = cfg['ssh']
assert int(ports) == c.HubFactory.regport, "regport mismatch"
+ def load_secondary_config(self):
+ """secondary config, loading from JSON and setting defaults"""
+ if self.reuse_files:
+ try:
+ self.load_config_from_json()
+ except (AssertionError,IOError) as e:
+ self.log.error("Could not load config from JSON: %s" % e)
+ self.reuse_files=False
+ # switch Session.key default to secure
+ default_secure(self.config)
+ self.log.debug("Config changed")
+ self.log.debug(repr(self.config))
+
def init_hub(self):
c = self.config
self.do_import_statements()
- reusing = self.reuse_files
- if reusing:
- try:
- self.load_config_from_json()
- except (AssertionError,IOError):
- reusing=False
- # check again, because reusing may have failed:
- if reusing:
- pass
- elif self.secure:
- key = str(uuid.uuid4())
- # keyfile = os.path.join(self.profile_dir.security_dir, self.exec_key)
- # with open(keyfile, 'w') as f:
- # f.write(key)
- # os.chmod(keyfile, stat.S_IRUSR|stat.S_IWUSR)
- c.Session.key = asbytes(key)
- else:
- key = c.Session.key = b''
try:
self.factory = HubFactory(config=c, log=self.log)
@@ -280,10 +267,10 @@ def init_hub(self):
self.log.error("Couldn't construct the Controller", exc_info=True)
self.exit(1)
- if not reusing:
+ if not self.reuse_files:
# save to new json config files
f = self.factory
- cdict = {'exec_key' : key,
+ cdict = {'exec_key' : f.session.key,
'ssh' : self.ssh_server,
'url' : "%s://%s:%s"%(f.client_transport, f.client_ip, f.regport),
'location' : self.location
@@ -397,11 +384,11 @@ def forward_logging(self):
handler.setLevel(self.log_level)
self.log.addHandler(handler)
self._log_handler = handler
- # #
def initialize(self, argv=None):
super(IPControllerApp, self).initialize(argv)
self.forward_logging()
+ self.load_secondary_config()
self.init_hub()
self.init_schedulers()
View
24 IPython/parallel/apps/ipengineapp.py
@@ -36,9 +36,12 @@
base_flags,
)
from IPython.zmq.log import EnginePUBHandler
+from IPython.zmq.session import (
+ Session, session_aliases, session_flags
+)
from IPython.config.configurable import Configurable
-from IPython.zmq.session import Session
+
from IPython.parallel.engine.engine import EngineFactory
from IPython.parallel.engine.streamkernel import Kernel
from IPython.parallel.util import disambiguate_url, asbytes
@@ -113,10 +116,6 @@ def _use_changed(self, name, old, new):
c = 'IPEngineApp.startup_command',
s = 'IPEngineApp.startup_script',
- ident = 'Session.session',
- user = 'Session.username',
- keyfile = 'Session.keyfile',
-
url = 'EngineFactory.url',
ssh = 'EngineFactory.sshserver',
sshkey = 'EngineFactory.sshkey',
@@ -131,7 +130,10 @@ def _use_changed(self, name, old, new):
)
aliases.update(base_aliases)
-
+aliases.update(session_aliases)
+flags = {}
+flags.update(base_flags)
+flags.update(session_flags)
class IPEngineApp(BaseParallelApplication):
@@ -172,6 +174,7 @@ def _cluster_id_changed(self, name, old, new):
logging to a central location.""")
aliases = Dict(aliases)
+ flags = Dict(flags)
# def find_key_file(self):
# """Set the key file.
@@ -214,12 +217,9 @@ def load_connector_file(self):
with open(self.url_file) as f:
d = json.loads(f.read())
- try:
- config.Session.key
- except AttributeError:
- if d['exec_key']:
- config.Session.key = asbytes(d['exec_key'])
-
+ if 'exec_key' in d:
+ config.Session.key = asbytes(d['exec_key'])
+
try:
config.EngineFactory.location
except AttributeError:
View
31 IPython/utils/path.py
@@ -476,3 +476,34 @@ def check_for_old_config(ipython_dir=None):
IPython and want to suppress this warning message, set
`c.InteractiveShellApp.ignore_old_config=True` in the new config.""")
+def get_security_file(filename, profile='default'):
+ """Return the absolute path of a security file given by filename and profile
+
+ This allows users and developers to find security files without
+ knowledge of the IPython directory structure. The search path
+ will be ['.', profile.security_dir]
+
+ Parameters
+ ----------
+
+ filename : str
+ The file to be found. If it is passed as an absolute path, it will
+ simply be returned.
+ profile : str [default: 'default']
+ The name of the profile to search. Leaving this unspecified
+ The file to be found. If it is passed as an absolute path, fname will
+ simply be returned.
+
+ Returns
+ -------
+ Raises :exc:`IOError` if file not found or returns absolute path to file.
+ """
+ # import here, because profiledir also imports from utils.path
+ from IPython.core.profiledir import ProfileDir
+ try:
+ pd = ProfileDir.find_profile_dir_by_name(get_ipython_dir(), profile)
+ except Exception:
+ # will raise ProfileDirError if no such profile
+ raise IOError("Profile %r not found")
+ return filefind(filename, ['.', pd.security_dir])
+
View
107 IPython/zmq/entry_point.py
@@ -8,20 +8,27 @@
import socket
from subprocess import Popen, PIPE
import sys
+import tempfile
-# Local imports.
-from parentpoller import ParentPollerWindows
+# System library imports
+from zmq.utils import jsonapi as json
+# IPython imports
+from IPython.utils.localinterfaces import LOCALHOST
+from IPython.utils.py3compat import bytes_to_str
-def base_launch_kernel(code, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0,
- ip=None, stdin=None, stdout=None, stderr=None,
- executable=None, independent=False, extra_arguments=[]):
- """ Launches a localhost kernel, binding to the specified ports.
+# Local imports.
+from parentpoller import ParentPollerWindows
+def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0,
+ ip=LOCALHOST, key=b''):
+ """Generates a JSON config file, including the selection of random ports.
+
Parameters
----------
- code : str,
- A string of Python code that imports and executes a kernel entry point.
+
+ fname : unicode
+ The path to the file to write
shell_port : int, optional
The port to use for XREP channel.
@@ -38,27 +45,14 @@ def base_launch_kernel(code, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0
ip : str, optional
The ip address the kernel will bind to.
- stdin, stdout, stderr : optional (default None)
- Standards streams, as defined in subprocess.Popen.
+ key : str, optional
+ The Session key used for HMAC authentication.
- executable : str, optional (default sys.executable)
- The Python executable to use for the kernel process.
-
- independent : bool, optional (default False)
- If set, the kernel process is guaranteed to survive if this process
- dies. If not set, an effort is made to ensure that the kernel is killed
- when this process dies. Note that in this case it is still good practice
- to kill kernels manually before exiting.
-
- extra_arguments = list, optional
- A list of extra arguments to pass when executing the launch code.
-
- Returns
- -------
- A tuple of form:
- (kernel_process, shell_port, iopub_port, stdin_port, hb_port)
- where kernel_process is a Popen object and the ports are integers.
"""
+ # default to temporary connector file
+ if not fname:
+ fname = tempfile.mktemp('.json')
+
# Find open ports as necessary.
ports = []
ports_needed = int(shell_port <= 0) + int(iopub_port <= 0) + \
@@ -79,15 +73,62 @@ def base_launch_kernel(code, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0
stdin_port = ports.pop(0)
if hb_port <= 0:
hb_port = ports.pop(0)
+
+ cfg = dict( shell_port=shell_port,
+ iopub_port=iopub_port,
+ stdin_port=stdin_port,
+ hb_port=hb_port,
+ )
+ cfg['ip'] = ip
+ cfg['key'] = bytes_to_str(key)
+
+ with open(fname, 'wb') as f:
+ f.write(json.dumps(cfg, indent=2))
+
+ return fname, cfg
+
+
+def base_launch_kernel(code, fname, stdin=None, stdout=None, stderr=None,
+ executable=None, independent=False, extra_arguments=[]):
+ """ Launches a localhost kernel, binding to the specified ports.
+
+ Parameters
+ ----------
+ code : str,
+ A string of Python code that imports and executes a kernel entry point.
+ stdin, stdout, stderr : optional (default None)
+ Standards streams, as defined in subprocess.Popen.
+
+ fname : unicode, optional
+ The JSON connector file, containing ip/port/hmac key information.
+
+ key : str, optional
+ The Session key used for HMAC authentication.
+
+ executable : str, optional (default sys.executable)
+ The Python executable to use for the kernel process.
+
+ independent : bool, optional (default False)
+ If set, the kernel process is guaranteed to survive if this process
+ dies. If not set, an effort is made to ensure that the kernel is killed
+ when this process dies. Note that in this case it is still good practice
+ to kill kernels manually before exiting.
+
+ extra_arguments = list, optional
+ A list of extra arguments to pass when executing the launch code.
+
+ Returns
+ -------
+ A tuple of form:
+ (kernel_process, shell_port, iopub_port, stdin_port, hb_port)
+ where kernel_process is a Popen object and the ports are integers.
+ """
+
# Build the kernel launch command.
if executable is None:
executable = sys.executable
- arguments = [ executable, '-c', code, '--shell=%i'%shell_port,
- '--iopub=%i'%iopub_port, '--stdin=%i'%stdin_port,
- '--hb=%i'%hb_port ]
- if ip is not None:
- arguments.append('--ip=%s'%ip)
+ arguments = [ executable, '-c', code, '-f', fname ]
arguments.extend(extra_arguments)
# Popen will fail (sometimes with a deadlock) if stdin, stdout, and stderr
@@ -164,4 +205,4 @@ def base_launch_kernel(code, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0
if stderr is None:
proc.stderr.close()
- return proc, shell_port, iopub_port, stdin_port, hb_port
+ return proc
View
76 IPython/zmq/kernelapp.py
@@ -21,6 +21,7 @@
# System library imports.
import zmq
+from zmq.utils import jsonapi as json
# IPython imports.
from IPython.core.ultratb import FormattedTB
@@ -29,13 +30,18 @@
)
from IPython.utils import io
from IPython.utils.localinterfaces import LOCALHOST
+from IPython.utils.path import filefind
+from IPython.utils.py3compat import str_to_bytes
from IPython.utils.traitlets import (Any, Instance, Dict, Unicode, Int, Bool,
DottedObjectName)
from IPython.utils.importstring import import_item
# local imports
+from IPython.zmq.entry_point import write_connection_file
from IPython.zmq.heartbeat import Heartbeat
from IPython.zmq.parentpoller import ParentPollerUnix, ParentPollerWindows
-from IPython.zmq.session import Session
+from IPython.zmq.session import (
+ Session, session_flags, session_aliases, default_secure,
+)
#-----------------------------------------------------------------------------
@@ -49,6 +55,7 @@
'shell' : 'KernelApp.shell_port',
'iopub' : 'KernelApp.iopub_port',
'stdin' : 'KernelApp.stdin_port',
+ 'f' : 'KernelApp.connection_file',
'parent': 'KernelApp.parent',
})
if sys.platform.startswith('win'):
@@ -64,6 +71,11 @@
"redirect stderr to the null device"),
})
+# inherit flags&aliases for Sessions
+kernel_aliases.update(session_aliases)
+kernel_flags.update(session_flags)
+
+
#-----------------------------------------------------------------------------
# Application class for starting a Kernel
@@ -99,6 +111,13 @@ def _parent_appname_changed(self, name, old, new):
shell_port = Int(0, config=True, help="set the shell (XREP) port [default: random]")
iopub_port = Int(0, config=True, help="set the iopub (PUB) port [default: random]")
stdin_port = Int(0, config=True, help="set the stdin (XREQ) port [default: random]")
+ connection_file = Unicode('', config=True,
+ help="""JSON file in which to store connection info [default: kernel-<pid>.json]
+
+ This file will contain the IP, ports, and authentication key needed to connect
+ clients to this kernel. By default, this file will be created in the security-dir
+ of the current profile, but can be specified by absolute path.
+ """)
# streams, etc.
no_stdout = Bool(False, config=True, help="redirect stdout to the null device")
@@ -138,6 +157,44 @@ def _bind_socket(self, s, port):
s.bind(iface + ':%i'%port)
return port
+ def load_connection_file(self):
+ """load ip/port/hmac config from JSON connection file"""
+ try:
+ fname = filefind(self.connection_file, ['.', self.profile_dir.security_dir])
+ except IOError:
+ self.log.debug("Connection file not found: %s", self.connection_file)
+ return
+ self.log.debug(u"Loading connection file %s", fname)
+ with open(fname) as f:
+ s = f.read()
+ cfg = json.loads(s)
+ if self.ip == LOCALHOST and 'ip' in cfg:
+ # not overridden by config or cl_args
+ self.ip = cfg['ip']
+ for channel in ('hb', 'shell', 'iopub', 'stdin'):
+ name = channel + '_port'
+ if getattr(self, name) == 0 and name in cfg:
+ # not overridden by config or cl_args
+ setattr(self, name, cfg[name])
+ if 'key' in cfg:
+ self.config.Session.key = str_to_bytes(cfg['key'])
+
+ def write_connection_file(self):
+ """write connection info to JSON file"""
+ if os.path.basename(self.connection_file) == self.connection_file:
+ cf = os.path.join(self.profile_dir.security_dir, self.connection_file)
+ else:
+ cf = self.connection_file
+ write_connection_file(cf, ip=self.ip, key=self.session.key,
+ shell_port=self.shell_port, stdin_port=self.stdin_port, hb_port=self.hb_port,
+ iopub_port=self.iopub_port)
+
+ def init_connection_file(self):
+ if not self.connection_file:
+ self.connection_file = "kernel-%s.json"%os.getpid()
+
+ self.load_connection_file()
+
def init_sockets(self):
# Create a context, a session, and the kernel sockets.
self.log.info("Starting the kernel at pid: %i", os.getpid())
@@ -161,19 +218,25 @@ def init_sockets(self):
self.hb_port = self.heartbeat.port
self.log.debug("Heartbeat REP Channel on port: %i"%self.hb_port)
- # Helper to make it easier to connect to an existing kernel, until we have
- # single-port connection negotiation fully implemented.
+ # Helper to make it easier to connect to an existing kernel.
# set log-level to critical, to make sure it is output
self.log.critical("To connect another client to this kernel, use:")
- self.log.critical("--existing --shell={0} --iopub={1} --stdin={2} --hb={3}".format(
- self.shell_port, self.iopub_port, self.stdin_port, self.hb_port))
+ if os.path.dirname(self.connection_file) == self.profile_dir.security_dir:
+ # use shortname
+ tail = os.path.basename(self.connection_file)
+ if self.profile != 'default':
+ tail += " --profile %s" % self.profile_name
+ else:
+ tail = self.connection_file
+ self.log.critical("--existing %s", tail)
self.ports = dict(shell=self.shell_port, iopub=self.iopub_port,
stdin=self.stdin_port, hb=self.hb_port)
def init_session(self):
"""create our session object"""
+ default_secure(self.config)
self.session = Session(config=self.config, username=u'kernel')
def init_blackhole(self):
@@ -209,9 +272,12 @@ def init_kernel(self):
def initialize(self, argv=None):
super(KernelApp, self).initialize(argv)
self.init_blackhole()
+ self.init_connection_file()
self.init_session()
self.init_poller()
self.init_sockets()
+ # writing connection file must be *after* init_sockets
+ self.write_connection_file()
self.init_io()
self.init_kernel()
View
98 IPython/zmq/kernelmanager.py
@@ -16,27 +16,30 @@
#-----------------------------------------------------------------------------
# Standard library imports.
-import atexit
import errno
from Queue import Queue, Empty
from subprocess import Popen
+import os
import signal
import sys
from threading import Thread
import time
-import logging
# System library imports.
import zmq
from zmq import POLLIN, POLLOUT, POLLERR
from zmq.eventloop import ioloop
+from zmq.utils import jsonapi as json
# Local imports.
from IPython.config.loader import Config
-from IPython.utils import io
from IPython.utils.localinterfaces import LOCALHOST, LOCAL_IPS
-from IPython.utils.traitlets import HasTraits, Any, Instance, Type, TCPAddress
-from session import Session, Message
+from IPython.utils.traitlets import (
+ HasTraits, Any, Instance, Type, Unicode, Int, Bool
+)
+from IPython.utils.py3compat import str_to_bytes
+from IPython.zmq.entry_point import write_connection_file
+from session import Session
#-----------------------------------------------------------------------------
# Constants and exceptions
@@ -705,10 +708,12 @@ def _context_default(self):
kernel = Instance(Popen)
# The addresses for the communication channels.
- shell_address = TCPAddress((LOCALHOST, 0))
- sub_address = TCPAddress((LOCALHOST, 0))
- stdin_address = TCPAddress((LOCALHOST, 0))
- hb_address = TCPAddress((LOCALHOST, 0))
+ connection_file = Unicode('')
+ ip = Unicode(LOCALHOST)
+ shell_port = Int(0)
+ iopub_port = Int(0)
+ stdin_port = Int(0)
+ hb_port = Int(0)
# The classes to use for the various channels.
shell_channel_class = Type(ShellSocketChannel)
@@ -722,13 +727,22 @@ def _context_default(self):
_sub_channel = Any
_stdin_channel = Any
_hb_channel = Any
+ _connection_file_written=Bool(False)
def __init__(self, **kwargs):
super(KernelManager, self).__init__(**kwargs)
if self.session is None:
self.session = Session(config=self.config)
- # Uncomment this to try closing the context.
- # atexit.register(self.context.term)
+
+ def __del__(self):
+ if self._connection_file_written:
+ # cleanup connection files on full shutdown of kernel we started
+ self._connection_file_written = False
+ try:
+ os.remove(self.connection_file)
+ except IOError:
+ pass
+
#--------------------------------------------------------------------------
# Channel management methods:
@@ -775,7 +789,35 @@ def channels_running(self):
#--------------------------------------------------------------------------
# Kernel process management methods:
#--------------------------------------------------------------------------
-
+
+ def load_connection_file(self):
+ """load connection info from JSON dict in self.connection_file"""
+ with open(self.connection_file) as f:
+ cfg = json.loads(f.read())
+
+ self.ip = cfg['ip']
+ self.shell_port = cfg['shell_port']
+ self.stdin_port = cfg['stdin_port']
+ self.iopub_port = cfg['iopub_port']
+ self.hb_port = cfg['hb_port']
+ self.session.key = str_to_bytes(cfg['key'])
+
+ def write_connection_file(self):
+ """write connection info to JSON dict in self.connection_file"""
+ if self._connection_file_written:
+ return
+ self.connection_file,cfg = write_connection_file(self.connection_file,
+ ip=self.ip, key=self.session.key,
+ stdin_port=self.stdin_port, iopub_port=self.iopub_port,
+ shell_port=self.shell_port, hb_port=self.hb_port)
+ # write_connection_file also sets default ports:
+ self.shell_port = cfg['shell_port']
+ self.stdin_port = cfg['stdin_port']
+ self.iopub_port = cfg['iopub_port']
+ self.hb_port = cfg['hb_port']
+
+ self._connection_file_written = True
+
def start_kernel(self, **kw):
"""Starts a kernel process and configures the manager to use it.
@@ -795,15 +837,15 @@ def start_kernel(self, **kw):
**kw : optional
See respective options for IPython and Python kernels.
"""
- shell, sub, stdin, hb = self.shell_address, self.sub_address, \
- self.stdin_address, self.hb_address
- if shell[0] not in LOCAL_IPS or sub[0] not in LOCAL_IPS or \
- stdin[0] not in LOCAL_IPS or hb[0] not in LOCAL_IPS:
+ if self.ip not in LOCAL_IPS:
raise RuntimeError("Can only launch a kernel on a local interface. "
"Make sure that the '*_address' attributes are "
"configured properly. "
"Currently valid addresses are: %s"%LOCAL_IPS
)
+
+ # write connection file / get default ports
+ self.write_connection_file()
self._launch_args = kw.copy()
launch_kernel = kw.pop('launcher', None)
@@ -812,13 +854,7 @@ def start_kernel(self, **kw):
from ipkernel import launch_kernel
else:
from pykernel import launch_kernel
- self.kernel, xrep, pub, req, _hb = launch_kernel(
- shell_port=shell[1], iopub_port=sub[1],
- stdin_port=stdin[1], hb_port=hb[1], **kw)
- self.shell_address = (shell[0], xrep)
- self.sub_address = (sub[0], pub)
- self.stdin_address = (stdin[0], req)
- self.hb_address = (hb[0], _hb)
+ self.kernel = launch_kernel(fname=self.connection_file, **kw)
def shutdown_kernel(self, restart=False):
""" Attempts to the stop the kernel process cleanly. If the kernel
@@ -847,6 +883,14 @@ def shutdown_kernel(self, restart=False):
if self.has_kernel:
self.kill_kernel()
+ if not restart and self._connection_file_written:
+ # cleanup connection files on full shutdown of kernel we started
+ self._connection_file_written = False
+ try:
+ os.remove(self.connection_file)
+ except IOError:
+ pass
+
def restart_kernel(self, now=False, **kw):
"""Restarts a kernel with the arguments that were used to launch it.
@@ -967,7 +1011,7 @@ def shell_channel(self):
if self._shell_channel is None:
self._shell_channel = self.shell_channel_class(self.context,
self.session,
- self.shell_address)
+ (self.ip, self.shell_port))
return self._shell_channel
@property
@@ -976,7 +1020,7 @@ def sub_channel(self):
if self._sub_channel is None:
self._sub_channel = self.sub_channel_class(self.context,
self.session,
- self.sub_address)
+ (self.ip, self.iopub_port))
return self._sub_channel
@property
@@ -985,7 +1029,7 @@ def stdin_channel(self):
if self._stdin_channel is None:
self._stdin_channel = self.stdin_channel_class(self.context,
self.session,
- self.stdin_address)
+ (self.ip, self.stdin_port))
return self._stdin_channel
@property
@@ -995,5 +1039,5 @@ def hb_channel(self):
if self._hb_channel is None:
self._hb_channel = self.hb_channel_class(self.context,
self.session,
- self.hb_address)
+ (self.ip, self.hb_port))
return self._hb_channel
View
41 IPython/zmq/session.py
@@ -43,6 +43,7 @@
from zmq.eventloop.ioloop import IOLoop
from zmq.eventloop.zmqstream import ZMQStream
+from IPython.config.application import Application, boolean_flag
from IPython.config.configurable import Configurable, LoggingConfigurable
from IPython.utils.importstring import import_item
from IPython.utils.jsonutil import extract_dates, squash_dates, date_default
@@ -71,6 +72,7 @@ def squash_unicode(obj):
#-----------------------------------------------------------------------------
# globals and defaults
#-----------------------------------------------------------------------------
+
key = 'on_unknown' if jsonapi.jsonmod.__name__ == 'jsonlib' else 'default'
json_packer = lambda obj: jsonapi.dumps(obj, **{key:date_default})
json_unpacker = lambda s: extract_dates(jsonapi.loads(s))
@@ -81,9 +83,43 @@ def squash_unicode(obj):
default_packer = json_packer
default_unpacker = json_unpacker
-
DELIM=b"<IDS|MSG>"
+
+#-----------------------------------------------------------------------------
+# Mixin tools for apps that use Sessions
+#-----------------------------------------------------------------------------
+
+session_aliases = dict(
+ ident = 'Session.session',
+ user = 'Session.username',
+ keyfile = 'Session.keyfile',
+)
+
+session_flags = {
+ 'secure' : ({'Session' : { 'key' : str_to_bytes(str(uuid.uuid4())),
+ 'keyfile' : '' }},
+ """Use HMAC digests for authentication of messages.
+ Setting this flag will generate a new UUID to use as the HMAC key.
+ """),
+ 'no-secure' : ({'Session' : { 'key' : b'', 'keyfile' : '' }},
+ """Don't authenticate messages."""),
+}
+
+def default_secure(cfg):
+ """Set the default behavior for a config environment to be secure.
+
+ If Session.key/keyfile have not been set, set Session.key to
+ a new random UUID.
+ """
+
+ if 'Session' in cfg:
+ if 'key' in cfg.Session or 'keyfile' in cfg.Session:
+ return
+ # key/keyfile not specified, generate new UUID:
+ cfg.Session.key = str_to_bytes(str(uuid.uuid4()))
+
+
#-----------------------------------------------------------------------------
# Classes
#-----------------------------------------------------------------------------
@@ -256,6 +292,7 @@ def _session_changed(self, name, old, new):
help="""Username for the Session. Default is your system username.""")
# message signature related traits:
+
key = CBytes(b'', config=True,
help="""execution key, for extra authentication.""")
def _key_changed(self, name, old, new):
@@ -272,6 +309,8 @@ def _keyfile_changed(self, name, old, new):
with open(new, 'rb') as f:
self.key = f.read().strip()
+ # serialization traits:
+
pack = Any(default_packer) # the actual packer function
def _pack_changed(self, name, old, new):
if not callable(new):
View
51 IPython/zmq/zmqshell.py
@@ -18,6 +18,8 @@
# Stdlib
import inspect
import os
+import sys
+from subprocess import Popen, PIPE
# Our own
from IPython.core.interactiveshell import (
@@ -29,11 +31,15 @@
from IPython.core.macro import Macro
from IPython.core.magic import MacroToEdit
from IPython.core.payloadpage import install_payload_page
+from IPython.lib.kernel import (
+ get_connection_file, get_connection_info, connect_qtconsole
+)
from IPython.utils import io
from IPython.utils.jsonutil import json_clean
from IPython.utils.path import get_py_filename
+from IPython.utils.process import arg_split
from IPython.utils.traitlets import Instance, Type, Dict, CBool
-from IPython.utils.warn import warn
+from IPython.utils.warn import warn, error
from IPython.zmq.displayhook import ZMQShellDisplayHook, _encode_binary
from IPython.zmq.session import extract_header
from session import Session
@@ -427,6 +433,49 @@ def magic_guiref(self, arg_s):
"""Show a basic reference about the GUI console."""
from IPython.core.usage import gui_reference
page.page(gui_reference, auto_html=True)
+
+ def magic_connect_info(self, arg_s):
+ """Print information for connecting other clients to this kernel
+
+ It will print the contents of this session's connection file, as well as
+ shortcuts for local clients.
+
+ In the simplest case, when called from the most recently launched kernel,
+ secondary clients can be connected, simply with:
+
+ $> ipython <app> --existing
+
+ """
+ try:
+ connection_file = get_connection_file()
+ info = get_connection_info(unpack=False)
+ except Exception as e:
+ error("Could not get connection info: %r" % e)
+ return
+
+ print (info + '\n')
+ print ("Paste the above JSON into a file, and connect with:\n"
+ " $> ipython <app> --existing <file>\n"
+ "or, if you are local, you can connect with just:\n"
+ " $> ipython <app> --existing %s\n"
+ "or even just:\n"
+ " $> ipython <app> --existing\n"
+ "if this is the most recent IPython session you have started."
+ % os.path.basename(connection_file)
+ )
+
+ def magic_qtconsole(self, arg_s):
+ """Open a qtconsole connected to this kernel.
+
+ Useful for connecting a qtconsole to running notebooks, for better
+ debugging.
+ """
+ try:
+ p = connect_qtconsole(argv=arg_split(arg_s, os.name=='posix'))
+ except Exception as e:
+ error("Could not start qtconsole: %r" % e)
+ return
+
def set_next_input(self, text):
"""Send the specified text to the frontend to be presented at the next

0 comments on commit f93a8ca

Please sign in to comment.
Something went wrong with that request. Please try again.