Skip to content
Browse files

use connection files instead of ports to connect to kernels

this addresses security issues (#688)
and ease-of-multi-frontend
  • Loading branch information...
1 parent fc5db52 commit ebac3b18b5fc10ca212464b51a0edfcd2945d3ec @minrk committed Oct 9, 2011
Showing with 264 additions and 54 deletions.
  1. +85 −6 IPython/frontend/qt/console/qtconsoleapp.py
  2. +73 −33 IPython/zmq/entry_point.py
  3. +61 −4 IPython/zmq/kernelapp.py
  4. +45 −11 IPython/zmq/kernelmanager.py
View
91 IPython/frontend/qt/console/qtconsoleapp.py
@@ -17,6 +17,7 @@
#-----------------------------------------------------------------------------
# stdlib imports
+import glob
import os
import signal
import sys
@@ -25,6 +26,7 @@
# System library imports
from IPython.external.qt import QtGui
from pygments.styles import get_all_styles
+from zmq.utils import jsonapi as json
# external imports
from IPython.external.ssh import tunnel
@@ -39,6 +41,7 @@
from IPython.frontend.qt.console import styles
from IPython.frontend.qt.kernelmanager import QtKernelManager
from IPython.parallel.util import select_random_ports
+from IPython.utils.path import filefind
from IPython.utils.traitlets import (
Dict, List, Unicode, Int, CaselessStrEnum, CBool, Any
)
@@ -182,8 +185,8 @@ def closeEvent(self, event):
flags = dict(ipkernel_flags)
qt_flags = {
- 'existing' : ({'IPythonQtConsoleApp' : {'existing' : True}},
- "Connect to an existing kernel."),
+ 'existing' : ({'IPythonQtConsoleApp' : {'existing' : 'kernel*.json'}},
+ "Connect to an existing kernel. If no argument specified, guess most recent"),
'pure' : ({'IPythonQtConsoleApp' : {'pure' : True}},
"Use a pure Python kernel instead of an IPython kernel."),
'plain' : ({'ConsoleWidget' : {'kind' : 'plain'}},
@@ -213,6 +216,8 @@ def closeEvent(self, event):
iopub = 'IPythonQtConsoleApp.iopub_port',
stdin = 'IPythonQtConsoleApp.stdin_port',
ip = 'IPythonQtConsoleApp.ip',
+ existing = 'IPythonQtConsoleApp.existing',
+ f = 'IPythonQtConsoleApp.connection_file',
style = 'IPythonWidget.syntax_style',
stylesheet = 'IPythonQtConsoleApp.stylesheet',
@@ -280,9 +285,18 @@ class IPythonQtConsoleApp(BaseIPythonApplication):
help="set the iopub (PUB) port [default: random]")
stdin_port = Int(0, config=True,
help="set the stdin (XREQ) port [default: random]")
+ connection_file = Unicode('', config=True,
+ help="""JSON file in which to store connection info [default: kernel-<pid>.json]
- existing = CBool(False, config=True,
- help="Whether to connect to an already running Kernel.")
+ This file will contain the IP, ports, and authentication key needed to connect
+ clients to this kernel. By default, this file will be created in the security-dir
+ of the current profile, but can be specified by absolute path.
+ """)
+ def _connection_file_default(self):
+ return 'kernel-%i.json' % os.getpid()
+
+ existing = Unicode('', config=True,
+ help="""Connect to an already running kernel""")
stylesheet = Unicode('', config=True,
help="path to a custom CSS stylesheet")
@@ -340,6 +354,58 @@ def parse_command_line(self, argv=None):
# alias passed with arg via space
swallow_next = True
+ def init_connection_file(self):
+ sec = self.profile_dir.security_dir
+ if self.existing:
+ try:
+ # first, try explicit name
+ cf = filefind(self.existing, ['.', sec])
+ except IOError:
+ # not found by full name
+ if '*' in self.existing:
+ # given as a glob already
+ pat = self.existing
+ else:
+ # accept any substring match
+ pat = '*%s*'
+ matches = glob.glob( os.path.join(sec, pat) )
+ if not matches:
+ self.log.critical("Could not find existing kernel connection file %s", self.existing)
+ self.exit(1)
+ else:
+ # get most recent match:
+ cf = sorted(matches, key=lambda f: os.stat(f).st_atime)[-1]
+ self.log.info("Connecting to existing kernel: %s" % cf)
+ self.connection_file = cf
+ # should load_connection_file only be used for existing?
+ # as it is now, this allows reusing ports if an existing
+ # file is requested
+ self.load_connection_file()
+
+ def load_connection_file(self):
+ """load ip/port/hmac config from JSON connection file"""
+ # this is identical to KernelApp.load_connection_file
+ # perhaps it can be centralized somewhere?
+ try:
+ fname = filefind(self.connection_file, ['.', self.profile_dir.security_dir])
+ except IOError:
+ self.log.debug("Connection File not found: %s", self.connection_file)
+ return
+ self.log.debug(u"Loading connection file %s", fname)
+ with open(fname) as f:
+ s = f.read()
+ cfg = json.loads(s)
+ if self.ip == LOCALHOST and 'ip' in cfg:
+ # not overridden by config or cl_args
+ self.ip = cfg['ip']
+ for channel in ('hb', 'shell', 'iopub', 'stdin'):
+ name = channel + '_port'
+ if getattr(self, name) == 0 and name in cfg:
+ # not overridden by config or cl_args
+ setattr(self, name, cfg[name])
+ if 'key' in cfg:
+ self.config.Session.key = cfg['key']
+
def init_ssh(self):
"""set up ssh tunnels, if needed."""
if not self.sshserver and not self.sshkey:
@@ -374,18 +440,30 @@ def init_ssh(self):
def init_kernel_manager(self):
# Don't let Qt or ZMQ swallow KeyboardInterupts.
signal.signal(signal.SIGINT, signal.SIG_DFL)
+ sec = self.profile_dir.security_dir
+ try:
+ cf = filefind(self.connection_file, ['.', sec])
+ except IOError:
+ # file might not exist, use
+ if self.connection_file == os.path.basename(self.connection_file):
+ # just shortname, put it in security dir
+ cf = os.path.join(sec, self.connection_file)
+ else:
+ cf = self.connection_file
# Create a KernelManager and start a kernel.
- self.kernel_manager = QtKernelManager(ip=self.ip,
+ self.kernel_manager = QtKernelManager(
+ ip=self.ip,
shell_port=self.shell_port,
sub_port=self.iopub_port,
stdin_port=self.stdin_port,
hb_port=self.hb_port,
+ connection_file=cf,
config=self.config,
)
# start the kernel
if not self.existing:
- kwargs = dict(ip=self.ip, ipython=not self.pure)
+ kwargs = dict(ipython=not self.pure)
kwargs['extra_arguments'] = self.kernel_argv
self.kernel_manager.start_kernel(**kwargs)
self.kernel_manager.start_channels()
@@ -469,6 +547,7 @@ def init_colors(self):
def initialize(self, argv=None):
super(IPythonQtConsoleApp, self).initialize(argv)
self.init_ssh()
+ self.init_connection_file()
self.init_kernel_manager()
self.init_qt_elements()
self.init_colors()
View
106 IPython/zmq/entry_point.py
@@ -8,20 +8,26 @@
import socket
from subprocess import Popen, PIPE
import sys
+import tempfile
-# Local imports.
-from parentpoller import ParentPollerWindows
+# System library imports
+from zmq.utils import jsonapi as json
+# IPython imports
+from IPython.utils.localinterfaces import LOCALHOST
-def base_launch_kernel(code, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0,
- ip=None, stdin=None, stdout=None, stderr=None,
- executable=None, independent=False, extra_arguments=[]):
- """ Launches a localhost kernel, binding to the specified ports.
+# Local imports.
+from parentpoller import ParentPollerWindows
+def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0,
+ ip=LOCALHOST, key=''):
+ """Generates a JSON config file, including the selection of random ports.
+
Parameters
----------
- code : str,
- A string of Python code that imports and executes a kernel entry point.
+
+ fname : unicode
+ The path to the file to write
shell_port : int, optional
The port to use for XREP channel.
@@ -38,27 +44,14 @@ def base_launch_kernel(code, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0
ip : str, optional
The ip address the kernel will bind to.
- stdin, stdout, stderr : optional (default None)
- Standards streams, as defined in subprocess.Popen.
+ key : str, optional
+ The Session key used for HMAC authentication.
- executable : str, optional (default sys.executable)
- The Python executable to use for the kernel process.
-
- independent : bool, optional (default False)
- If set, the kernel process is guaranteed to survive if this process
- dies. If not set, an effort is made to ensure that the kernel is killed
- when this process dies. Note that in this case it is still good practice
- to kill kernels manually before exiting.
-
- extra_arguments = list, optional
- A list of extra arguments to pass when executing the launch code.
-
- Returns
- -------
- A tuple of form:
- (kernel_process, shell_port, iopub_port, stdin_port, hb_port)
- where kernel_process is a Popen object and the ports are integers.
"""
+ # default to temporary connector file
+ if not fname:
+ fname = tempfile.mktemp('.json')
+
# Find open ports as necessary.
ports = []
ports_needed = int(shell_port <= 0) + int(iopub_port <= 0) + \
@@ -79,15 +72,62 @@ def base_launch_kernel(code, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0
stdin_port = ports.pop(0)
if hb_port <= 0:
hb_port = ports.pop(0)
+
+ cfg = dict( shell_port=shell_port,
+ iopub_port=iopub_port,
+ stdin_port=stdin_port,
+ hb_port=hb_port,
+ )
+ cfg['ip'] = ip
+ cfg['key'] = key
+
+ with open(fname, 'wb') as f:
+ f.write(json.dumps(cfg, indent=2))
+
+ return fname, cfg
+
+
+def base_launch_kernel(code, fname, stdin=None, stdout=None, stderr=None,
+ executable=None, independent=False, extra_arguments=[]):
+ """ Launches a localhost kernel, binding to the specified ports.
+
+ Parameters
+ ----------
+ code : str,
+ A string of Python code that imports and executes a kernel entry point.
+ stdin, stdout, stderr : optional (default None)
+ Standards streams, as defined in subprocess.Popen.
+
+ fname : unicode, optional
+ The JSON connector file, containing ip/port/hmac key information.
+
+ key : str, optional
+ The Session key used for HMAC authentication.
+
+ executable : str, optional (default sys.executable)
+ The Python executable to use for the kernel process.
+
+ independent : bool, optional (default False)
+ If set, the kernel process is guaranteed to survive if this process
+ dies. If not set, an effort is made to ensure that the kernel is killed
+ when this process dies. Note that in this case it is still good practice
+ to kill kernels manually before exiting.
+
+ extra_arguments = list, optional
+ A list of extra arguments to pass when executing the launch code.
+
+ Returns
+ -------
+ A tuple of form:
+ (kernel_process, shell_port, iopub_port, stdin_port, hb_port)
+ where kernel_process is a Popen object and the ports are integers.
+ """
+
# Build the kernel launch command.
if executable is None:
executable = sys.executable
- arguments = [ executable, '-c', code, '--shell=%i'%shell_port,
- '--iopub=%i'%iopub_port, '--stdin=%i'%stdin_port,
- '--hb=%i'%hb_port ]
- if ip is not None:
- arguments.append('--ip=%s'%ip)
+ arguments = [ executable, '-c', code, '-f', fname ]
arguments.extend(extra_arguments)
# Popen will fail (sometimes with a deadlock) if stdin, stdout, and stderr
@@ -164,4 +204,4 @@ def base_launch_kernel(code, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0
if stderr is None:
proc.stderr.close()
- return proc, shell_port, iopub_port, stdin_port, hb_port
+ return proc
View
65 IPython/zmq/kernelapp.py
@@ -21,6 +21,7 @@
# System library imports.
import zmq
+from zmq.utils import jsonapi as json
# IPython imports.
from IPython.core.ultratb import FormattedTB
@@ -29,10 +30,12 @@
)
from IPython.utils import io
from IPython.utils.localinterfaces import LOCALHOST
+from IPython.utils.path import filefind
from IPython.utils.traitlets import (Any, Instance, Dict, Unicode, Int, Bool,
DottedObjectName)
from IPython.utils.importstring import import_item
# local imports
+from IPython.zmq.entry_point import write_connection_file
from IPython.zmq.heartbeat import Heartbeat
from IPython.zmq.parentpoller import ParentPollerUnix, ParentPollerWindows
from IPython.zmq.session import Session
@@ -49,6 +52,7 @@
'shell' : 'KernelApp.shell_port',
'iopub' : 'KernelApp.iopub_port',
'stdin' : 'KernelApp.stdin_port',
+ 'f' : 'KernelApp.connection_file',
'parent': 'KernelApp.parent',
})
if sys.platform.startswith('win'):
@@ -99,6 +103,13 @@ def _parent_appname_changed(self, name, old, new):
shell_port = Int(0, config=True, help="set the shell (XREP) port [default: random]")
iopub_port = Int(0, config=True, help="set the iopub (PUB) port [default: random]")
stdin_port = Int(0, config=True, help="set the stdin (XREQ) port [default: random]")
+ connection_file = Unicode('', config=True,
+ help="""JSON file in which to store connection info [default: kernel-<pid>.json]
+
+ This file will contain the IP, ports, and authentication key needed to connect
+ clients to this kernel. By default, this file will be created in the security-dir
+ of the current profile, but can be specified by absolute path.
+ """)
# streams, etc.
no_stdout = Bool(False, config=True, help="redirect stdout to the null device")
@@ -138,6 +149,44 @@ def _bind_socket(self, s, port):
s.bind(iface + ':%i'%port)
return port
+ def load_connection_file(self):
+ """load ip/port/hmac config from JSON connection file"""
+ try:
+ fname = filefind(self.connection_file, ['.', self.profile_dir.security_dir])
+ except IOError:
+ self.log.debug("Connection file not found: %s", self.connection_file)
+ return
+ self.log.debug(u"Loading connection file %s", fname)
+ with open(fname) as f:
+ s = f.read()
+ cfg = json.loads(s)
+ if self.ip == LOCALHOST and 'ip' in cfg:
+ # not overridden by config or cl_args
+ self.ip = cfg['ip']
+ for channel in ('hb', 'shell', 'iopub', 'stdin'):
+ name = channel + '_port'
+ if getattr(self, name) == 0 and name in cfg:
+ # not overridden by config or cl_args
+ setattr(self, name, cfg[name])
+ if 'key' in cfg:
+ self.config.Session.key = cfg['key']
+
+ def write_connection_file(self):
+ """write connection info to JSON file"""
+ if os.path.basename(self.connection_file) == self.connection_file:
+ cf = os.path.join(self.profile_dir.security_dir, self.connection_file)
+ else:
+ cf = self.connection_file
+ write_connection_file(cf, ip=self.ip, key=self.session.key,
+ shell_port=self.shell_port, stdin_port=self.stdin_port, hb_port=self.hb_port,
+ iopub_port=self.iopub_port)
+
+ def init_connection_file(self):
+ if not self.connection_file:
+ self.connection_file = "kernel-%s.json"%os.getpid()
+
+ self.load_connection_file()
+
def init_sockets(self):
# Create a context, a session, and the kernel sockets.
self.log.info("Starting the kernel at pid: %i", os.getpid())
@@ -161,12 +210,17 @@ def init_sockets(self):
self.hb_port = self.heartbeat.port
self.log.debug("Heartbeat REP Channel on port: %i"%self.hb_port)
- # Helper to make it easier to connect to an existing kernel, until we have
- # single-port connection negotiation fully implemented.
+ # Helper to make it easier to connect to an existing kernel.
# set log-level to critical, to make sure it is output
self.log.critical("To connect another client to this kernel, use:")
- self.log.critical("--existing --shell={0} --iopub={1} --stdin={2} --hb={3}".format(
- self.shell_port, self.iopub_port, self.stdin_port, self.hb_port))
+ if os.path.dirname(self.connection_file) == self.profile_dir.security_dir:
+ # use shortname
+ tail = os.path.basename(self.connection_file)
+ if self.profile != 'default':
+ tail += " --profile %s" % self.profile_name
+ else:
+ tail = self.connection_file
+ self.log.critical("--existing %s", tail)
self.ports = dict(shell=self.shell_port, iopub=self.iopub_port,
@@ -209,9 +263,12 @@ def init_kernel(self):
def initialize(self, argv=None):
super(KernelApp, self).initialize(argv)
self.init_blackhole()
+ self.init_connection_file()
self.init_session()
self.init_poller()
self.init_sockets()
+ # writing connection file must be *after* init_sockets
+ self.write_connection_file()
self.init_io()
self.init_kernel()
View
56 IPython/zmq/kernelmanager.py
@@ -19,6 +19,7 @@
import errno
from Queue import Queue, Empty
from subprocess import Popen
+import os
import signal
import sys
from threading import Thread
@@ -32,7 +33,10 @@
# Local imports.
from IPython.config.loader import Config
from IPython.utils.localinterfaces import LOCALHOST, LOCAL_IPS
-from IPython.utils.traitlets import HasTraits, Any, Instance, Type, Unicode, Int
+from IPython.utils.traitlets import (
+ HasTraits, Any, Instance, Type, Unicode, Int, Bool
+)
+from IPython.zmq.entry_point import write_connection_file
from session import Session
#-----------------------------------------------------------------------------
@@ -702,6 +706,7 @@ def _context_default(self):
kernel = Instance(Popen)
# The addresses for the communication channels.
+ connection_file = Unicode('')
ip = Unicode(LOCALHOST)
shell_port = Int(0)
sub_port = Int(0)
@@ -720,13 +725,22 @@ def _context_default(self):
_sub_channel = Any
_stdin_channel = Any
_hb_channel = Any
+ _connection_file_written=Bool(False)
def __init__(self, **kwargs):
super(KernelManager, self).__init__(**kwargs)
if self.session is None:
self.session = Session(config=self.config)
- # Uncomment this to try closing the context.
- # atexit.register(self.context.term)
+
+ def __del__(self):
+ if self._connection_file_written:
+ # cleanup connection files on full shutdown of kernel we started
+ self._connection_file_written = False
+ try:
+ os.remove(self.connection_file)
+ except IOError:
+ pass
+
#--------------------------------------------------------------------------
# Channel management methods:
@@ -773,7 +787,22 @@ def channels_running(self):
#--------------------------------------------------------------------------
# Kernel process management methods:
#--------------------------------------------------------------------------
-
+
+ def write_connection_file(self):
+ if self._connection_file_written:
+ return
+ self.connection_file,cfg = write_connection_file(self.connection_file,
+ ip=self.ip, key=self.session.key,
+ stdin_port=self.stdin_port, iopub_port=self.sub_port,
+ shell_port=self.shell_port, hb_port=self.hb_port)
+ # write_connection_file also sets default ports:
+ self.shell_port = cfg['shell_port']
+ self.stdin_port = cfg['stdin_port']
+ self.sub_port = cfg['iopub_port']
+ self.hb_port = cfg['hb_port']
+
+ self._connection_file_written = True
+
def start_kernel(self, **kw):
"""Starts a kernel process and configures the manager to use it.
@@ -799,6 +828,9 @@ def start_kernel(self, **kw):
"configured properly. "
"Currently valid addresses are: %s"%LOCAL_IPS
)
+
+ # write connection file / get default ports
+ self.write_connection_file()
self._launch_args = kw.copy()
launch_kernel = kw.pop('launcher', None)
@@ -807,13 +839,7 @@ def start_kernel(self, **kw):
from ipkernel import launch_kernel
else:
from pykernel import launch_kernel
- self.kernel, shell, sub, stdin, hb = launch_kernel(
- shell_port=self.shell_port, iopub_port=self.sub_port,
- stdin_port=self.stdin_port, hb_port=self.hb_port, **kw)
- self.shell_port = shell
- self.sub_port = sub
- self.stdin_port = stdin
- self.hb_port = hb
+ self.kernel = launch_kernel(fname=self.connection_file, **kw)
def shutdown_kernel(self, restart=False):
""" Attempts to the stop the kernel process cleanly. If the kernel
@@ -842,6 +868,14 @@ def shutdown_kernel(self, restart=False):
if self.has_kernel:
self.kill_kernel()
+ if not restart and self._connection_file_written:
+ # cleanup connection files on full shutdown of kernel we started
+ self._connection_file_written = False
+ try:
+ os.remove(self.connection_file)
+ except IOError:
+ pass
+
def restart_kernel(self, now=False, **kw):
"""Restarts a kernel with the arguments that were used to launch it.

0 comments on commit ebac3b1

Please sign in to comment.
Something went wrong with that request. Please try again.