Skip to content
Browse files

Merge remote-tracking branch 'upstream/master'

  • Loading branch information...
2 parents c0c597c + 71ae352 commit ad500830627eecd1fa6152b213045a178bdab8de Felix Werner committed Oct 17, 2011
View
4 IPython/core/application.py
@@ -27,6 +27,7 @@
# Imports
#-----------------------------------------------------------------------------
+import atexit
import glob
import logging
import os
@@ -156,6 +157,9 @@ def init_crash_handler(self):
"""Create a crash handler, typically setting sys.excepthook to it."""
self.crash_handler = self.crash_handler_class(self)
sys.excepthook = self.crash_handler
+ def unset_crashhandler():
+ sys.excepthook = sys.__excepthook__
+ atexit.register(unset_crashhandler)
def _ipython_dir_changed(self, name, old, new):
if old in sys.path:
View
8 IPython/core/debugger.py
@@ -38,7 +38,7 @@
has_pydb = False
prompt = 'ipdb> '
#We have to check this directly from sys.argv, config struct not yet available
-if '-pydb' in sys.argv:
+if '--pydb' in sys.argv:
try:
import pydb
if hasattr(pydb.pydb, "runl") and pydb.version>'1.17':
@@ -64,7 +64,7 @@ def BdbQuit_excepthook(et,ev,tb):
else:
BdbQuit_excepthook.excepthook_ori(et,ev,tb)
-def BdbQuit_IPython_excepthook(self,et,ev,tb):
+def BdbQuit_IPython_excepthook(self,et,ev,tb,tb_offset=None):
print 'Exiting Debugger.'
@@ -104,8 +104,8 @@ def __init__(self,colors=None):
"""
try:
- ip = ipapi.get()
- except:
+ ip = get_ipython()
+ except NameError:
# Outside of ipython, we set our own exception hook manually
BdbQuit_excepthook.excepthook_ori = sys.excepthook
sys.excepthook = BdbQuit_excepthook
View
104 IPython/core/interactiveshell.py
@@ -1447,29 +1447,37 @@ def set_custom_exc(self, exc_tuple, handler):
Set a custom exception handler, which will be called if any of the
exceptions in exc_tuple occur in the mainloop (specifically, in the
- run_code() method.
+ run_code() method).
- Inputs:
+ Parameters
+ ----------
+
+ exc_tuple : tuple of exception classes
+ A *tuple* of exception classes, for which to call the defined
+ handler. It is very important that you use a tuple, and NOT A
+ LIST here, because of the way Python's except statement works. If
+ you only want to trap a single exception, use a singleton tuple::
- - exc_tuple: a *tuple* of valid exceptions to call the defined
- handler for. It is very important that you use a tuple, and NOT A
- LIST here, because of the way Python's except statement works. If
- you only want to trap a single exception, use a singleton tuple:
+ exc_tuple == (MyCustomException,)
- exc_tuple == (MyCustomException,)
+ handler : callable
+ handler must have the following signature::
- - handler: this must be defined as a function with the following
- basic interface::
+ def my_handler(self, etype, value, tb, tb_offset=None):
+ ...
+ return structured_traceback
- def my_handler(self, etype, value, tb, tb_offset=None)
- ...
- # The return value must be
- return structured_traceback
+ Your handler must return a structured traceback (a list of strings),
+ or None.
- This will be made into an instance method (via types.MethodType)
- of IPython itself, and it will be called if any of the exceptions
- listed in the exc_tuple are caught. If the handler is None, an
- internal basic one is used, which just prints basic info.
+ This will be made into an instance method (via types.MethodType)
+ of IPython itself, and it will be called if any of the exceptions
+ listed in the exc_tuple are caught. If the handler is None, an
+ internal basic one is used, which just prints basic info.
+
+ To protect IPython from crashes, if your handler ever raises an
+ exception or returns an invalid result, it will be immediately
+ disabled.
WARNING: by putting in your own exception handler into IPython's main
execution loop, you run a very good chance of nasty crashes. This
@@ -1478,16 +1486,62 @@ def my_handler(self, etype, value, tb, tb_offset=None)
assert type(exc_tuple)==type(()) , \
"The custom exceptions must be given AS A TUPLE."
- def dummy_handler(self,etype,value,tb):
+ def dummy_handler(self,etype,value,tb,tb_offset=None):
print '*** Simple custom exception handler ***'
print 'Exception type :',etype
print 'Exception value:',value
print 'Traceback :',tb
#print 'Source code :','\n'.join(self.buffer)
-
- if handler is None: handler = dummy_handler
-
- self.CustomTB = types.MethodType(handler,self)
+
+ def validate_stb(stb):
+ """validate structured traceback return type
+
+ return type of CustomTB *should* be a list of strings, but allow
+ single strings or None, which are harmless.
+
+ This function will *always* return a list of strings,
+ and will raise a TypeError if stb is inappropriate.
+ """
+ msg = "CustomTB must return list of strings, not %r" % stb
+ if stb is None:
+ return []
+ elif isinstance(stb, basestring):
+ return [stb]
+ elif not isinstance(stb, list):
+ raise TypeError(msg)
+ # it's a list
+ for line in stb:
+ # check every element
+ if not isinstance(line, basestring):
+ raise TypeError(msg)
+ return stb
+
+ if handler is None:
+ wrapped = dummy_handler
+ else:
+ def wrapped(self,etype,value,tb,tb_offset=None):
+ """wrap CustomTB handler, to protect IPython from user code
+
+ This makes it harder (but not impossible) for custom exception
+ handlers to crash IPython.
+ """
+ try:
+ stb = handler(self,etype,value,tb,tb_offset=tb_offset)
+ return validate_stb(stb)
+ except:
+ # clear custom handler immediately
+ self.set_custom_exc((), None)
+ print >> io.stderr, "Custom TB Handler failed, unregistering"
+ # show the exception in handler first
+ stb = self.InteractiveTB.structured_traceback(*sys.exc_info())
+ print >> io.stdout, self.InteractiveTB.stb2text(stb)
+ print >> io.stdout, "The original exception:"
+ stb = self.InteractiveTB.structured_traceback(
+ (etype,value,tb), tb_offset=tb_offset
+ )
+ return stb
+
+ self.CustomTB = types.MethodType(wrapped,self)
self.custom_exceptions = exc_tuple
def excepthook(self, etype, value, tb):
@@ -1556,11 +1610,7 @@ def showtraceback(self,exc_tuple = None,filename=None,tb_offset=None,
sys.last_value = value
sys.last_traceback = tb
if etype in self.custom_exceptions:
- # FIXME: Old custom traceback objects may just return a
- # string, in that case we just put it into a list
stb = self.CustomTB(etype, value, tb, tb_offset)
- if isinstance(ctb, basestring):
- stb = [stb]
else:
if exception_only:
stb = ['An exception has occurred, use %tb to see '
@@ -1571,9 +1621,11 @@ def showtraceback(self,exc_tuple = None,filename=None,tb_offset=None,
stb = self.InteractiveTB.structured_traceback(etype,
value, tb, tb_offset=tb_offset)
+ self._showtraceback(etype, value, stb)
if self.call_pdb:
# drop into debugger
self.debugger(force=True)
+ return
# Actually show the traceback
self._showtraceback(etype, value, stb)
View
8 IPython/core/shellapp.py
@@ -50,6 +50,14 @@
"Enable auto calling the pdb debugger after every exception.",
"Disable auto calling the pdb debugger after every exception."
)
+# pydb flag doesn't do any config, as core.debugger switches on import,
+# which is before parsing. This just allows the flag to be passed.
+shell_flags.update(dict(
+ pydb = ({},
+ """"Use the third party 'pydb' package as debugger, instead of pdb.
+ Requires that pydb is installed."""
+ )
+))
addflag('pprint', 'PlainTextFormatter.pprint',
"Enable auto pretty printing of results.",
"Disable auto auto pretty printing of results."
View
34 IPython/core/tests/test_interactiveshell.py
@@ -146,3 +146,37 @@ def test_future_unicode(self):
finally:
# Reset compiler flags so we don't mess up other tests.
ip.compile.reset_compiler_flags()
+
+ def test_bad_custom_tb(self):
+ """Check that InteractiveShell is protected from bad custom exception handlers"""
+ ip = get_ipython()
+ from IPython.utils import io
+ save_stderr = io.stderr
+ try:
+ # capture stderr
+ io.stderr = StringIO()
+ ip.set_custom_exc((IOError,), lambda etype,value,tb: 1/0)
+ self.assertEquals(ip.custom_exceptions, (IOError,))
+ ip.run_cell(u'raise IOError("foo")')
+ self.assertEquals(ip.custom_exceptions, ())
+ self.assertTrue("Custom TB Handler failed" in io.stderr.getvalue())
+ finally:
+ io.stderr = save_stderr
+
+ def test_bad_custom_tb_return(self):
+ """Check that InteractiveShell is protected from bad return types in custom exception handlers"""
+ ip = get_ipython()
+ from IPython.utils import io
+ save_stderr = io.stderr
+ try:
+ # capture stderr
+ io.stderr = StringIO()
+ ip.set_custom_exc((NameError,),lambda etype,value,tb, tb_offset=None: 1)
+ self.assertEquals(ip.custom_exceptions, (NameError,))
+ ip.run_cell(u'a=abracadabra')
+ self.assertEquals(ip.custom_exceptions, ())
+ self.assertTrue("Custom TB Handler failed" in io.stderr.getvalue())
+ finally:
+ io.stderr = save_stderr
+
+
View
2 IPython/frontend/html/notebook/notebookapp.py
@@ -223,7 +223,7 @@ def parse_command_line(self, argv=None):
swallow_next = False
continue
if a.startswith('-'):
- split = a.lstrip('-').split('=')[0]
+ split = a.lstrip('-').split('=')
alias = split[0]
if alias in notebook_aliases:
self.kernel_argv.remove(a)
View
2 IPython/frontend/qt/console/qtconsoleapp.py
@@ -342,7 +342,7 @@ def parse_command_line(self, argv=None):
swallow_next = False
continue
if a.startswith('-'):
- split = a.lstrip('-').split('=')[0]
+ split = a.lstrip('-').split('=')
alias = split[0]
if alias in qt_aliases:
self.kernel_argv.remove(a)
View
234 IPython/lib/backgroundjobs.py
@@ -17,6 +17,9 @@
(although ultimately no code from this text was used, as IPython's system is a
separate implementation).
+
+An example notebook is provided in our documentation illustrating interactive
+use of the system.
"""
#*****************************************************************************
@@ -33,7 +36,8 @@
from IPython.core.ultratb import AutoFormattedTB
from IPython.utils.warn import warn, error
-class BackgroundJobManager:
+
+class BackgroundJobManager(object):
"""Class to manage a pool of backgrounded threaded jobs.
Below, we assume that 'jobs' is a BackgroundJobManager instance.
@@ -52,7 +56,7 @@ class BackgroundJobManager:
jobs.remove(N) -> remove (finished) job N
- jobs.flush_finished() -> remove all finished jobs
+ jobs.flush() -> remove all finished jobs
As a convenience feature, BackgroundJobManager instances provide the
utility result and traceback methods which retrieve the corresponding
@@ -63,17 +67,16 @@ class BackgroundJobManager:
While this appears minor, it allows you to use tab completion
interactively on the job manager instance.
-
- In interactive mode, IPython provides the magic fuction %bg for quick
- creation of backgrounded expression-based jobs. Type bg? for details."""
+ """
def __init__(self):
- # Lists for job management
- self.jobs_run = []
- self.jobs_comp = []
- self.jobs_dead = []
+ # Lists for job management, accessed via a property to ensure they're
+ # up to date.x
+ self._running = []
+ self._completed = []
+ self._dead = []
# A dict of all jobs, so users can easily access any of them
- self.jobs_all = {}
+ self.all = {}
# For reporting
self._comp_report = []
self._dead_report = []
@@ -83,7 +86,22 @@ def __init__(self):
self._s_completed = BackgroundJobBase.stat_completed_c
self._s_dead = BackgroundJobBase.stat_dead_c
- def new(self,func_or_exp,*args,**kwargs):
+ @property
+ def running(self):
+ self._update_status()
+ return self._running
+
+ @property
+ def dead(self):
+ self._update_status()
+ return self._dead
+
+ @property
+ def completed(self):
+ self._update_status()
+ return self._completed
+
+ def new(self, func_or_exp, *args, **kwargs):
"""Add a new background job and start it in a separate thread.
There are two types of jobs which can be created:
@@ -106,14 +124,14 @@ def new(self,func_or_exp,*args,**kwargs):
2. Jobs given a function object, optionally passing additional
positional arguments:
- job_manager.new(myfunc,x,y)
+ job_manager.new(myfunc, x, y)
The function is called with the given arguments.
If you need to pass keyword arguments to your function, you must
supply them as a dict named kw:
- job_manager.new(myfunc,x,y,kw=dict(z=1))
+ job_manager.new(myfunc, x, y, kw=dict(z=1))
The reason for this assymmetry is that the new() method needs to
maintain access to its own keywords, and this prevents name collisions
@@ -149,7 +167,7 @@ def new(self,func_or_exp,*args,**kwargs):
if callable(func_or_exp):
kw = kwargs.get('kw',{})
job = BackgroundJobFunc(func_or_exp,*args,**kw)
- elif isinstance(func_or_exp,basestring):
+ elif isinstance(func_or_exp, basestring):
if not args:
frame = sys._getframe(1)
glob, loc = frame.f_globals, frame.f_locals
@@ -158,60 +176,63 @@ def new(self,func_or_exp,*args,**kwargs):
elif len(args)==2:
glob,loc = args
else:
- raise ValueError,\
- 'Expression jobs take at most 2 args (globals,locals)'
- job = BackgroundJobExpr(func_or_exp,glob,loc)
+ raise ValueError(
+ 'Expression jobs take at most 2 args (globals,locals)')
+ job = BackgroundJobExpr(func_or_exp, glob, loc)
else:
- raise
- jkeys = self.jobs_all.keys()
- if jkeys:
- job.num = max(jkeys)+1
- else:
- job.num = 0
- self.jobs_run.append(job)
- self.jobs_all[job.num] = job
+ raise TypeError('invalid args for new job')
+
+ job.num = len(self.all)+1 if self.all else 0
+ self.running.append(job)
+ self.all[job.num] = job
print 'Starting job # %s in a separate thread.' % job.num
job.start()
return job
- def __getitem__(self,key):
- return self.jobs_all[key]
+ def __getitem__(self, job_key):
+ num = job_key if isinstance(job_key, int) else job_key.num
+ return self.all[num]
def __call__(self):
"""An alias to self.status(),
This allows you to simply call a job manager instance much like the
- Unix jobs shell command."""
+ Unix `jobs` shell command."""
return self.status()
def _update_status(self):
"""Update the status of the job lists.
This method moves finished jobs to one of two lists:
- - self.jobs_comp: jobs which completed successfully
- - self.jobs_dead: jobs which finished but died.
+ - self.completed: jobs which completed successfully
+ - self.dead: jobs which finished but died.
It also copies those jobs to corresponding _report lists. These lists
are used to report jobs completed/dead since the last update, and are
then cleared by the reporting function after each call."""
-
- run,comp,dead = self._s_running,self._s_completed,self._s_dead
- jobs_run = self.jobs_run
- for num in range(len(jobs_run)):
- job = jobs_run[num]
+
+ # Status codes
+ srun, scomp, sdead = self._s_running, self._s_completed, self._s_dead
+ # State lists, use the actual lists b/c the public names are properties
+ # that call this very function on access
+ running, completed, dead = self._running, self._completed, self._dead
+
+ # Now, update all state lists
+ for num, job in enumerate(running):
stat = job.stat_code
- if stat == run:
+ if stat == srun:
continue
- elif stat == comp:
- self.jobs_comp.append(job)
+ elif stat == scomp:
+ completed.append(job)
self._comp_report.append(job)
- jobs_run[num] = False
- elif stat == dead:
- self.jobs_dead.append(job)
+ running[num] = False
+ elif stat == sdead:
+ dead.append(job)
self._dead_report.append(job)
- jobs_run[num] = False
- self.jobs_run = filter(None,self.jobs_run)
+ running[num] = False
+ # Remove dead/completed jobs from running list
+ running[:] = filter(None, running)
def _group_report(self,group,name):
"""Report summary for a given job group.
@@ -246,7 +267,7 @@ def _status_new(self):
which have finished since the last time it was called."""
self._update_status()
- new_comp = self._group_report(self._comp_report,'Completed')
+ new_comp = self._group_report(self._comp_report, 'Completed')
new_dead = self._group_report(self._dead_report,
'Dead, call jobs.traceback() for details')
self._comp_report[:] = []
@@ -257,9 +278,9 @@ def status(self,verbose=0):
"""Print a status of all jobs currently being managed."""
self._update_status()
- self._group_report(self.jobs_run,'Running')
- self._group_report(self.jobs_comp,'Completed')
- self._group_report(self.jobs_dead,'Dead')
+ self._group_report(self.running,'Running')
+ self._group_report(self.completed,'Completed')
+ self._group_report(self.dead,'Dead')
# Also flush the report queues
self._comp_report[:] = []
self._dead_report[:] = []
@@ -268,7 +289,7 @@ def remove(self,num):
"""Remove a finished (completed or dead) job."""
try:
- job = self.jobs_all[num]
+ job = self.all[num]
except KeyError:
error('Job #%s not found' % num)
else:
@@ -277,48 +298,54 @@ def remove(self,num):
error('Job #%s is still running, it can not be removed.' % num)
return
elif stat_code == self._s_completed:
- self.jobs_comp.remove(job)
+ self.completed.remove(job)
elif stat_code == self._s_dead:
- self.jobs_dead.remove(job)
+ self.dead.remove(job)
- def flush_finished(self):
- """Flush all jobs finished (completed and dead) from lists.
+ def flush(self):
+ """Flush all finished jobs (completed and dead) from lists.
Running jobs are never flushed.
It first calls _status_new(), to update info. If any jobs have
completed since the last _status_new() call, the flush operation
aborts."""
- if self._status_new():
- error('New jobs completed since last '\
- '_status_new(), aborting flush.')
- return
-
# Remove the finished jobs from the master dict
- jobs_all = self.jobs_all
- for job in self.jobs_comp+self.jobs_dead:
- del(jobs_all[job.num])
+ alljobs = self.all
+ for job in self.completed+self.dead:
+ del(alljobs[job.num])
# Now flush these lists completely
- fl_comp = self._group_flush(self.jobs_comp,'Completed')
- fl_dead = self._group_flush(self.jobs_dead,'Dead')
+ fl_comp = self._group_flush(self.completed, 'Completed')
+ fl_dead = self._group_flush(self.dead, 'Dead')
if not (fl_comp or fl_dead):
print 'No jobs to flush.'
def result(self,num):
"""result(N) -> return the result of job N."""
try:
- return self.jobs_all[num].result
+ return self.all[num].result
except KeyError:
error('Job #%s not found' % num)
- def traceback(self,num):
+ def _traceback(self, job):
+ num = job if isinstance(job, int) else job.num
try:
- self.jobs_all[num].traceback()
+ self.all[num].traceback()
except KeyError:
error('Job #%s not found' % num)
+ def traceback(self, job=None):
+ if job is None:
+ self._update_status()
+ for deadjob in self.dead:
+ print "Traceback for: %r" % deadjob
+ self._traceback(deadjob)
+ print
+ else:
+ self._traceback(job)
+
class BackgroundJobBase(threading.Thread):
"""Base class to build BackgroundJob classes.
@@ -360,14 +387,19 @@ def _init(self):
self.stat_code = BackgroundJobBase.stat_created_c
self.finished = False
self.result = '<BackgroundJob has not completed>'
+
# reuse the ipython traceback handler if we can get to it, otherwise
# make a new one
try:
- self._make_tb = __IPYTHON__.InteractiveTB.text
+ make_tb = get_ipython().InteractiveTB.text
except:
- self._make_tb = AutoFormattedTB(mode = 'Context',
- color_scheme='NoColor',
- tb_offset = 1).text
+ make_tb = AutoFormattedTB(mode = 'Context',
+ color_scheme='NoColor',
+ tb_offset = 1).text
+ # Note that the actual API for text() requires the three args to be
+ # passed in, so we wrap it in a simple lambda.
+ self._make_tb = lambda : make_tb(None, None, None)
+
# Hold a formatted traceback if one is generated.
self._tb = None
@@ -377,7 +409,7 @@ def __str__(self):
return self.strform
def __repr__(self):
- return '<BackgroundJob: %s>' % self.strform
+ return '<BackgroundJob #%d: %s>' % (self.num, self.strform)
def traceback(self):
print self._tb
@@ -398,10 +430,11 @@ def run(self):
self.stat_code = BackgroundJobBase.stat_completed_c
self.finished = True
+
class BackgroundJobExpr(BackgroundJobBase):
"""Evaluate an expression as a background job (uses a separate thread)."""
- def __init__(self,expression,glob=None,loc=None):
+ def __init__(self, expression, glob=None, loc=None):
"""Create a new job from a string which can be fed to eval().
global/locals dicts can be provided, which will be passed to the eval
@@ -410,11 +443,8 @@ def __init__(self,expression,glob=None,loc=None):
# fail immediately if the given expression can't be compiled
self.code = compile(expression,'<BackgroundJob compilation>','eval')
- if glob is None:
- glob = {}
- if loc is None:
- loc = {}
-
+ glob = {} if glob is None else glob
+ loc = {} if loc is None else loc
self.expression = self.strform = expression
self.glob = glob
self.loc = loc
@@ -423,21 +453,19 @@ def __init__(self,expression,glob=None,loc=None):
def call(self):
return eval(self.code,self.glob,self.loc)
+
class BackgroundJobFunc(BackgroundJobBase):
"""Run a function call as a background job (uses a separate thread)."""
- def __init__(self,func,*args,**kwargs):
+ def __init__(self, func, *args, **kwargs):
"""Create a new job from a callable object.
Any positional arguments and keyword args given to this constructor
after the initial callable are passed directly to it."""
- assert callable(func),'first argument must be callable'
-
- if args is None:
- args = []
- if kwargs is None:
- kwargs = {}
+ if not callable(func):
+ raise TypeError(
+ 'first argument to BackgroundJobFunc must be callable')
self.func = func
self.args = args
@@ -449,42 +477,4 @@ def __init__(self,func,*args,**kwargs):
self._init()
def call(self):
- return self.func(*self.args,**self.kwargs)
-
-
-if __name__=='__main__':
-
- import time
-
- def sleepfunc(interval=2,*a,**kw):
- args = dict(interval=interval,
- args=a,
- kwargs=kw)
- time.sleep(interval)
- return args
-
- def diefunc(interval=2,*a,**kw):
- time.sleep(interval)
- die
-
- def printfunc(interval=1,reps=5):
- for n in range(reps):
- time.sleep(interval)
- print 'In the background...'
-
- jobs = BackgroundJobManager()
- # first job will have # 0
- jobs.new(sleepfunc,4)
- jobs.new(sleepfunc,kw={'reps':2})
- # This makes a job which will die
- jobs.new(diefunc,1)
- jobs.new('printfunc(1,3)')
-
- # after a while, you can get the traceback of a dead job. Run the line
- # below again interactively until it prints a traceback (check the status
- # of the job):
- print jobs[1].status
- jobs[1].traceback()
-
- # Run this line again until the printed result changes
- print "The result of job #0 is:",jobs[0].result
+ return self.func(*self.args, **self.kwargs)
View
91 IPython/lib/tests/test_backgroundjobs.py
@@ -0,0 +1,91 @@
+"""Tests for pylab tools module.
+"""
+#-----------------------------------------------------------------------------
+# Copyright (c) 2011, the IPython Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+from __future__ import print_function
+
+# Stdlib imports
+import sys
+import time
+
+# Third-party imports
+import nose.tools as nt
+
+# Our own imports
+from IPython.lib import backgroundjobs as bg
+from IPython.testing import decorators as dec
+
+#-----------------------------------------------------------------------------
+# Globals and constants
+#-----------------------------------------------------------------------------
+t_short = 0.0001 # very short interval to wait on jobs
+
+#-----------------------------------------------------------------------------
+# Local utilities
+#-----------------------------------------------------------------------------
+def sleeper(interval=t_short, *a, **kw):
+ args = dict(interval=interval,
+ other_args=a,
+ kw_args=kw)
+ time.sleep(interval)
+ return args
+
+def crasher(interval=t_short, *a, **kw):
+ time.sleep(interval)
+ raise Exception("Dead job with interval %s" % interval)
+
+#-----------------------------------------------------------------------------
+# Classes and functions
+#-----------------------------------------------------------------------------
+
+def test_result():
+ """Test job submission and result retrieval"""
+ jobs = bg.BackgroundJobManager()
+ j = jobs.new(sleeper)
+ j.join()
+ nt.assert_equals(j.result['interval'], t_short)
+
+
+def test_flush():
+ """Test job control"""
+ jobs = bg.BackgroundJobManager()
+ j = jobs.new(sleeper)
+ j.join()
+ nt.assert_equals(len(jobs.completed), 1)
+ nt.assert_equals(len(jobs.dead), 0)
+ jobs.flush()
+ nt.assert_equals(len(jobs.completed), 0)
+
+
+def test_dead():
+ """Test control of dead jobs"""
+ jobs = bg.BackgroundJobManager()
+ j = jobs.new(crasher)
+ j.join()
+ nt.assert_equals(len(jobs.completed), 0)
+ nt.assert_equals(len(jobs.dead), 1)
+ jobs.flush()
+ nt.assert_equals(len(jobs.dead), 0)
+
+
+def test_longer():
+ """Test control of longer-running jobs"""
+ jobs = bg.BackgroundJobManager()
+ # Sleep for long enough for the following two checks to still report the
+ # job as running, but not so long that it makes the test suite noticeably
+ # slower.
+ j = jobs.new(sleeper, 0.1)
+ nt.assert_equals(len(jobs.running), 1)
+ nt.assert_equals(len(jobs.completed), 0)
+ j.join()
+ nt.assert_equals(len(jobs.running), 0)
+ nt.assert_equals(len(jobs.completed), 1)
View
193 docs/examples/lib/BackgroundJobs.ipynb
@@ -0,0 +1,193 @@
+{
+ "worksheets": [
+ {
+ "cells": [
+ {
+ "source": "# Simple interactive bacgkround jobs with IPython\n\nWe start by loading the `backgroundjobs` library and defining a few trivial functions to illustrate things with.",
+ "cell_type": "markdown"
+ },
+ {
+ "cell_type": "code",
+ "language": "python",
+ "outputs": [],
+ "collapsed": false,
+ "prompt_number": 35,
+ "input": "from IPython.lib import backgroundjobs as bg\n\nimport sys\nimport time\n\ndef sleepfunc(interval=2, *a, **kw):\n args = dict(interval=interval,\n args=a,\n kwargs=kw)\n time.sleep(interval)\n return args\n\ndef diefunc(interval=2, *a, **kw):\n time.sleep(interval)\n raise Exception(\"Dead job with interval %s\" % interval)\n\ndef printfunc(interval=1, reps=5):\n for n in range(reps):\n time.sleep(interval)\n print 'In the background...', n\n sys.stdout.flush()\n print 'All done!'\n sys.stdout.flush()"
+ },
+ {
+ "source": "Now, we can create a job manager (called simply `jobs`) and use it to submit new jobs.\n<br>\nRun the cell below and wait a few seconds for the whole thing to finish, until you see the \"All done!\" printout.",
+ "cell_type": "markdown"
+ },
+ {
+ "cell_type": "code",
+ "language": "python",
+ "outputs": [
+ {
+ "output_type": "stream",
+ "stream": "stdout",
+ "text": "Starting job # 0 in a separate thread.\nStarting job # 2 in a separate thread.\nStarting job # 3 in a separate thread.\nStarting job # 4 in a separate thread.\nStarting job # 5 in a separate thread.\n"
+ },
+ {
+ "output_type": "stream",
+ "stream": "stdout",
+ "text": "In the background... 0\n"
+ },
+ {
+ "output_type": "stream",
+ "stream": "stdout",
+ "text": "In the background... 1\n"
+ },
+ {
+ "output_type": "stream",
+ "stream": "stdout",
+ "text": "In the background... 2\n"
+ },
+ {
+ "output_type": "stream",
+ "stream": "stdout",
+ "text": "All done!\n"
+ }
+ ],
+ "collapsed": false,
+ "prompt_number": 36,
+ "input": "jobs = bg.BackgroundJobManager()\n\n# Start a few jobs, the first one will have ID # 0\njobs.new(sleepfunc, 4)\njobs.new(sleepfunc, kw={'reps':2})\njobs.new('printfunc(1,3)')\n\n# This makes a couple of jobs which will die. Let's keep a reference to\n# them for easier traceback reporting later\ndiejob1 = jobs.new(diefunc, 1)\ndiejob2 = jobs.new(diefunc, 2)"
+ },
+ {
+ "source": "You can check the status of your jobs at any time:",
+ "cell_type": "markdown"
+ },
+ {
+ "cell_type": "code",
+ "language": "python",
+ "outputs": [
+ {
+ "output_type": "stream",
+ "stream": "stdout",
+ "text": "Completed jobs:\n0 : &lt;function sleepfunc at 0x30e1578&gt;\n2 : &lt;function sleepfunc at 0x30e1578&gt;\n3 : printfunc(1,3)\n\nDead jobs:\n4 : &lt;function diefunc at 0x304d488&gt;\n5 : &lt;function diefunc at 0x304d488&gt;\n\n"
+ }
+ ],
+ "collapsed": false,
+ "prompt_number": 37,
+ "input": "jobs.status()"
+ },
+ {
+ "source": "For any completed job, you can get its result easily:",
+ "cell_type": "markdown"
+ },
+ {
+ "cell_type": "code",
+ "language": "python",
+ "outputs": [],
+ "collapsed": false,
+ "prompt_number": 43,
+ "input": "jobs[0].result\nj0 = jobs[0]\nj0.join?"
+ },
+ {
+ "source": "You can get the traceback of any dead job. Run the line\nbelow again interactively until it prints a traceback (check the status\nof the job):\n",
+ "cell_type": "markdown"
+ },
+ {
+ "cell_type": "code",
+ "language": "python",
+ "outputs": [
+ {
+ "output_type": "stream",
+ "stream": "stdout",
+ "text": "Status of diejob1: Dead (Exception), call jobs.traceback() for details\n<span class=\"ansired\">---------------------------------------------------------------------------</span>\n<span class=\"ansired\">Exception</span> Traceback (most recent call last)\n<span class=\"ansigreen\">/home/fperez/usr/lib/python2.6/site-packages/IPython/lib/backgroundjobs.py</span> in <span class=\"ansicyan\">call</span><span class=\"ansiblue\">(self)</span>\n<span class=\"ansigreen\"> 462</span> <span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\"> 463</span> <span class=\"ansigreen\">def</span> call<span class=\"ansiyellow\">(</span>self<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">--&gt; 464</span><span class=\"ansiyellow\"> </span><span class=\"ansigreen\">return</span> self<span class=\"ansiyellow\">.</span>func<span class=\"ansiyellow\">(</span><span class=\"ansiyellow\">*</span>self<span class=\"ansiyellow\">.</span>args<span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">**</span>self<span class=\"ansiyellow\">.</span>kwargs<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n\n<span class=\"ansigreen\">/home/fperez/ipython/ipython/docs/examples/lib/&lt;ipython-input-15-54795a097787&gt;</span> in <span class=\"ansicyan\">diefunc</span><span class=\"ansiblue\">(interval, *a, **kw)</span>\n<span class=\"ansigreen\"> 14</span> <span class=\"ansigreen\">def</span> diefunc<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">2</span><span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">*</span>a<span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">**</span>kw<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\"> 15</span> time<span class=\"ansiyellow\">.</span>sleep<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">---&gt; 16</span><span class=\"ansiyellow\"> </span><span class=\"ansigreen\">raise</span> Exception<span class=\"ansiyellow\">(</span><span class=\"ansiblue\">&quot;Dead job with interval %s&quot;</span> <span class=\"ansiyellow\">%</span> interval<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\"> 17</span> <span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\"> 18</span> <span class=\"ansigreen\">def</span> printfunc<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">1</span><span class=\"ansiyellow\">,</span> reps<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">5</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n\n<span class=\"ansired\">Exception</span>: Dead job with interval 1\n"
+ }
+ ],
+ "collapsed": false,
+ "prompt_number": 34,
+ "input": "print \"Status of diejob1:\", diejob1.status\ndiejob1.traceback() # jobs.traceback(4) would also work here, with the job number"
+ },
+ {
+ "source": "This will print all tracebacks for all dead jobs:",
+ "cell_type": "markdown"
+ },
+ {
+ "cell_type": "code",
+ "language": "python",
+ "outputs": [
+ {
+ "output_type": "stream",
+ "stream": "stdout",
+ "text": "Traceback for: &lt;BackgroundJob #4: &lt;function diefunc at 0x30df758&gt;&gt;\n<span class=\"ansired\">---------------------------------------------------------------------------</span>\n<span class=\"ansired\">Exception</span> Traceback (most recent call last)\n<span class=\"ansigreen\">/home/fperez/usr/lib/python2.6/site-packages/IPython/lib/backgroundjobs.py</span> in <span class=\"ansicyan\">call</span><span class=\"ansiblue\">(self)</span>\n<span class=\"ansigreen\"> 462</span> <span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\"> 463</span> <span class=\"ansigreen\">def</span> call<span class=\"ansiyellow\">(</span>self<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">--&gt; 464</span><span class=\"ansiyellow\"> </span><span class=\"ansigreen\">return</span> self<span class=\"ansiyellow\">.</span>func<span class=\"ansiyellow\">(</span><span class=\"ansiyellow\">*</span>self<span class=\"ansiyellow\">.</span>args<span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">**</span>self<span class=\"ansiyellow\">.</span>kwargs<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n\n<span class=\"ansigreen\">/home/fperez/ipython/ipython/docs/examples/lib/&lt;ipython-input-15-54795a097787&gt;</span> in <span class=\"ansicyan\">diefunc</span><span class=\"ansiblue\">(interval, *a, **kw)</span>\n<span class=\"ansigreen\"> 14</span> <span class=\"ansigreen\">def</span> diefunc<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">2</span><span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">*</span>a<span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">**</span>kw<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\"> 15</span> time<span class=\"ansiyellow\">.</span>sleep<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">---&gt; 16</span><span class=\"ansiyellow\"> </span><span class=\"ansigreen\">raise</span> Exception<span class=\"ansiyellow\">(</span><span class=\"ansiblue\">&quot;Dead job with interval %s&quot;</span> <span class=\"ansiyellow\">%</span> interval<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\"> 17</span> <span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\"> 18</span> <span class=\"ansigreen\">def</span> printfunc<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">1</span><span class=\"ansiyellow\">,</span> reps<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">5</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n\n<span class=\"ansired\">Exception</span>: Dead job with interval 1\n\nTraceback for: &lt;BackgroundJob #5: &lt;function diefunc at 0x30df758&gt;&gt;\n<span class=\"ansired\">---------------------------------------------------------------------------</span>\n<span class=\"ansired\">Exception</span> Traceback (most recent call last)\n<span class=\"ansigreen\">/home/fperez/usr/lib/python2.6/site-packages/IPython/lib/backgroundjobs.py</span> in <span class=\"ansicyan\">call</span><span class=\"ansiblue\">(self)</span>\n<span class=\"ansigreen\"> 462</span> <span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\"> 463</span> <span class=\"ansigreen\">def</span> call<span class=\"ansiyellow\">(</span>self<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">--&gt; 464</span><span class=\"ansiyellow\"> </span><span class=\"ansigreen\">return</span> self<span class=\"ansiyellow\">.</span>func<span class=\"ansiyellow\">(</span><span class=\"ansiyellow\">*</span>self<span class=\"ansiyellow\">.</span>args<span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">**</span>self<span class=\"ansiyellow\">.</span>kwargs<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n\n<span class=\"ansigreen\">/home/fperez/ipython/ipython/docs/examples/lib/&lt;ipython-input-15-54795a097787&gt;</span> in <span class=\"ansicyan\">diefunc</span><span class=\"ansiblue\">(interval, *a, **kw)</span>\n<span class=\"ansigreen\"> 14</span> <span class=\"ansigreen\">def</span> diefunc<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">2</span><span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">*</span>a<span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">**</span>kw<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\"> 15</span> time<span class=\"ansiyellow\">.</span>sleep<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">---&gt; 16</span><span class=\"ansiyellow\"> </span><span class=\"ansigreen\">raise</span> Exception<span class=\"ansiyellow\">(</span><span class=\"ansiblue\">&quot;Dead job with interval %s&quot;</span> <span class=\"ansiyellow\">%</span> interval<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\"> 17</span> <span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\"> 18</span> <span class=\"ansigreen\">def</span> printfunc<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">1</span><span class=\"ansiyellow\">,</span> reps<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">5</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n\n<span class=\"ansired\">Exception</span>: Dead job with interval 2\n\n"
+ }
+ ],
+ "collapsed": false,
+ "prompt_number": 33,
+ "input": "jobs.traceback()"
+ },
+ {
+ "source": "The job manager can be flushed of all completed jobs at any time:",
+ "cell_type": "markdown"
+ },
+ {
+ "cell_type": "code",
+ "language": "python",
+ "outputs": [
+ {
+ "output_type": "stream",
+ "stream": "stdout",
+ "text": "No jobs to flush.\n"
+ }
+ ],
+ "collapsed": false,
+ "prompt_number": 25,
+ "input": "jobs.flush()"
+ },
+ {
+ "source": "After that, the status is simply empty:",
+ "cell_type": "markdown"
+ },
+ {
+ "cell_type": "code",
+ "language": "python",
+ "outputs": [],
+ "collapsed": true,
+ "prompt_number": 27,
+ "input": "jobs.status()"
+ },
+ {
+ "source": "It's easy to wait on a job:",
+ "cell_type": "markdown"
+ },
+ {
+ "cell_type": "code",
+ "language": "python",
+ "outputs": [
+ {
+ "output_type": "stream",
+ "stream": "stdout",
+ "text": "Starting job # 7 in a separate thread.\nWill wait for j now...\n"
+ },
+ {
+ "output_type": "stream",
+ "stream": "stdout",
+ "text": "Result from j:\n"
+ },
+ {
+ "output_type": "pyout",
+ "prompt_number": 46,
+ "text": "{&apos;args&apos;: (), &apos;interval&apos;: 2, &apos;kwargs&apos;: {}}"
+ }
+ ],
+ "collapsed": false,
+ "prompt_number": 46,
+ "input": "j = jobs.new(sleepfunc, 2)\nprint \"Will wait for j now...\"\nsys.stdout.flush()\nj.join()\nprint \"Result from j:\"\nj.result"
+ },
+ {
+ "input": "",
+ "cell_type": "code",
+ "collapsed": true,
+ "language": "python",
+ "outputs": []
+ }
+ ]
+ }
+ ],
+ "metadata": {
+ "name": "BackgroundJobs"
+ },
+ "nbformat": 2
+}
View
26 docs/source/interactive/htmlnotebook.txt
@@ -132,15 +132,31 @@ console, is that it can not run any code that expects input from the kernel
that the ``%debug`` magic does *not* work in the notebook! We intend to
correct this limitation, but in the meantime, there is a way to debug problems
in the notebook: you can attach a Qt console to your existing notebook kernel,
-and run ``%debug`` from the Qt console. Simply look for the lines in the
-terminal where you started the kernel that read something like::
+and run ``%debug`` from the Qt console. If your notebook is running on a local
+computer (i.e. if you are accessing it via your localhost address at
+127.0.0.1), you can just type ``%qtconsole`` in the notebook and a Qt console
+will open up connected to that same kernel.
+
+In general, the notebook server prints the full details of how to connect to
+each kernel at the terminal, with lines like:
[IPKernelApp] To connect another client to this kernel, use:
- [IPKernelApp] --existing --shell=53328 --iopub=53817 --stdin=34736 --hb=45543
+ [IPKernelApp] --existing kernel-3bb93edd-6b5a-455c-99c8-3b658f45dde5.json
+
+This is the name of a JSON file that contains all the port and validation
+information necessary to connect to the kernel. You can manually start a
+qt console with::
+
+ ipython qtconsole --existing kernel-3bb93edd-6b5a-455c-99c8-3b658f45dde5.json
+
+and if you only have a single kernel running, simply typing::
-and then start a qt console pointing to that kernel::
+ ipython qtconsole --existing
- ipython qtconsole --existing --shell=53328 --iopub=53817 --stdin=34736 --hb=45543
+will automatically find it (it will always find the most recently started
+kernel if there is more than one). You can also request this connection data
+by typing ``%connect_info``; this will print the same file information as well
+as the content of the JSON data structure it contains.
Text input

0 comments on commit ad50083

Please sign in to comment.
Something went wrong with that request. Please try again.