From 31c15b8f73713c934f7b0c64ae6fc602bea7d99c Mon Sep 17 00:00:00 2001 From: MinRK Date: Sat, 9 Apr 2011 12:40:14 -0700 Subject: [PATCH 1/4] fix %autopx in scripts by calling run_code for each ast node Previously, there was no call to override if %autopx was called inside a single cell (e.g. %run script.ipy). This splits run_ast_nodes, so that each node gets a separate call to run_code. reviewed changes: * remove old use of 'new' * fix interactiveshell.run_code docstring Additional change: automatically load parallelmagic extension on view.activate() --- IPython/core/interactiveshell.py | 35 +++++++----- IPython/extensions/parallelmagic.py | 84 +++++++++++++++++++++-------- IPython/parallel/client/view.py | 10 ++-- 3 files changed, 89 insertions(+), 40 deletions(-) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 89fa68796ae..72c4de9b846 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -2211,15 +2211,20 @@ def run_ast_nodes(self, nodelist, cell_name, interactivity='last'): exec_count = self.execution_count if to_run_exec: - mod = ast.Module(to_run_exec) - self.code_to_run = code = self.compile(mod, cell_name, "exec") - if self.run_code(code) == 1: - return - + for i, node in enumerate(to_run_exec): + mod = ast.Module([node]) + self.code_to_run = code = self.compile(mod, cell_name+str(i), "exec") + if self.run_code(code) == 1: + return 1 + if to_run_interactive: - mod = ast.Interactive(to_run_interactive) - self.code_to_run = code = self.compile(mod, cell_name, "single") - return self.run_code(code) + for i, node in enumerate(to_run_interactive): + mod = ast.Interactive([node]) + self.code_to_run = code = self.compile(mod, cell_name, "single") + if self.run_code(code) == 1: + return 1 + + return 0 # PENDING REMOVAL: this method is slated for deletion, once our new @@ -2336,11 +2341,17 @@ def run_code(self, code_obj): When an exception occurs, self.showtraceback() is called to display a traceback. - Return value: a flag indicating whether the code to be run completed - successfully: + Parameters + ---------- + code_obj : code object + A compiled code object, to be executed + post_execute : bool [default: True] + whether to call post_execute hooks after this particular execution. - - 0: successful execution. - - 1: an error occurred. + Returns + ------- + 0 : successful execution. + 1 : an error occurred. """ # Set our own excepthook in case the user code tries to call it diff --git a/IPython/extensions/parallelmagic.py b/IPython/extensions/parallelmagic.py index ba937623b29..ea672a171cd 100755 --- a/IPython/extensions/parallelmagic.py +++ b/IPython/extensions/parallelmagic.py @@ -15,12 +15,10 @@ #----------------------------------------------------------------------------- import ast -import new import re from IPython.core.plugin import Plugin from IPython.utils.traitlets import Bool, Any, Instance -from IPython.utils.autoattr import auto_attr from IPython.testing import decorators as testdec #----------------------------------------------------------------------------- @@ -146,10 +144,12 @@ def _enable_autopx(self): print NO_ACTIVE_VIEW return + # override run_cell and run_code self._original_run_cell = self.shell.run_cell - self.shell.run_cell = new.instancemethod( - self.pxrun_cell, self.shell, self.shell.__class__ - ) + self.shell.run_cell = self.pxrun_cell + self._original_run_code = self.shell.run_code + self.shell.run_code = self.pxrun_code + self.autopx = True print "%autopx enabled" @@ -158,14 +158,43 @@ def _disable_autopx(self): """ if self.autopx: self.shell.run_cell = self._original_run_cell + self.shell.run_code = self._original_run_code self.autopx = False print "%autopx disabled" - def pxrun_cell(self, ipself, cell, store_history=True): + def _maybe_display_output(self, result): + """Maybe display the output of a parallel result. + + If self.active_view.block is True, wait for the result + and display the result. Otherwise, this is a noop. + """ + if self.active_view.block: + try: + result.get() + except: + self.shell.showtraceback() + return True + else: + targets = self.active_view.targets + if isinstance(targets, int): + targets = [targets] + if targets == 'all': + targets = self.active_view.client.ids + stdout = [s.rstrip() for s in result.stdout] + if any(stdout): + for i,eid in enumerate(targets): + print '[stdout:%i]'%eid, stdout[i] + return False + + + def pxrun_cell(self, cell, store_history=True): """drop-in replacement for InteractiveShell.run_cell. This executes code remotely, instead of in the local namespace. + + See InteractiveShell.run_cell for details. """ + ipself = self.shell raw_cell = cell with ipself.builtin_trap: cell = ipself.prefilter_manager.prefilter_lines(cell) @@ -187,6 +216,7 @@ def pxrun_cell(self, ipself, cell, store_history=True): ipself.execution_count += 1 return None except NameError: + # ignore name errors, because we don't know the remote keys pass if store_history: @@ -195,7 +225,6 @@ def pxrun_cell(self, ipself, cell, store_history=True): ipself.history_manager.store_output(ipself.execution_count) # Each cell is a *single* input, regardless of how many lines it has ipself.execution_count += 1 - print cell if re.search(r'get_ipython\(\)\.magic\(u?"%?autopx', cell): self._disable_autopx() @@ -206,23 +235,32 @@ def pxrun_cell(self, ipself, cell, store_history=True): except: ipself.showtraceback() return False - - if self.active_view.block: - try: - result.get() - except: - ipself.showtraceback() - else: - targets = self.active_view.targets - if isinstance(targets, int): - targets = [targets] - if targets == 'all': - targets = self.active_view.client.ids - stdout = [s.rstrip() for s in result.stdout] - if any(stdout): - for i,eid in enumerate(targets): - print '[stdout:%i]'%eid, stdout[i] + else: + return self._maybe_display_output(result) + + def pxrun_code(self, code_obj, post_execute=True): + """drop-in replacement for InteractiveShell.run_code. + + This executes code remotely, instead of in the local namespace. + + See InteractiveShell.run_code for details. + """ + ipself = self.shell + # check code object for the autopx magic + if 'get_ipython' in code_obj.co_names and 'magic' in code_obj.co_names and \ + any( [ isinstance(c, basestring) and 'autopx' in c for c in code_obj.co_consts ]): + self._disable_autopx() return False + else: + try: + result = self.active_view.execute(code_obj, block=False) + except: + ipself.showtraceback() + return False + else: + return self._maybe_display_output(result) + + _loaded = False diff --git a/IPython/parallel/client/view.py b/IPython/parallel/client/view.py index fb6903b98b9..647729df2f8 100644 --- a/IPython/parallel/client/view.py +++ b/IPython/parallel/client/view.py @@ -765,11 +765,11 @@ def activate(self): print "The IPython parallel magics (%result, %px, %autopx) only work within IPython." else: pmagic = ip.plugin_manager.get_plugin('parallelmagic') - if pmagic is not None: - pmagic.active_view = self - else: - print "You must first load the parallelmagic extension " \ - "by doing '%load_ext parallelmagic'" + if pmagic is None: + ip.magic_load_ext('parallelmagic') + pmagic = ip.plugin_manager.get_plugin('parallelmagic') + + pmagic.active_view = self @testdec.skip_doctest From 8cf5bb69b5750be6c22e883acee6c69ede3f9777 Mon Sep 17 00:00:00 2001 From: MinRK Date: Sun, 10 Apr 2011 02:10:04 -0700 Subject: [PATCH 2/4] add tests for parallel magics tests revealed some changes that should be made: %px also prints remote stdout in blocking mode, and actually raises the RemoteError (not just display) if there is one --- IPython/extensions/parallelmagic.py | 68 ++++++++++------- IPython/parallel/tests/test_view.py | 111 ++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+), 27 deletions(-) diff --git a/IPython/extensions/parallelmagic.py b/IPython/extensions/parallelmagic.py index ea672a171cd..f04309231a3 100755 --- a/IPython/extensions/parallelmagic.py +++ b/IPython/extensions/parallelmagic.py @@ -34,7 +34,7 @@ class ParalleMagic(Plugin): """A component to manage the %result, %px and %autopx magics.""" - active_view = Any() + active_view = Instance('IPython.parallel.client.view.DirectView') verbose = Bool(False, config=True) shell = Instance('IPython.core.interactiveshell.InteractiveShellABC') @@ -103,8 +103,10 @@ def magic_px(self, ipself, parameter_s=''): print NO_ACTIVE_VIEW return print "Parallel execution on engines: %s" % self.active_view.targets - result = self.active_view.execute(parameter_s) - return result + result = self.active_view.execute(parameter_s, block=False) + if self.active_view.block: + result.get() + self._maybe_display_output(result) @testdec.skip_doctest def magic_autopx(self, ipself, parameter_s=''): @@ -123,9 +125,13 @@ def magic_autopx(self, ipself, parameter_s=''): %autopx to enabled In [26]: a = 10 - - [0] In [8]: a = 10 - [1] In [8]: a = 10 + Parallel execution on engines: [0,1,2,3] + In [27]: print a + Parallel execution on engines: [0,1,2,3] + [stdout:0] 10 + [stdout:1] 10 + [stdout:2] 10 + [stdout:3] 10 In [27]: %autopx @@ -168,23 +174,15 @@ def _maybe_display_output(self, result): If self.active_view.block is True, wait for the result and display the result. Otherwise, this is a noop. """ - if self.active_view.block: - try: - result.get() - except: - self.shell.showtraceback() - return True - else: - targets = self.active_view.targets - if isinstance(targets, int): - targets = [targets] - if targets == 'all': - targets = self.active_view.client.ids - stdout = [s.rstrip() for s in result.stdout] - if any(stdout): - for i,eid in enumerate(targets): - print '[stdout:%i]'%eid, stdout[i] - return False + targets = self.active_view.targets + if isinstance(targets, int): + targets = [targets] + if targets == 'all': + targets = self.active_view.client.ids + stdout = [s.rstrip() for s in result.stdout] + if any(stdout): + for i,eid in enumerate(targets): + print '[stdout:%i]'%eid, stdout[i] def pxrun_cell(self, cell, store_history=True): @@ -234,9 +232,17 @@ def pxrun_cell(self, cell, store_history=True): result = self.active_view.execute(cell, block=False) except: ipself.showtraceback() - return False + return True else: - return self._maybe_display_output(result) + if self.active_view.block: + try: + result.get() + except: + self.shell.showtraceback() + return True + else: + self._maybe_display_output(result) + return False def pxrun_code(self, code_obj, post_execute=True): """drop-in replacement for InteractiveShell.run_code. @@ -256,9 +262,17 @@ def pxrun_code(self, code_obj, post_execute=True): result = self.active_view.execute(code_obj, block=False) except: ipself.showtraceback() - return False + return True else: - return self._maybe_display_output(result) + if self.active_view.block: + try: + result.get() + except: + self.shell.showtraceback() + return True + else: + self._maybe_display_output(result) + return False diff --git a/IPython/parallel/tests/test_view.py b/IPython/parallel/tests/test_view.py index 629d9acc6f4..c6b72b5a5b7 100644 --- a/IPython/parallel/tests/test_view.py +++ b/IPython/parallel/tests/test_view.py @@ -10,8 +10,10 @@ # Imports #------------------------------------------------------------------------------- +import sys import time from tempfile import mktemp +from StringIO import StringIO import zmq @@ -300,3 +302,112 @@ def findall(pat, s): self.assertEquals(view.apply_sync(findall, '\w+', 'hello world'), 'hello world'.split()) + # parallel magic tests + + def test_magic_px_blocking(self): + ip = get_ipython() + v = self.client[-1] + v.activate() + v.block=True + + ip.magic_px('a=5') + self.assertEquals(v['a'], 5) + ip.magic_px('a=10') + self.assertEquals(v['a'], 10) + sio = StringIO() + savestdout = sys.stdout + sys.stdout = sio + ip.magic_px('print a') + sys.stdout = savestdout + sio.read() + self.assertTrue('[stdout:%i]'%v.targets in sio.buf) + self.assertRaisesRemote(ZeroDivisionError, ip.magic_px, '1/0') + + def test_magic_px_nonblocking(self): + ip = get_ipython() + v = self.client[-1] + v.activate() + v.block=False + + ip.magic_px('a=5') + self.assertEquals(v['a'], 5) + ip.magic_px('a=10') + self.assertEquals(v['a'], 10) + sio = StringIO() + savestdout = sys.stdout + sys.stdout = sio + ip.magic_px('print a') + sys.stdout = savestdout + sio.read() + self.assertFalse('[stdout:%i]'%v.targets in sio.buf) + ip.magic_px('1/0') + ar = v.get_result(-1) + self.assertRaisesRemote(ZeroDivisionError, ar.get) + + def test_magic_autopx_blocking(self): + ip = get_ipython() + v = self.client[-1] + v.activate() + v.block=True + + sio = StringIO() + savestdout = sys.stdout + sys.stdout = sio + ip.magic_autopx() + ip.run_cell('\n'.join(('a=5','b=10','c=0'))) + ip.run_cell('print b') + ip.run_cell("b/c") + ip.run_code(compile('b*=2', '', 'single')) + ip.magic_autopx() + sys.stdout = savestdout + sio.read() + output = sio.buf.strip() + self.assertTrue(output.startswith('%autopx enabled')) + self.assertTrue(output.endswith('%autopx disabled')) + self.assertTrue('RemoteError: ZeroDivisionError' in output) + ar = v.get_result(-2) + self.assertEquals(v['a'], 5) + self.assertEquals(v['b'], 20) + self.assertRaisesRemote(ZeroDivisionError, ar.get) + + def test_magic_autopx_nonblocking(self): + ip = get_ipython() + v = self.client[-1] + v.activate() + v.block=False + + sio = StringIO() + savestdout = sys.stdout + sys.stdout = sio + ip.magic_autopx() + ip.run_cell('\n'.join(('a=5','b=10','c=0'))) + ip.run_cell('print b') + ip.run_cell("b/c") + ip.run_code(compile('b*=2', '', 'single')) + ip.magic_autopx() + sys.stdout = savestdout + sio.read() + output = sio.buf.strip() + self.assertTrue(output.startswith('%autopx enabled')) + self.assertTrue(output.endswith('%autopx disabled')) + self.assertFalse('ZeroDivisionError' in output) + ar = v.get_result(-2) + self.assertEquals(v['a'], 5) + self.assertEquals(v['b'], 20) + self.assertRaisesRemote(ZeroDivisionError, ar.get) + + def test_magic_result(self): + ip = get_ipython() + v = self.client[-1] + v.activate() + v['a'] = 111 + ra = v['a'] + + ar = ip.magic_result() + self.assertEquals(ar.msg_ids, [v.history[-1]]) + self.assertEquals(ar.get(), 111) + ar = ip.magic_result('-2') + self.assertEquals(ar.msg_ids, [v.history[-2]]) + + + From b5a53eccf84368771a9697a096beb66842a30807 Mon Sep 17 00:00:00 2001 From: MinRK Date: Mon, 11 Apr 2011 11:24:50 -0700 Subject: [PATCH 3/4] remove code_to_run attribute, updates per review. review changes: * change 1/0 to True/False in run_code * remove interactiveshell.code_to_run * remove superfluous if-tests checking for empty lists in run_ast_nodes closes gh-349 --- IPython/core/interactiveshell.py | 48 +++++++++++--------------------- 1 file changed, 17 insertions(+), 31 deletions(-) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 72c4de9b846..2198484cc9b 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -443,12 +443,6 @@ def init_instance_attrs(self): # ipython names that may develop later. self.meta = Struct() - # Object variable to store code object waiting execution. This is - # used mainly by the multithreaded shells, but it can come in handy in - # other situations. No need to use a Queue here, since it's a single - # item which gets cleared once run. - self.code_to_run = None - # Temporary files used for various purposes. Deleted at exit. self.tempfiles = [] @@ -2210,21 +2204,20 @@ def run_ast_nodes(self, nodelist, cell_name, interactivity='last'): raise ValueError("Interactivity was %r" % interactivity) exec_count = self.execution_count - if to_run_exec: - for i, node in enumerate(to_run_exec): - mod = ast.Module([node]) - self.code_to_run = code = self.compile(mod, cell_name+str(i), "exec") - if self.run_code(code) == 1: - return 1 - - if to_run_interactive: - for i, node in enumerate(to_run_interactive): - mod = ast.Interactive([node]) - self.code_to_run = code = self.compile(mod, cell_name, "single") - if self.run_code(code) == 1: - return 1 - - return 0 + + for i, node in enumerate(to_run_exec): + mod = ast.Module([node]) + code = self.compile(mod, cell_name, "exec") + if self.run_code(code): + return True + + for i, node in enumerate(to_run_interactive): + mod = ast.Interactive([node]) + code = self.compile(mod, cell_name, "single") + if self.run_code(code): + return True + + return False # PENDING REMOVAL: this method is slated for deletion, once our new @@ -2321,13 +2314,8 @@ def run_source(self, source, filename=None, symbol='single'): return True # Case 3 - # We store the code object so that threaded shells and - # custom exception handlers can access all this info if needed. - # The source corresponding to this can be obtained from the - # buffer attribute as '\n'.join(self.buffer). - self.code_to_run = code # now actually execute the code object - if self.run_code(code) == 0: + if not self.run_code(code): return False else: return None @@ -2350,8 +2338,8 @@ def run_code(self, code_obj): Returns ------- - 0 : successful execution. - 1 : an error occurred. + False : successful execution. + True : an error occurred. """ # Set our own excepthook in case the user code tries to call it @@ -2384,8 +2372,6 @@ def run_code(self, code_obj): if softspace(sys.stdout, 0): print - # Flush out code object which has been run (and source) - self.code_to_run = None return outflag # For backwards compatibility From e68cf3acfde04fb1164c3b13d32f69fceb68eaab Mon Sep 17 00:00:00 2001 From: MinRK Date: Mon, 11 Apr 2011 11:36:19 -0700 Subject: [PATCH 4/4] update parallel %autopx magics to reflect recent changes in run_cell, run_code --- IPython/extensions/parallelmagic.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/IPython/extensions/parallelmagic.py b/IPython/extensions/parallelmagic.py index f04309231a3..61c1dfb8c17 100755 --- a/IPython/extensions/parallelmagic.py +++ b/IPython/extensions/parallelmagic.py @@ -185,17 +185,21 @@ def _maybe_display_output(self, result): print '[stdout:%i]'%eid, stdout[i] - def pxrun_cell(self, cell, store_history=True): + def pxrun_cell(self, raw_cell, store_history=True): """drop-in replacement for InteractiveShell.run_cell. This executes code remotely, instead of in the local namespace. See InteractiveShell.run_cell for details. """ + + if (not raw_cell) or raw_cell.isspace(): + return + ipself = self.shell - raw_cell = cell + with ipself.builtin_trap: - cell = ipself.prefilter_manager.prefilter_lines(cell) + cell = ipself.prefilter_manager.prefilter_lines(raw_cell) # Store raw and processed history if store_history: @@ -244,7 +248,7 @@ def pxrun_cell(self, cell, store_history=True): self._maybe_display_output(result) return False - def pxrun_code(self, code_obj, post_execute=True): + def pxrun_code(self, code_obj): """drop-in replacement for InteractiveShell.run_code. This executes code remotely, instead of in the local namespace.