From 8bb887c8c2c447bf7db59c904e6280742fe3ff07 Mon Sep 17 00:00:00 2001 From: Fernando Perez Date: Wed, 16 Nov 2011 22:04:35 -0800 Subject: [PATCH 1/3] Fix paste/cpaste bug and refactor/cleanup that code a lot. In fixing a pasting bug (mishandling of whitespace when input had prompts) it became clear the pasting code hadn't been updated when the new prefiltering machinery was added. Furthermore, the pasting magics are only for the terminal, but the code was in the base classes. This refactors and simplifies the pasting code, moving it to the terminal shell only, and removing unnecessary methods from the main class (using small utility functions instead). The tests were simplified because the previous regexp supported some odd edge cases that are not valid in the normal prefiltering code. We want to have a single location and set of rules for input prefiltering, so I changed some of the test cases to be consistent with what prefilter allows. --- IPython/core/magic.py | 60 -------------- IPython/core/tests/test_magic.py | 17 ++-- IPython/frontend/terminal/interactiveshell.py | 80 ++++++++++++++----- 3 files changed, 69 insertions(+), 88 deletions(-) diff --git a/IPython/core/magic.py b/IPython/core/magic.py index a64acadcdbd..a6b905ee836 100644 --- a/IPython/core/magic.py +++ b/IPython/core/magic.py @@ -25,7 +25,6 @@ import shutil import re import time -import textwrap from StringIO import StringIO from getopt import getopt,GetoptError from pprint import pformat @@ -3203,65 +3202,6 @@ def magic_pycat(self, parameter_s=''): page.page(self.shell.pycolorize(cont)) - def _rerun_pasted(self): - """ Rerun a previously pasted command. - """ - b = self.user_ns.get('pasted_block', None) - if b is None: - raise UsageError('No previous pasted block available') - print "Re-executing '%s...' (%d chars)"% (b.split('\n',1)[0], len(b)) - exec b in self.user_ns - - def _get_pasted_lines(self, sentinel): - """ Yield pasted lines until the user enters the given sentinel value. - """ - from IPython.core import interactiveshell - print "Pasting code; enter '%s' alone on the line to stop." % sentinel - while True: - try: - l = self.shell.raw_input_original(':') - if l == sentinel: - return - else: - yield l - except EOFError: - print '' - return - - def _strip_pasted_lines_for_code(self, raw_lines): - """ Strip non-code parts of a sequence of lines to return a block of - code. - """ - # Regular expressions that declare text we strip from the input: - strip_re = [r'^\s*In \[\d+\]:', # IPython input prompt - r'^\s*(\s?>)+', # Python input prompt - r'^\s*\.{3,}', # Continuation prompts - r'^\++', - ] - - strip_from_start = map(re.compile,strip_re) - - lines = [] - for l in raw_lines: - for pat in strip_from_start: - l = pat.sub('',l) - lines.append(l) - - block = "\n".join(lines) + '\n' - #print "block:\n",block - return block - - def _execute_block(self, block, par): - """ Execute a block, or store it in a variable, per the user's request. - """ - if not par: - b = textwrap.dedent(block) - self.user_ns['pasted_block'] = b - self.run_cell(b) - else: - self.user_ns[par] = SList(block.splitlines()) - print "Block assigned to '%s'" % par - def magic_quickref(self,arg): """ Show a quick reference sheet """ import IPython.core.usage diff --git a/IPython/core/tests/test_magic.py b/IPython/core/tests/test_magic.py index ebcd51023dc..96def1ac86a 100644 --- a/IPython/core/tests/test_magic.py +++ b/IPython/core/tests/test_magic.py @@ -355,14 +355,15 @@ def run(): _ip.user_ns['code_ran'] = True return 'run' # return string so '+ run()' doesn't result in success - tests = {'pass': ["> > > run()", - ">>> > run()", - "+++ run()", - "++ run()", - " >>> run()"], - - 'fail': ["+ + run()", - " ++ run()"]} + tests = {'pass': ["run()", + "In [1]: run()", + "In [1]: if 1:\n ...: run()", + ">>> run()", + " >>> run()", + ], + + 'fail': ["1 + run()", + "++ run()"]} _ip.user_ns['run'] = run diff --git a/IPython/frontend/terminal/interactiveshell.py b/IPython/frontend/terminal/interactiveshell.py index 1f7369196a0..42ef31ea9b4 100644 --- a/IPython/frontend/terminal/interactiveshell.py +++ b/IPython/frontend/terminal/interactiveshell.py @@ -19,6 +19,7 @@ import os import re import sys +import textwrap try: from contextlib import nested @@ -35,7 +36,7 @@ from IPython.utils.terminal import toggle_set_term_title, set_term_title from IPython.utils.process import abbrev_cwd from IPython.utils.warn import warn, error -from IPython.utils.text import num_ini_spaces +from IPython.utils.text import num_ini_spaces, SList from IPython.utils.traitlets import Integer, CBool, Unicode #----------------------------------------------------------------------------- @@ -52,6 +53,48 @@ def get_default_editor(): ed = 'notepad' # same in Windows! return ed + +def get_pasted_lines(sentinel, input=raw_input): + """ Yield pasted lines until the user enters the given sentinel value. + """ + print "Pasting code; enter '%s' alone on the line to stop or use Ctrl-D." \ + % sentinel + while True: + try: + l = input(':') + if l == sentinel: + return + else: + yield l + except EOFError: + print '' + return + + +def store_or_execute(shell, block, name): + """ Execute a block, or store it in a variable, per the user's request. + """ + if name: + # If storing it for further editing, run the prefilter on it + shell.user_ns[name] = SList(shell.prefilter(block).splitlines()) + print "Block assigned to '%s'" % name + else: + # For execution we just dedent it, as all other filtering is + # automatically applied by run_cell + b = textwrap.dedent(block) + shell.user_ns['pasted_block'] = b + shell.run_cell(b) + + +def rerun_pasted(shell): + """ Rerun a previously pasted command. + """ + b = shell.user_ns.get('pasted_block') + if b is None: + raise UsageError('No previous pasted block available') + print "Re-executing '%s...' (%d chars)"% (b.split('\n',1)[0], len(b)) + shell.run_cell(b) + #----------------------------------------------------------------------------- # Main class #----------------------------------------------------------------------------- @@ -523,9 +566,9 @@ def magic_autoindent(self, parameter_s = ''): def magic_cpaste(self, parameter_s=''): """Paste & execute a pre-formatted code block from clipboard. - You must terminate the block with '--' (two minus-signs) or Ctrl-D alone on the - line. You can also provide your own sentinel with '%paste -s %%' ('%%' - is the new sentinel for this operation) + You must terminate the block with '--' (two minus-signs) or Ctrl-D + alone on the line. You can also provide your own sentinel with '%paste + -s %%' ('%%' is the new sentinel for this operation) The block is dedented prior to execution to enable execution of method definitions. '>' and '+' characters at the beginning of a line are @@ -562,18 +605,15 @@ def magic_cpaste(self, parameter_s=''): Hello world! """ - opts,args = self.parse_options(parameter_s,'rs:',mode='string') - par = args.strip() - if opts.has_key('r'): - self._rerun_pasted() + opts, args = self.parse_options(parameter_s, 'rs:', mode='string') + name = args.strip() + if 'r' in opts: + rerun_pasted(self.shell) return - sentinel = opts.get('s','--') - - block = self._strip_pasted_lines_for_code( - self._get_pasted_lines(sentinel)) - - self._execute_block(block, par) + sentinel = opts.get('s', '--') + block = '\n'.join(get_pasted_lines(sentinel)) + store_or_execute(self.shell, block, name) def magic_paste(self, parameter_s=''): """Paste & execute a pre-formatted code block from clipboard. @@ -606,10 +646,10 @@ def magic_paste(self, parameter_s=''): -------- cpaste: manually paste code into terminal until you mark its end. """ - opts,args = self.parse_options(parameter_s,'rq',mode='string') - par = args.strip() - if opts.has_key('r'): - self._rerun_pasted() + opts, args = self.parse_options(parameter_s, 'rq', mode='string') + name = args.strip() + if 'r' in opts: + rerun_pasted(self.shell) return try: text = self.shell.hooks.clipboard_get() @@ -623,14 +663,14 @@ def magic_paste(self, parameter_s=''): return # By default, echo back to terminal unless quiet mode is requested - if not opts.has_key('q'): + if 'q' not in opts: write = self.shell.write write(self.shell.pycolorize(block)) if not block.endswith('\n'): write('\n') write("## -- End pasted text --\n") - self._execute_block(block, par) + store_or_execute(self.shell, block, name) if sys.platform == 'win32': def magic_cls(self, s): From 825995346945ee7d19a8b0534ff81bcdf0b773db Mon Sep 17 00:00:00 2001 From: Fernando Perez Date: Fri, 25 Nov 2011 22:48:02 -0800 Subject: [PATCH 2/3] Move tests for magics that are terminal-specific to their own file. --- IPython/core/tests/test_magic.py | 123 +--------------- IPython/core/tests/test_magic_terminal.py | 164 ++++++++++++++++++++++ 2 files changed, 165 insertions(+), 122 deletions(-) create mode 100644 IPython/core/tests/test_magic_terminal.py diff --git a/IPython/core/tests/test_magic.py b/IPython/core/tests/test_magic.py index 96def1ac86a..6ce6e6a9fb8 100644 --- a/IPython/core/tests/test_magic.py +++ b/IPython/core/tests/test_magic.py @@ -9,14 +9,9 @@ #----------------------------------------------------------------------------- import os -import sys -import tempfile -import types -from StringIO import StringIO import nose.tools as nt -from IPython.utils.path import get_long_path_name from IPython.testing import decorators as dec from IPython.testing import tools as tt from IPython.utils import py3compat @@ -24,6 +19,7 @@ #----------------------------------------------------------------------------- # Test functions begin #----------------------------------------------------------------------------- + def test_rehashx(): # clear up everything _ip = get_ipython() @@ -202,67 +198,6 @@ def test_numpy_clear_array_undec(): yield (nt.assert_false, 'a' in _ip.user_ns) -# Multiple tests for clipboard pasting -@dec.parametric -def test_paste(): - _ip = get_ipython() - def paste(txt, flags='-q'): - """Paste input text, by default in quiet mode""" - hooks.clipboard_get = lambda : txt - _ip.magic('paste '+flags) - - # Inject fake clipboard hook but save original so we can restore it later - hooks = _ip.hooks - user_ns = _ip.user_ns - original_clip = hooks.clipboard_get - - try: - # Run tests with fake clipboard function - user_ns.pop('x', None) - paste('x=1') - yield nt.assert_equal(user_ns['x'], 1) - - user_ns.pop('x', None) - paste('>>> x=2') - yield nt.assert_equal(user_ns['x'], 2) - - paste(""" - >>> x = [1,2,3] - >>> y = [] - >>> for i in x: - ... y.append(i**2) - ... - """) - yield nt.assert_equal(user_ns['x'], [1,2,3]) - yield nt.assert_equal(user_ns['y'], [1,4,9]) - - # Now, test that paste -r works - user_ns.pop('x', None) - yield nt.assert_false('x' in user_ns) - _ip.magic('paste -r') - yield nt.assert_equal(user_ns['x'], [1,2,3]) - - # Also test paste echoing, by temporarily faking the writer - w = StringIO() - writer = _ip.write - _ip.write = w.write - code = """ - a = 100 - b = 200""" - try: - paste(code,'') - out = w.getvalue() - finally: - _ip.write = writer - yield nt.assert_equal(user_ns['a'], 100) - yield nt.assert_equal(user_ns['b'], 200) - yield nt.assert_equal(out, code+"\n## -- End pasted text --\n") - - finally: - # Restore original hook - hooks.clipboard_get = original_clip - - def test_time(): _ip.magic('time None') @@ -317,62 +252,6 @@ def test_dirops(): os.chdir(startdir) -def check_cpaste(code, should_fail=False): - """Execute code via 'cpaste' and ensure it was executed, unless - should_fail is set. - """ - _ip.user_ns['code_ran'] = False - - src = StringIO() - if not hasattr(src, 'encoding'): - # IPython expects stdin to have an encoding attribute - src.encoding = None - src.write('\n') - src.write(code) - src.write('\n--\n') - src.seek(0) - - stdin_save = sys.stdin - sys.stdin = src - - try: - context = tt.AssertPrints if should_fail else tt.AssertNotPrints - with context("Traceback (most recent call last)"): - _ip.magic('cpaste') - - if not should_fail: - assert _ip.user_ns['code_ran'] - finally: - sys.stdin = stdin_save - - -def test_cpaste(): - """Test cpaste magic""" - - def run(): - """Marker function: sets a flag when executed. - """ - _ip.user_ns['code_ran'] = True - return 'run' # return string so '+ run()' doesn't result in success - - tests = {'pass': ["run()", - "In [1]: run()", - "In [1]: if 1:\n ...: run()", - ">>> run()", - " >>> run()", - ], - - 'fail': ["1 + run()", - "++ run()"]} - - _ip.user_ns['run'] = run - - for code in tests['pass']: - check_cpaste(code) - - for code in tests['fail']: - check_cpaste(code, should_fail=True) - def test_xmode(): # Calling xmode three times should be a no-op xmode = _ip.InteractiveTB.mode diff --git a/IPython/core/tests/test_magic_terminal.py b/IPython/core/tests/test_magic_terminal.py new file mode 100644 index 00000000000..6b58075702a --- /dev/null +++ b/IPython/core/tests/test_magic_terminal.py @@ -0,0 +1,164 @@ +"""Tests for various magic functions specific to the terminal frontend. + +Needs to be run by nose (to make ipython session available). +""" +from __future__ import absolute_import + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +import sys +from StringIO import StringIO + +import nose.tools as nt + +from IPython.testing import decorators as dec +from IPython.testing import tools as tt + +#----------------------------------------------------------------------------- +# Test functions begin +#----------------------------------------------------------------------------- + +def check_cpaste(code, should_fail=False): + """Execute code via 'cpaste' and ensure it was executed, unless + should_fail is set. + """ + _ip.user_ns['code_ran'] = False + + src = StringIO() + if not hasattr(src, 'encoding'): + # IPython expects stdin to have an encoding attribute + src.encoding = None + src.write('\n') + src.write(code) + src.write('\n--\n') + src.seek(0) + + stdin_save = sys.stdin + sys.stdin = src + + try: + context = tt.AssertPrints if should_fail else tt.AssertNotPrints + with context("Traceback (most recent call last)"): + _ip.magic('cpaste') + + if not should_fail: + assert _ip.user_ns['code_ran'] + finally: + sys.stdin = stdin_save + + +def test_cpaste(): + """Test cpaste magic""" + + def run(): + """Marker function: sets a flag when executed. + """ + _ip.user_ns['code_ran'] = True + return 'run' # return string so '+ run()' doesn't result in success + + tests = {'pass': ["run()", + "In [1]: run()", + "In [1]: if 1:\n ...: run()", + "> > > run()", + ">>> run()", + " >>> run()", + ], + + 'fail': ["1 + run()", + "++ run()"]} + + _ip.user_ns['run'] = run + + for code in tests['pass']: + check_cpaste(code) + + for code in tests['fail']: + check_cpaste(code, should_fail=True) + + +# Multiple tests for clipboard pasting +def test_paste(): + _ip = get_ipython() + + def paste(txt, flags='-q'): + """Paste input text, by default in quiet mode""" + hooks.clipboard_get = lambda : txt + _ip.magic('paste '+flags) + + # Inject fake clipboard hook but save original so we can restore it later + hooks = _ip.hooks + user_ns = _ip.user_ns + original_clip = hooks.clipboard_get + + try: + # Run tests with fake clipboard function + user_ns.pop('x', None) + paste('x=1') + nt.assert_equal(user_ns['x'], 1) + + user_ns.pop('x', None) + paste('>>> x=2') + nt.assert_equal(user_ns['x'], 2) + + paste(""" + >>> x = [1,2,3] + >>> y = [] + >>> for i in x: + ... y.append(i**2) + ... + """) + nt.assert_equal(user_ns['x'], [1,2,3]) + nt.assert_equal(user_ns['y'], [1,4,9]) + + # Now, test that paste -r works + user_ns.pop('x', None) + nt.assert_false('x' in user_ns) + _ip.magic('paste -r') + nt.assert_equal(user_ns['x'], [1,2,3]) + + # Test pasting of email-quoted contents + paste(""" + >> def foo(x): + >> return x + 1 + >> x = foo(1.1) + """) + nt.assert_equal(user_ns['x'], 2.1) + + # Email again; some programs add a space also at each quoting level + paste(""" + > > def foo(x): + > > return x + 1 + > > x = foo(2.1) + """) + nt.assert_equal(user_ns['x'], 3.1) + + # Email quoting of interactive input + paste(""" + >> >>> def f(x): + >> ... return x+1 + >> ... + >> >>> x = f(2.5) + """) + nt.assert_equal(user_ns['x'], 3.5) + + # Also test paste echoing, by temporarily faking the writer + w = StringIO() + writer = _ip.write + _ip.write = w.write + code = """ + a = 100 + b = 200""" + try: + paste(code,'') + out = w.getvalue() + finally: + _ip.write = writer + nt.assert_equal(user_ns['a'], 100) + nt.assert_equal(user_ns['b'], 200) + nt.assert_equal(out, code+"\n## -- End pasted text --\n") + + finally: + # Restore original hook + hooks.clipboard_get = original_clip From 84830ec7adfa812434aa0baeeb2d47d7736aad3a Mon Sep 17 00:00:00 2001 From: Fernando Perez Date: Fri, 25 Nov 2011 22:48:37 -0800 Subject: [PATCH 3/3] Add tests for email quote stripping. Also, clarify why certain utilities are kept as standalone functions instead of making them methods. --- IPython/frontend/terminal/interactiveshell.py | 51 ++++++++++++++----- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/IPython/frontend/terminal/interactiveshell.py b/IPython/frontend/terminal/interactiveshell.py index 42ef31ea9b4..414404f822e 100644 --- a/IPython/frontend/terminal/interactiveshell.py +++ b/IPython/frontend/terminal/interactiveshell.py @@ -71,30 +71,55 @@ def get_pasted_lines(sentinel, input=raw_input): return +def strip_email_quotes(raw_lines): + """ Strip email quotation marks at the beginning of each line. + + We don't do any more input transofrmations here because the main shell's + prefiltering handles other cases. + """ + lines = [re.sub(r'^\s*(\s?>)+', '', l) for l in raw_lines] + return '\n'.join(lines) + '\n' + + +# These two functions are needed by the %paste/%cpaste magics. In practice +# they are basically methods (they take the shell as their first argument), but +# we leave them as standalone functions because eventually the magics +# themselves will become separate objects altogether. At that point, the +# magics will have access to the shell object, and these functions can be made +# methods of the magic object, but not of the shell. + def store_or_execute(shell, block, name): """ Execute a block, or store it in a variable, per the user's request. """ + # Dedent and prefilter so what we store matches what is executed by + # run_cell. + b = shell.prefilter(textwrap.dedent(block)) + if name: # If storing it for further editing, run the prefilter on it - shell.user_ns[name] = SList(shell.prefilter(block).splitlines()) + shell.user_ns[name] = SList(b.splitlines()) print "Block assigned to '%s'" % name else: - # For execution we just dedent it, as all other filtering is - # automatically applied by run_cell - b = textwrap.dedent(block) shell.user_ns['pasted_block'] = b shell.run_cell(b) -def rerun_pasted(shell): +def rerun_pasted(shell, name='pasted_block'): """ Rerun a previously pasted command. """ - b = shell.user_ns.get('pasted_block') + b = shell.user_ns.get(name) + + # Sanity checks if b is None: raise UsageError('No previous pasted block available') + if not isinstance(b, basestring): + raise UsageError( + "Variable 'pasted_block' is not a string, can't execute") + print "Re-executing '%s...' (%d chars)"% (b.split('\n',1)[0], len(b)) shell.run_cell(b) + #----------------------------------------------------------------------------- # Main class #----------------------------------------------------------------------------- @@ -605,14 +630,13 @@ def magic_cpaste(self, parameter_s=''): Hello world! """ - opts, args = self.parse_options(parameter_s, 'rs:', mode='string') - name = args.strip() + opts, name = self.parse_options(parameter_s, 'rs:', mode='string') if 'r' in opts: rerun_pasted(self.shell) return sentinel = opts.get('s', '--') - block = '\n'.join(get_pasted_lines(sentinel)) + block = strip_email_quotes(get_pasted_lines(sentinel)) store_or_execute(self.shell, block, name) def magic_paste(self, parameter_s=''): @@ -646,14 +670,13 @@ def magic_paste(self, parameter_s=''): -------- cpaste: manually paste code into terminal until you mark its end. """ - opts, args = self.parse_options(parameter_s, 'rq', mode='string') - name = args.strip() + opts, name = self.parse_options(parameter_s, 'rq', mode='string') if 'r' in opts: rerun_pasted(self.shell) return try: text = self.shell.hooks.clipboard_get() - block = self._strip_pasted_lines_for_code(text.splitlines()) + block = strip_email_quotes(text.splitlines()) except TryNext as clipboard_exc: message = getattr(clipboard_exc, 'args') if message: @@ -672,6 +695,7 @@ def magic_paste(self, parameter_s=''): store_or_execute(self.shell, block, name) + # Class-level: add a '%cls' magic only on Windows if sys.platform == 'win32': def magic_cls(self, s): """Clear screen. @@ -680,7 +704,8 @@ def magic_cls(self, s): def showindentationerror(self): super(TerminalInteractiveShell, self).showindentationerror() - print("If you want to paste code into IPython, try the %paste magic function.") + print("If you want to paste code into IPython, try the " + "%paste and %cpaste magic functions.") InteractiveShellABC.register(TerminalInteractiveShell)