Skip to content

Commit

Permalink
Merge pull request #2447 from takluyver/ipep2-transformer
Browse files Browse the repository at this point in the history 
Stateful line transformers
  • Loading branch information
@takluyver
takluyver committed Apr 11, 2013
2 parents a4d5e24 + 4ed49bf commit 331b90a
Show file tree
Hide file tree
Showing 14 changed files with 2,178 additions and 586 deletions.
473 changes: 139 additions & 334 deletions IPython/core/inputsplitter.py
View file

Large diffs are not rendered by default.

436 changes: 436 additions & 0 deletions IPython/core/inputtransformer.py
View file

Large diffs are not rendered by default.

7 changes: 0 additions & 7 deletions IPython/core/interactiveshell.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2581,13 +2581,6 @@ def run_cell(self, raw_cell, store_history=False, silent=False, shell_futures=Tr

self.input_splitter.push(raw_cell)

# Check for cell magics, which leave state behind. This interface is
# ugly, we need to do something cleaner later... Now the logic is
# simply that the input_splitter remembers if there was a cell magic,
# and in that case we grab the cell body.
if self.input_splitter.cell_magic_parts:
self._current_cell_magic_body = \
''.join(self.input_splitter.cell_magic_parts)
cell = self.input_splitter.source_reset()

# Our own compiler remembers the __future__ environment. If we want to
Expand Down
253 changes: 17 additions & 236 deletions IPython/core/tests/test_inputsplitter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@

# Our own
from IPython.core import inputsplitter as isp
from IPython.core.tests.test_inputtransformer import syntax, syntax_ml
from IPython.testing import tools as tt
from IPython.utils import py3compat

Expand Down Expand Up @@ -108,18 +109,6 @@ def test_remove_comments():
]
tt.check_pairs(isp.remove_comments, tests)

def test_has_comment():
tests = [('text', False),
('text #comment', True),
('text #comment\n', True),
('#comment', True),
('#comment\n', True),
('a = "#string"', False),
('a = "#string" # comment', True),
('a #comment not "string"', True),
]
tt.check_pairs(isp.has_comment, tests)


def test_get_input_encoding():
encoding = isp.get_input_encoding()
Expand Down Expand Up @@ -281,6 +270,7 @@ def test_push2(self):
isp = self.isp
self.assertFalse(isp.push('if 1:'))
for line in [' x=1', '# a comment', ' y=2']:
print(line)
self.assertTrue(isp.push(line))

def test_push3(self):
Expand Down Expand Up @@ -427,216 +417,7 @@ def test_LineInfo():
linfo = isp.LineInfo(' %cd /home')
nt.assert_equal(str(linfo), 'LineInfo [ |%|cd|/home]')

# Transformer tests
def transform_checker(tests, func):
"""Utility to loop over test inputs"""
for inp, tr in tests:
nt.assert_equal(func(inp), tr)

# Data for all the syntax tests in the form of lists of pairs of
# raw/transformed input. We store it here as a global dict so that we can use
# it both within single-function tests and also to validate the behavior of the
# larger objects

syntax = \
dict(assign_system =
[(i,py3compat.u_format(o)) for i,o in \
[(u'a =! ls', "a = get_ipython().getoutput({u}'ls')"),
(u'b = !ls', "b = get_ipython().getoutput({u}'ls')"),
('x=1', 'x=1'), # normal input is unmodified
(' ',' '), # blank lines are kept intact
]],

assign_magic =
[(i,py3compat.u_format(o)) for i,o in \
[(u'a =% who', "a = get_ipython().magic({u}'who')"),
(u'b = %who', "b = get_ipython().magic({u}'who')"),
('x=1', 'x=1'), # normal input is unmodified
(' ',' '), # blank lines are kept intact
]],

classic_prompt =
[('>>> x=1', 'x=1'),
('x=1', 'x=1'), # normal input is unmodified
(' ', ' '), # blank lines are kept intact
('... ', ''), # continuation prompts
],

ipy_prompt =
[('In [1]: x=1', 'x=1'),
('x=1', 'x=1'), # normal input is unmodified
(' ',' '), # blank lines are kept intact
(' ....: ', ''), # continuation prompts
],

# Tests for the escape transformer to leave normal code alone
escaped_noesc =
[ (' ', ' '),
('x=1', 'x=1'),
],

# System calls
escaped_shell =
[(i,py3compat.u_format(o)) for i,o in \
[ (u'!ls', "get_ipython().system({u}'ls')"),
# Double-escape shell, this means to capture the output of the
# subprocess and return it
(u'!!ls', "get_ipython().getoutput({u}'ls')"),
]],

# Help/object info
escaped_help =
[(i,py3compat.u_format(o)) for i,o in \
[ (u'?', 'get_ipython().show_usage()'),
(u'?x1', "get_ipython().magic({u}'pinfo x1')"),
(u'??x2', "get_ipython().magic({u}'pinfo2 x2')"),
(u'?a.*s', "get_ipython().magic({u}'psearch a.*s')"),
(u'?%hist1', "get_ipython().magic({u}'pinfo %hist1')"),
(u'?%%hist2', "get_ipython().magic({u}'pinfo %%hist2')"),
(u'?abc = qwe', "get_ipython().magic({u}'pinfo abc')"),
]],

end_help =
[(i,py3compat.u_format(o)) for i,o in \
[ (u'x3?', "get_ipython().magic({u}'pinfo x3')"),
(u'x4??', "get_ipython().magic({u}'pinfo2 x4')"),
(u'%hist1?', "get_ipython().magic({u}'pinfo %hist1')"),
(u'%hist2??', "get_ipython().magic({u}'pinfo2 %hist2')"),
(u'%%hist3?', "get_ipython().magic({u}'pinfo %%hist3')"),
(u'%%hist4??', "get_ipython().magic({u}'pinfo2 %%hist4')"),
(u'f*?', "get_ipython().magic({u}'psearch f*')"),
(u'ax.*aspe*?', "get_ipython().magic({u}'psearch ax.*aspe*')"),
(u'a = abc?', "get_ipython().set_next_input({u}'a = abc');"
"get_ipython().magic({u}'pinfo abc')"),
(u'a = abc.qe??', "get_ipython().set_next_input({u}'a = abc.qe');"
"get_ipython().magic({u}'pinfo2 abc.qe')"),
(u'a = *.items?', "get_ipython().set_next_input({u}'a = *.items');"
"get_ipython().magic({u}'psearch *.items')"),
(u'plot(a?', "get_ipython().set_next_input({u}'plot(a');"
"get_ipython().magic({u}'pinfo a')"),
(u'a*2 #comment?', 'a*2 #comment?'),
]],

# Explicit magic calls
escaped_magic =
[(i,py3compat.u_format(o)) for i,o in \
[ (u'%cd', "get_ipython().magic({u}'cd')"),
(u'%cd /home', "get_ipython().magic({u}'cd /home')"),
# Backslashes need to be escaped.
(u'%cd C:\\User', "get_ipython().magic({u}'cd C:\\\\User')"),
(u' %magic', " get_ipython().magic({u}'magic')"),
]],

# Quoting with separate arguments
escaped_quote =
[ (',f', 'f("")'),
(',f x', 'f("x")'),
(' ,f y', ' f("y")'),
(',f a b', 'f("a", "b")'),
],

# Quoting with single argument
escaped_quote2 =
[ (';f', 'f("")'),
(';f x', 'f("x")'),
(' ;f y', ' f("y")'),
(';f a b', 'f("a b")'),
],

# Simply apply parens
escaped_paren =
[ ('/f', 'f()'),
('/f x', 'f(x)'),
(' /f y', ' f(y)'),
('/f a b', 'f(a, b)'),
],

# Check that we transform prompts before other transforms
mixed =
[(i,py3compat.u_format(o)) for i,o in \
[ (u'In [1]: %lsmagic', "get_ipython().magic({u}'lsmagic')"),
(u'>>> %lsmagic', "get_ipython().magic({u}'lsmagic')"),
(u'In [2]: !ls', "get_ipython().system({u}'ls')"),
(u'In [3]: abs?', "get_ipython().magic({u}'pinfo abs')"),
(u'In [4]: b = %who', "b = get_ipython().magic({u}'who')"),
]],
)

# multiline syntax examples. Each of these should be a list of lists, with
# each entry itself having pairs of raw/transformed input. The union (with
# '\n'.join() of the transformed inputs is what the splitter should produce
# when fed the raw lines one at a time via push.
syntax_ml = \
dict(classic_prompt =
[ [('>>> for i in range(10):','for i in range(10):'),
('... print i',' print i'),
('... ', ''),
],
],

ipy_prompt =
[ [('In [24]: for i in range(10):','for i in range(10):'),
(' ....: print i',' print i'),
(' ....: ', ''),
],
],

multiline_datastructure =
[ [('>>> a = [1,','a = [1,'),
('... 2]','2]'),
],
],
)


def test_assign_system():
tt.check_pairs(isp.transform_assign_system, syntax['assign_system'])


def test_assign_magic():
tt.check_pairs(isp.transform_assign_magic, syntax['assign_magic'])


def test_classic_prompt():
transform_checker(syntax['classic_prompt'], isp.transform_classic_prompt)
for example in syntax_ml['classic_prompt']:
transform_checker(example, isp.transform_classic_prompt)


def test_ipy_prompt():
transform_checker(syntax['ipy_prompt'], isp.transform_ipy_prompt)
for example in syntax_ml['ipy_prompt']:
transform_checker(example, isp.transform_ipy_prompt)

def test_end_help():
tt.check_pairs(isp.transform_help_end, syntax['end_help'])

def test_escaped_noesc():
tt.check_pairs(isp.transform_escaped, syntax['escaped_noesc'])


def test_escaped_shell():
tt.check_pairs(isp.transform_escaped, syntax['escaped_shell'])


def test_escaped_help():
tt.check_pairs(isp.transform_escaped, syntax['escaped_help'])


def test_escaped_magic():
tt.check_pairs(isp.transform_escaped, syntax['escaped_magic'])


def test_escaped_quote():
tt.check_pairs(isp.transform_escaped, syntax['escaped_quote'])


def test_escaped_quote2():
tt.check_pairs(isp.transform_escaped, syntax['escaped_quote2'])


def test_escaped_paren():
tt.check_pairs(isp.transform_escaped, syntax['escaped_paren'])


class IPythonInputTestCase(InputSplitterTestCase):
Expand Down Expand Up @@ -669,13 +450,16 @@ def test_syntax(self):
def test_syntax_multiline(self):
isp = self.isp
for example in syntax_ml.itervalues():
out_t_parts = []
raw_parts = []
for line_pairs in example:
out_t_parts = []
raw_parts = []
for lraw, out_t_part in line_pairs:
isp.push(lraw)
out_t_parts.append(out_t_part)
raw_parts.append(lraw)
if out_t_part is not None:
out_t_parts.append(out_t_part)

if lraw is not None:
isp.push(lraw)
raw_parts.append(lraw)

out, out_raw = isp.source_raw_reset()
out_t = '\n'.join(out_t_parts).rstrip()
Expand All @@ -698,12 +482,10 @@ def test_syntax_multiline(self):
raw_parts = []
out_t_parts = []
for line_pairs in example:
for raw, out_t_part in line_pairs:
raw_parts.append(raw)
out_t_parts.append(out_t_part)
raw_parts, out_t_parts = zip(*line_pairs)

raw = '\n'.join(raw_parts)
out_t = '\n'.join(out_t_parts)
raw = '\n'.join(r for r in raw_parts if r is not None)
out_t = '\n'.join(o for o in out_t_parts if o is not None)

isp.push(raw)
out, out_raw = isp.source_raw_reset()
Expand All @@ -717,8 +499,8 @@ def test_syntax_multiline_cell(self):

out_t_parts = []
for line_pairs in example:
raw = '\n'.join(r for r, _ in line_pairs)
out_t = '\n'.join(t for _,t in line_pairs)
raw = '\n'.join(r for r, _ in line_pairs if r is not None)
out_t = '\n'.join(t for _,t in line_pairs if t is not None)
out = isp.transform_cell(raw)
# Match ignoring trailing whitespace
self.assertEqual(out.rstrip(), out_t.rstrip())
Expand Down Expand Up @@ -803,9 +585,8 @@ def test_whole_cell(self):
src = "%%cellm line\nbody\n"
sp = self.sp
sp.push(src)
nt.assert_equal(sp.cell_magic_parts, ['body\n'])
out = sp.source
ref = u"get_ipython()._run_cached_cell_magic({u}'cellm', {u}'line')\n"
out = sp.source_reset()
ref = u"get_ipython().run_cell_magic({u}'cellm', {u}'line', {u}'body')\n"
nt.assert_equal(out, py3compat.u_format(ref))

def tearDown(self):
Expand Down
Loading

0 comments on commit 331b90a

Please sign in to comment.