Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fixes and improvements to the input splitter #2110

Merged
merged 4 commits into from Jul 8, 2012
Merged

Fixes and improvements to the input splitter #2110

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9bd0c90
Line continuations now terminate after one blank line (#2108)
asmeurer Jul 8, 2012
df408e2
`yield`, `break`, and `continue` automatically dedent
asmeurer Jul 8, 2012
b4c0a20
Make it more clear in the README how exactly to build the docs
asmeurer Jul 8, 2012
91f6c75
Don't dedent on yield, as yield doesn't always end a block
asmeurer Jul 8, 2012
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
52 changes: 27 additions & 25 deletions IPython/core/inputsplitter.py
View file
Open in desktop
Expand Up @@ -115,7 +115,9 @@
r'^\s+raise\([^\)]*\).*$', # wacky raise with immediate open paren r'^\s+raise\([^\)]*\).*$', # wacky raise with immediate open paren
r'^\s+return(\s.*)?$', # normal return (+ space + other stuff, maybe) r'^\s+return(\s.*)?$', # normal return (+ space + other stuff, maybe)
r'^\s+return\([^\)]*\).*$', # wacky return with immediate open paren r'^\s+return\([^\)]*\).*$', # wacky return with immediate open paren
r'^\s+pass\s*$' # pass (optionally followed by trailing spaces) r'^\s+pass\s*$', # pass (optionally followed by trailing spaces)
r'^\s+break\s*$', # break (optionally followed by trailing spaces)
r'^\s+continue\s*$', # continue (optionally followed by trailing spaces)
])) ]))
ini_spaces_re = re.compile(r'^([ \t\r\f\v]+)') ini_spaces_re = re.compile(r'^([ \t\r\f\v]+)')


Expand Down Expand Up @@ -202,17 +204,17 @@ def remove_comments(src):
""" """


Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In general we prefer not to have too much of these whitespace-only changes because they make the diffs longer to read, and we don't have a super-strict whitespace policy, so different contributors have their editors set differently.

In this case there's not that much to be a bother, as fortunately this PR only touches few files, so we'll let it pass. But if you ever make a PR that touches a ton of file, please refrain from adding whitespace-only changes all over, as it will make review harder.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry about that.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no worries, no biggie. Just a tip for the future.

return re.sub('#.*', '', src) return re.sub('#.*', '', src)

def has_comment(src): def has_comment(src):
"""Indicate whether an input line has (i.e. ends in, or is) a comment. """Indicate whether an input line has (i.e. ends in, or is) a comment.

This uses tokenize, so it can distinguish comments from # inside strings. This uses tokenize, so it can distinguish comments from # inside strings.

Parameters Parameters
---------- ----------
src : string src : string
A single line input string. A single line input string.

Returns Returns
------- -------
Boolean: True if source has a comment. Boolean: True if source has a comment.
Expand Down Expand Up @@ -280,9 +282,9 @@ class InputSplitter(object):
code = None code = None
# Input mode # Input mode
input_mode = 'line' input_mode = 'line'

# Private attributes # Private attributes

# List with lines of input accumulated so far # List with lines of input accumulated so far
_buffer = None _buffer = None
# Command compiler # Command compiler
Expand All @@ -291,7 +293,7 @@ class InputSplitter(object):
_full_dedent = False _full_dedent = False
# Boolean indicating whether the current block is complete # Boolean indicating whether the current block is complete
_is_complete = None _is_complete = None

def __init__(self, input_mode=None): def __init__(self, input_mode=None):
"""Create a new InputSplitter instance. """Create a new InputSplitter instance.


Expand Down Expand Up @@ -348,7 +350,7 @@ def push(self, lines):
---------- ----------
lines : string lines : string
One or more lines of Python input. One or more lines of Python input.

Returns Returns
------- -------
is_complete : boolean is_complete : boolean
Expand All @@ -359,7 +361,7 @@ def push(self, lines):
""" """
if self.input_mode == 'cell': if self.input_mode == 'cell':
self.reset() self.reset()

self._store(lines) self._store(lines)
source = self.source source = self.source


Expand All @@ -369,7 +371,7 @@ def push(self, lines):
self.code, self._is_complete = None, None self.code, self._is_complete = None, None


# Honor termination lines properly # Honor termination lines properly
if source.rstrip().endswith('\\'): if source.endswith('\\\n'):
return False return False


self._update_indent(lines) self._update_indent(lines)
Expand Down Expand Up @@ -400,11 +402,11 @@ def push_accepts_more(self):
SyntaxError is raised, or *all* of the following are true: SyntaxError is raised, or *all* of the following are true:


1. The input compiles to a complete statement. 1. The input compiles to a complete statement.

2. The indentation level is flush-left (because if we are indented, 2. The indentation level is flush-left (because if we are indented,
like inside a function definition or for loop, we need to keep like inside a function definition or for loop, we need to keep
reading new input). reading new input).

3. There is one extra line consisting only of whitespace. 3. There is one extra line consisting only of whitespace.


Because of condition #3, this method should be used only by Because of condition #3, this method should be used only by
Expand Down Expand Up @@ -464,7 +466,7 @@ def _find_indent(self, line):
---------- ----------
line : str line : str
A single new line of non-whitespace, non-comment Python input. A single new line of non-whitespace, non-comment Python input.

Returns Returns
------- -------
indent_spaces : int indent_spaces : int
Expand All @@ -476,7 +478,7 @@ def _find_indent(self, line):
""" """
indent_spaces = self.indent_spaces indent_spaces = self.indent_spaces
full_dedent = self._full_dedent full_dedent = self._full_dedent

inisp = num_ini_spaces(line) inisp = num_ini_spaces(line)
if inisp < indent_spaces: if inisp < indent_spaces:
indent_spaces = inisp indent_spaces = inisp
Expand All @@ -495,9 +497,9 @@ def _find_indent(self, line):
if indent_spaces < 0: if indent_spaces < 0:
indent_spaces = 0 indent_spaces = 0
#print 'safety' # dbg #print 'safety' # dbg

return indent_spaces, full_dedent return indent_spaces, full_dedent

def _update_indent(self, lines): def _update_indent(self, lines):
for line in remove_comments(lines).splitlines(): for line in remove_comments(lines).splitlines():
if line and not line.isspace(): if line and not line.isspace():
Expand All @@ -508,10 +510,10 @@ def _store(self, lines, buffer=None, store='source'):


If input lines are not newline-terminated, a newline is automatically If input lines are not newline-terminated, a newline is automatically
appended.""" appended."""

if buffer is None: if buffer is None:
buffer = self._buffer buffer = self._buffer

if lines.endswith('\n'): if lines.endswith('\n'):
buffer.append(lines) buffer.append(lines)
else: else:
Expand Down Expand Up @@ -627,10 +629,10 @@ def transform_help_end(line):
target = m.group(1) target = m.group(1)
esc = m.group(3) esc = m.group(3)
lspace = _initial_space_re.match(line).group(0) lspace = _initial_space_re.match(line).group(0)

# If we're mid-command, put it back on the next prompt for the user. # If we're mid-command, put it back on the next prompt for the user.
next_input = line.rstrip('?') if line.strip() != m.group(0) else None next_input = line.rstrip('?') if line.strip() != m.group(0) else None

return _make_help_call(target, esc, lspace, next_input) return _make_help_call(target, esc, lspace, next_input)




Expand All @@ -647,7 +649,7 @@ def __init__(self):
ESC_QUOTE2 : self._tr_quote2, ESC_QUOTE2 : self._tr_quote2,
ESC_PAREN : self._tr_paren } ESC_PAREN : self._tr_paren }
self.tr = tr self.tr = tr

# Support for syntax transformations that use explicit escapes typed by the # Support for syntax transformations that use explicit escapes typed by the
# user at the beginning of a line # user at the beginning of a line
@staticmethod @staticmethod
Expand All @@ -668,7 +670,7 @@ def _tr_help(line_info):
# A naked help line should just fire the intro help screen # A naked help line should just fire the intro help screen
if not line_info.line[1:]: if not line_info.line[1:]:
return 'get_ipython().show_usage()' return 'get_ipython().show_usage()'

return _make_help_call(line_info.ifun, line_info.esc, line_info.pre) return _make_help_call(line_info.ifun, line_info.esc, line_info.pre)


@staticmethod @staticmethod
Expand Down Expand Up @@ -745,7 +747,7 @@ def __init__(self, input_mode=None):
super(IPythonInputSplitter, self).__init__(input_mode) super(IPythonInputSplitter, self).__init__(input_mode)
self._buffer_raw = [] self._buffer_raw = []
self._validate = True self._validate = True

def reset(self): def reset(self):
"""Reset the input buffer and associated state.""" """Reset the input buffer and associated state."""
super(IPythonInputSplitter, self).reset() super(IPythonInputSplitter, self).reset()
Expand Down Expand Up @@ -897,7 +899,7 @@ def push(self, lines):
# that this must be done *after* the reset() call that would otherwise # that this must be done *after* the reset() call that would otherwise
# flush the buffer. # flush the buffer.
self._store(lines, self._buffer_raw, 'source_raw') self._store(lines, self._buffer_raw, 'source_raw')

try: try:
push = super(IPythonInputSplitter, self).push push = super(IPythonInputSplitter, self).push
buf = self._buffer buf = self._buffer
Expand Down
40 changes: 38 additions & 2 deletions IPython/core/tests/test_inputsplitter.py
View file
Open in desktop
Expand Up @@ -227,6 +227,26 @@ def test_dedent_pass(self):
isp.push('if 1:\n pass ') isp.push('if 1:\n pass ')
self.assertEqual(isp.indent_spaces, 0) self.assertEqual(isp.indent_spaces, 0)


def test_dedent_break(self):
isp = self.isp # shorthand
# should NOT cause dedent
isp.push('while 1:\n breaks = 5')
self.assertEqual(isp.indent_spaces, 4)
isp.push('while 1:\n break')
self.assertEqual(isp.indent_spaces, 0)
isp.push('while 1:\n break ')
self.assertEqual(isp.indent_spaces, 0)

def test_dedent_continue(self):
isp = self.isp # shorthand
# should NOT cause dedent
isp.push('while 1:\n continues = 5')
self.assertEqual(isp.indent_spaces, 4)
isp.push('while 1:\n continue')
self.assertEqual(isp.indent_spaces, 0)
isp.push('while 1:\n continue ')
self.assertEqual(isp.indent_spaces, 0)

def test_dedent_raise(self): def test_dedent_raise(self):
isp = self.isp # shorthand isp = self.isp # shorthand
# should NOT cause dedent # should NOT cause dedent
Expand Down Expand Up @@ -351,6 +371,22 @@ def test_unicode(self):
self.isp.push(u'\xc3\xa9') self.isp.push(u'\xc3\xa9')
self.isp.push(u"u'\xc3\xa9'") self.isp.push(u"u'\xc3\xa9'")


def test_line_continuation(self):
""" Test issue #2108."""
isp = self.isp
# A blank line after a line continuation should not accept more
isp.push("1 \\\n\n")
self.assertFalse(isp.push_accepts_more())
# Whitespace after a \ is a SyntaxError. The only way to test that
# here is to test that push doesn't accept more (as with
# test_syntax_error() above).
isp.push(r"1 \ ")
self.assertFalse(isp.push_accepts_more())
# Even if the line is continuable (c.f. the regular Python
# interpreter)
isp.push(r"(1 \ ")
self.assertFalse(isp.push_accepts_more())

class InteractiveLoopTestCase(unittest.TestCase): class InteractiveLoopTestCase(unittest.TestCase):
"""Tests for an interactive loop like a python shell. """Tests for an interactive loop like a python shell.
""" """
Expand Down Expand Up @@ -678,7 +714,7 @@ def test_syntax_multiline(self):
def test_syntax_multiline_cell(self): def test_syntax_multiline_cell(self):
isp = self.isp isp = self.isp
for example in syntax_ml.itervalues(): for example in syntax_ml.itervalues():

out_t_parts = [] out_t_parts = []
for line_pairs in example: for line_pairs in example:
raw = '\n'.join(r for r, _ in line_pairs) raw = '\n'.join(r for r, _ in line_pairs)
Expand Down Expand Up @@ -762,7 +798,7 @@ def test_last_two_blanks():




class CellMagicsCommon(object): class CellMagicsCommon(object):

def test_whole_cell(self): def test_whole_cell(self):
src = "%%cellm line\nbody\n" src = "%%cellm line\nbody\n"
sp = self.sp sp = self.sp
Expand Down
4 changes: 2 additions & 2 deletions README.rst
View file
Open in desktop
Expand Up @@ -9,7 +9,7 @@ Welcome to IPython. Our full documentation is available on `our website
<http://ipython.org/documentation.html>`_; if you downloaded a built source <http://ipython.org/documentation.html>`_; if you downloaded a built source
distribution the ``docs/source`` directory contains the plaintext version of distribution the ``docs/source`` directory contains the plaintext version of
these manuals. If you have Sphinx installed, you can build them by typing these manuals. If you have Sphinx installed, you can build them by typing
``make html`` for local browsing. ``cd docs; make html`` for local browsing.




Dependencies and supported Python versions Dependencies and supported Python versions
Expand All @@ -21,7 +21,7 @@ functionality requires extra packages.


Officially, IPython requires Python version 2.6, 2.7, or 3.1 and above. Officially, IPython requires Python version 2.6, 2.7, or 3.1 and above.



Instant running Instant running
=============== ===============


Expand Down