From 22577d9371b5d2fff126807becaada500e663426 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 25 Jan 2022 22:33:57 +0000 Subject: [PATCH] [3.9] bpo-46091: Correctly calculate indentation levels for whitespace lines with continuation characters (GH-30130). (GH-30898) (cherry picked from commit a0efc0c1960e2c49e0092694d98395555270914c) Co-authored-by: Pablo Galindo Salgado . (cherry picked from commit 3fc8b74ace033a17346a992f661928ba619e61e8) Co-authored-by: Pablo Galindo Salgado --- Lib/test/test_ast.py | 8 +++- Lib/test/test_eof.py | 7 ++- Lib/test/test_syntax.py | 30 ++++++++++++ Lib/test/test_tokenize.py | 2 +- .../2021-12-16-00-24-00.bpo-46091.rJ_e_e.rst | 2 + Parser/tokenizer.c | 46 +++++++++++++------ 6 files changed, 76 insertions(+), 19 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2021-12-16-00-24-00.bpo-46091.rJ_e_e.rst diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index c3e3be6335340b9..3bb5841e001d1b8 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1005,7 +1005,13 @@ def test_literal_eval_malformed_dict_nodes(self): malformed = ast.Dict(keys=[ast.Constant(1)], values=[ast.Constant(2), ast.Constant(3)]) self.assertRaises(ValueError, ast.literal_eval, malformed) - def test_bad_integer(self): + def test_literal_eval_syntax_errors(self): + with self.assertRaisesRegex(SyntaxError, "unexpected indent"): + ast.literal_eval(r''' + \ + (\ + \ ''') + # issue13436: Bad error message with invalid numeric values body = [ast.ImportFrom(module='time', names=[ast.alias(name='sleep')], diff --git a/Lib/test/test_eof.py b/Lib/test/test_eof.py index 51cbbd8eed664fa..284e6a37facbdaf 100644 --- a/Lib/test/test_eof.py +++ b/Lib/test/test_eof.py @@ -45,20 +45,19 @@ def test_line_continuation_EOF(self): exec('\\') self.assertEqual(str(excinfo.exception), expect) - @unittest.skipIf(not sys.executable, "sys.executable required") def test_line_continuation_EOF_from_file_bpo2180(self): """Ensure tok_nextc() does not add too many ending newlines.""" - with support.temp_dir() as temp_dir: + with os_helper.temp_dir() as temp_dir: file_name = script_helper.make_script(temp_dir, 'foo', '\\') rc, out, err = script_helper.assert_python_failure(file_name) self.assertIn(b'unexpected EOF while parsing', err) - self.assertIn(b'line 2', err) + self.assertIn(b'line 1', err) self.assertIn(b'\\', err) file_name = script_helper.make_script(temp_dir, 'foo', 'y = 6\\') rc, out, err = script_helper.assert_python_failure(file_name) self.assertIn(b'unexpected EOF while parsing', err) - self.assertIn(b'line 2', err) + self.assertIn(b'line 1', err) self.assertIn(b'y = 6\\', err) if __name__ == "__main__": diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 2f151b44a418d3b..7a2380fe584924d 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -923,6 +923,36 @@ def test_empty_line_after_linecont(self): except SyntaxError: self.fail("Empty line after a line continuation character is valid.") + # See issue-46091 + s1 = r"""\ +def fib(n): + \ +'''Print a Fibonacci series up to n.''' + \ +a, b = 0, 1 +""" + s2 = r"""\ +def fib(n): + '''Print a Fibonacci series up to n.''' + a, b = 0, 1 +""" + try: + self.assertEqual(compile(s1, '', 'exec'), compile(s2, '', 'exec')) + except SyntaxError: + self.fail("Indented statement over multiple lines is valid") + + def test_continuation_bad_indentation(self): + # Check that code that breaks indentation across multiple lines raises a syntax error + + code = r"""\ +if x: + y = 1 + \ + foo = 1 + """ + + self.assertRaises(IndentationError, exec, code) + @support.cpython_only def test_nested_named_except_blocks(self): code = "" diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index e05a07fab9fe75a..543f4b8d08f740b 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -5,6 +5,7 @@ NEWLINE) from io import BytesIO, StringIO import unittest +from textwrap import dedent from unittest import TestCase, mock from test.test_grammar import (VALID_UNDERSCORE_LITERALS, INVALID_UNDERSCORE_LITERALS) @@ -44,7 +45,6 @@ def check_tokenize(self, s, expected): # The ENDMARKER and final NEWLINE are omitted. f = BytesIO(s.encode('utf-8')) result = stringify_tokens_from_source(tokenize(f.readline), s) - self.assertEqual(result, [" ENCODING 'utf-8' (0, 0) (0, 0)"] + expected.rstrip().splitlines()) diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-12-16-00-24-00.bpo-46091.rJ_e_e.rst b/Misc/NEWS.d/next/Core and Builtins/2021-12-16-00-24-00.bpo-46091.rJ_e_e.rst new file mode 100644 index 000000000000000..a2eee0f3ebd5156 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-12-16-00-24-00.bpo-46091.rJ_e_e.rst @@ -0,0 +1,2 @@ +Correctly calculate indentation levels for lines with whitespace character +that are ended by line continuation characters. Patch by Pablo Galindo diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 41bfdb853c7a689..003442ba0569e34 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1170,6 +1170,24 @@ tok_decimal_tail(struct tok_state *tok) /* Get next token, after space stripping etc. */ +static inline int +tok_continuation_line(struct tok_state *tok) { + int c = tok_nextc(tok); + if (c != '\n') { + tok->done = E_LINECONT; + return -1; + } + c = tok_nextc(tok); + if (c == EOF) { + tok->done = E_EOF; + tok->cur = tok->inp; + return -1; + } else { + tok_backup(tok, c); + } + return c; +} + static int tok_get(struct tok_state *tok, const char **p_start, const char **p_end) { @@ -1186,6 +1204,7 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) int col = 0; int altcol = 0; tok->atbol = 0; + int cont_line_col = 0; for (;;) { c = tok_nextc(tok); if (c == ' ') { @@ -1198,14 +1217,23 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) else if (c == '\014') {/* Control-L (formfeed) */ col = altcol = 0; /* For Emacs users */ } + else if (c == '\\') { + // Indentation cannot be split over multiple physical lines + // using backslashes. This means that if we found a backslash + // preceded by whitespace, **the first one we find** determines + // the level of indentation of whatever comes next. + cont_line_col = cont_line_col ? cont_line_col : col; + if ((c = tok_continuation_line(tok)) == -1) { + return ERRORTOKEN; + } + } else { break; } } tok_backup(tok, c); - if (c == '#' || c == '\n' || c == '\\') { + if (c == '#' || c == '\n') { /* Lines with only whitespace and/or comments - and/or a line continuation character shouldn't affect the indentation and are not passed to the parser as NEWLINE tokens, except *totally* empty lines in interactive @@ -1226,6 +1254,8 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) may need to skip to the end of a comment */ } if (!blankline && tok->level == 0) { + col = cont_line_col ? cont_line_col : col; + altcol = cont_line_col ? cont_line_col : altcol; if (col == tok->indstack[tok->indent]) { /* No change */ if (altcol != tok->altindstack[tok->indent]) { @@ -1749,19 +1779,9 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) /* Line continuation */ if (c == '\\') { - c = tok_nextc(tok); - if (c != '\n') { - tok->done = E_LINECONT; + if ((c = tok_continuation_line(tok)) == -1) { return ERRORTOKEN; } - c = tok_nextc(tok); - if (c == EOF) { - tok->done = E_EOF; - tok->cur = tok->inp; - return ERRORTOKEN; - } else { - tok_backup(tok, c); - } tok->cont_line = 1; goto again; /* Read next line */ }