From e9b20c5d782b52d961229b7754a704d1b2192b2d Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 6 Jun 2021 00:28:46 +0100 Subject: [PATCH 1/5] bpo-44317: Improve tokenizer errors with more informative locations --- Lib/test/test_exceptions.py | 4 +- .../2021-06-06-00-29-14.bpo-44317.xPPhcZ.rst | 1 + Parser/tokenizer.c | 70 ++++++++++++++----- 3 files changed, 57 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2021-06-06-00-29-14.bpo-44317.xPPhcZ.rst diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 5fb651f4c22e55..f6362fa2748e55 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -224,9 +224,9 @@ def testSyntaxErrorOffset(self): # Errors thrown by tokenizer.c check('(0x+1)', 1, 3) check('x = 0xI', 1, 6) - check('0010 + 2', 1, 4) + check('0010 + 2', 1, 1) check('x = 32e-+4', 1, 8) - check('x = 0o9', 1, 6) + check('x = 0o9', 1, 7) check('\u03b1 = 0xI', 1, 6) check(b'\xce\xb1 = 0xI', 1, 6) check(b'# -*- coding: iso8859-7 -*-\n\xe1 = 0xI', 2, 6, diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-06-00-29-14.bpo-44317.xPPhcZ.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-06-00-29-14.bpo-44317.xPPhcZ.rst new file mode 100644 index 00000000000000..8ac32adf8b5535 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-06-06-00-29-14.bpo-44317.xPPhcZ.rst @@ -0,0 +1 @@ +Improve tokenizer error with improved locations. Patch by Pablo Galindo. diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index a86af9bc0620ca..e2b413715c6d7f 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1067,19 +1067,13 @@ tok_backup(struct tok_state *tok, int c) } } - static int -syntaxerror(struct tok_state *tok, const char *format, ...) +_syntaxerror_range(struct tok_state *tok, const char *format, + Py_ssize_t col_offset, Py_ssize_t end_col_offset, + va_list vargs) { PyObject *errmsg, *errtext, *args; - va_list vargs; -#ifdef HAVE_STDARG_PROTOTYPES - va_start(vargs, format); -#else - va_start(vargs); -#endif errmsg = PyUnicode_FromFormatV(format, vargs); - va_end(vargs); if (!errmsg) { goto error; } @@ -1089,7 +1083,14 @@ syntaxerror(struct tok_state *tok, const char *format, ...) if (!errtext) { goto error; } - int offset = (int)PyUnicode_GET_LENGTH(errtext); + + if (col_offset == 0) { + col_offset = (int)PyUnicode_GET_LENGTH(errtext); + } + if (end_col_offset == 0) { + col_offset = col_offset; + } + Py_ssize_t line_len = strcspn(tok->line_start, "\n"); if (line_len != tok->cur - tok->line_start) { Py_DECREF(errtext); @@ -1100,8 +1101,7 @@ syntaxerror(struct tok_state *tok, const char *format, ...) goto error; } - args = Py_BuildValue("(O(OiiN))", errmsg, - tok->filename, tok->lineno, offset, errtext); + args = Py_BuildValue("(O(OiiNii))", errmsg, tok->filename, tok->lineno, col_offset, errtext, tok->lineno, end_col_offset); if (args) { PyErr_SetObject(PyExc_SyntaxError, args); Py_DECREF(args); @@ -1113,6 +1113,36 @@ syntaxerror(struct tok_state *tok, const char *format, ...) return ERRORTOKEN; } +static int +syntaxerror(struct tok_state *tok, const char *format, ...) { + va_list vargs; +#ifdef HAVE_STDARG_PROTOTYPES + va_start(vargs, format); +#else + va_start(vargs); +#endif + int ret = _syntaxerror_range(tok, format, 0, 0, vargs); + va_end(vargs); + return ret; +} + +static int +syntaxerror_known_range(struct tok_state *tok, + Py_ssize_t col_offset, Py_ssize_t end_col_offset, + const char *format, ...) { + va_list vargs; +#ifdef HAVE_STDARG_PROTOTYPES + va_start(vargs, format); +#else + va_start(vargs); +#endif + int ret = _syntaxerror_range(tok, format, col_offset, end_col_offset, vargs); + va_end(vargs); + return ret; +} + + + static int indenterror(struct tok_state *tok) { @@ -1552,6 +1582,7 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) /* Number */ if (isdigit(c)) { if (c == '0') { + char* number_start = tok->cur; /* Hex, octal or binary -- maybe. */ c = tok_nextc(tok); if (c == 'x' || c == 'X') { @@ -1580,6 +1611,8 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) if (c < '0' || c >= '8') { tok_backup(tok, c); if (isdigit(c)) { + // Move to the actual current token that is incorrect + tok_nextc(tok); return syntaxerror(tok, "invalid digit '%c' in octal literal", c); } @@ -1606,6 +1639,8 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) if (c != '0' && c != '1') { tok_backup(tok, c); if (isdigit(c)) { + // Move to the actual current token that is incorrect + tok_nextc(tok); return syntaxerror(tok, "invalid digit '%c' in binary literal", c); } @@ -1639,6 +1674,7 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) } c = tok_nextc(tok); } + char* zeros_end = tok->cur; if (isdigit(c)) { nonzero = 1; c = tok_decimal_tail(tok); @@ -1659,10 +1695,12 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) else if (nonzero) { /* Old-style octal: now disallowed. */ tok_backup(tok, c); - return syntaxerror(tok, - "leading zeros in decimal integer " - "literals are not permitted; " - "use an 0o prefix for octal integers"); + return syntaxerror_known_range( + tok, number_start - tok->line_start, + zeros_end - tok->line_start, + "leading zeros in decimal integer " + "literals are not permitted; " + "use an 0o prefix for octal integers"); } } } From ec8a7c50a046f13b62d9bc634b26c41a4041a7a4 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 6 Jun 2021 23:16:40 +0100 Subject: [PATCH 2/5] fixup! bpo-44317: Improve tokenizer errors with more informative locations --- Parser/tokenizer.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index e2b413715c6d7f..272222164bd6ca 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1069,7 +1069,7 @@ tok_backup(struct tok_state *tok, int c) static int _syntaxerror_range(struct tok_state *tok, const char *format, - Py_ssize_t col_offset, Py_ssize_t end_col_offset, + int col_offset, int end_col_offset, va_list vargs) { PyObject *errmsg, *errtext, *args; @@ -1101,7 +1101,8 @@ _syntaxerror_range(struct tok_state *tok, const char *format, goto error; } - args = Py_BuildValue("(O(OiiNii))", errmsg, tok->filename, tok->lineno, col_offset, errtext, tok->lineno, end_col_offset); + args = Py_BuildValue("(O(OiiNii))", errmsg, tok->filename, tok->lineno, + col_offset, errtext, tok->lineno, end_col_offset); if (args) { PyErr_SetObject(PyExc_SyntaxError, args); Py_DECREF(args); @@ -1114,7 +1115,8 @@ _syntaxerror_range(struct tok_state *tok, const char *format, } static int -syntaxerror(struct tok_state *tok, const char *format, ...) { +syntaxerror(struct tok_state *tok, const char *format, ...) +{ va_list vargs; #ifdef HAVE_STDARG_PROTOTYPES va_start(vargs, format); @@ -1128,8 +1130,9 @@ syntaxerror(struct tok_state *tok, const char *format, ...) { static int syntaxerror_known_range(struct tok_state *tok, - Py_ssize_t col_offset, Py_ssize_t end_col_offset, - const char *format, ...) { + int col_offset, int end_col_offset, + const char *format, ...) +{ va_list vargs; #ifdef HAVE_STDARG_PROTOTYPES va_start(vargs, format); @@ -1582,7 +1585,7 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) /* Number */ if (isdigit(c)) { if (c == '0') { - char* number_start = tok->cur; + const char* number_start = tok->cur; /* Hex, octal or binary -- maybe. */ c = tok_nextc(tok); if (c == 'x' || c == 'X') { @@ -1609,14 +1612,12 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) c = tok_nextc(tok); } if (c < '0' || c >= '8') { - tok_backup(tok, c); if (isdigit(c)) { - // Move to the actual current token that is incorrect - tok_nextc(tok); return syntaxerror(tok, "invalid digit '%c' in octal literal", c); } else { + tok_backup(tok, c); return syntaxerror(tok, "invalid octal literal"); } } From ffc3279fdf44c388bdb5be4cd2b87b5de22dbb59 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Mon, 7 Jun 2021 10:36:34 +0100 Subject: [PATCH 3/5] Update Parser/tokenizer.c Co-authored-by: Serhiy Storchaka --- Parser/tokenizer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 272222164bd6ca..be38f4171dd08b 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1697,7 +1697,7 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) /* Old-style octal: now disallowed. */ tok_backup(tok, c); return syntaxerror_known_range( - tok, number_start - tok->line_start, + tok, (int)(number_start - tok->line_start), zeros_end - tok->line_start, "leading zeros in decimal integer " "literals are not permitted; " From c66a1f0fdb79dd719842fd45450d7781e35d1c04 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Mon, 7 Jun 2021 10:53:20 +0100 Subject: [PATCH 4/5] fixup! Update Parser/tokenizer.c --- Parser/tokenizer.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index be38f4171dd08b..760352fbb95830 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1084,11 +1084,11 @@ _syntaxerror_range(struct tok_state *tok, const char *format, goto error; } - if (col_offset == 0) { + if (col_offset == -1) { col_offset = (int)PyUnicode_GET_LENGTH(errtext); } - if (end_col_offset == 0) { - col_offset = col_offset; + if (end_col_offset == -1) { + end_col_offset = col_offset; } Py_ssize_t line_len = strcspn(tok->line_start, "\n"); @@ -1123,7 +1123,7 @@ syntaxerror(struct tok_state *tok, const char *format, ...) #else va_start(vargs); #endif - int ret = _syntaxerror_range(tok, format, 0, 0, vargs); + int ret = _syntaxerror_range(tok, format, -1, -1, vargs); va_end(vargs); return ret; } @@ -1585,7 +1585,6 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) /* Number */ if (isdigit(c)) { if (c == '0') { - const char* number_start = tok->cur; /* Hex, octal or binary -- maybe. */ c = tok_nextc(tok); if (c == 'x' || c == 'X') { @@ -1638,14 +1637,12 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) c = tok_nextc(tok); } if (c != '0' && c != '1') { - tok_backup(tok, c); if (isdigit(c)) { - // Move to the actual current token that is incorrect - tok_nextc(tok); return syntaxerror(tok, "invalid digit '%c' in binary literal", c); } else { + tok_backup(tok, c); return syntaxerror(tok, "invalid binary literal"); } } @@ -1697,7 +1694,7 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) /* Old-style octal: now disallowed. */ tok_backup(tok, c); return syntaxerror_known_range( - tok, (int)(number_start - tok->line_start), + tok, (int)(tok->start + 1 - tok->line_start), zeros_end - tok->line_start, "leading zeros in decimal integer " "literals are not permitted; " From 4efe8e34e1bb11df66fa7eaa71e76215c0be661d Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 8 Jun 2021 23:56:06 +0100 Subject: [PATCH 5/5] Update Parser/tokenizer.c --- Parser/tokenizer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 760352fbb95830..14a7aeba4d9333 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1695,7 +1695,7 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) tok_backup(tok, c); return syntaxerror_known_range( tok, (int)(tok->start + 1 - tok->line_start), - zeros_end - tok->line_start, + (int)(zeros_end - tok->line_start), "leading zeros in decimal integer " "literals are not permitted; " "use an 0o prefix for octal integers");