From 5020c5514c7ddd9b3ff0f1946ec6a3efd18269dd Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Wed, 29 Oct 2025 16:27:35 +0300 Subject: [PATCH] gh-140576: Fixed crash produced by lexer in case of dedented zero byte (#140583) --- Lib/test/test_tokenize.py | 1 + .../2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst | 2 ++ Parser/lexer/lexer.c | 3 +++ 3 files changed, 6 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 9aff15eb530845..4a7b02ee05b895 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -3097,6 +3097,7 @@ def get_tokens(string): f'__{ x:d }__'""", + " a\n\x00", ]: with self.subTest(case=case): self.assertRaises(tokenize.TokenError, get_tokens, case) diff --git a/Misc/NEWS.d/next/Core and Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst b/Misc/NEWS.d/next/Core and Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst new file mode 100644 index 00000000000000..2c27525d9f782c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst @@ -0,0 +1,2 @@ +Fixed crash in :func:`tokenize.generate_tokens` in case of +specific incorrect input. Patch by Mikhail Efimov. diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 66a7cbb769aa46..cc89f0b9cc9af0 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -481,6 +481,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t return MAKE_TOKEN(ERRORTOKEN); } } + else if (c == EOF && PyErr_Occurred()) { + return MAKE_TOKEN(ERRORTOKEN); + } else { break; }