[3.12] gh-104866: Tokenize should emit NEWLINE after exiting block wi…

…th comment (GH-104870) (#104872) gh-104866: Tokenize should emit NEWLINE after exiting block with comment (GH-104870) (cherry picked from commit c90a862) Co-authored-by: Lysandros Nikolaou <lisandrosnik@gmail.com>
python · May 24, 2023 · ddc29c8 · ddc29c8
1 parent 45b88e0
commit ddc29c8
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 3 deletions.
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
@@ -1057,6 +1057,23 @@ async def bar(): pass
     DEDENT     ''            (6, 12) (6, 12)
     """)
 
+    def test_newline_after_parenthesized_block_with_comment(self):
+        self.check_tokenize('''\
+[
+    # A comment here
+    1
+]
+''', """\
+    OP         '['           (1, 0) (1, 1)
+    NL         '\\n'          (1, 1) (1, 2)
+    COMMENT    '# A comment here' (2, 4) (2, 20)
+    NL         '\\n'          (2, 20) (2, 21)
+    NUMBER     '1'           (3, 4) (3, 5)
+    NL         '\\n'          (3, 5) (3, 6)
+    OP         ']'           (4, 0) (4, 1)
+    NEWLINE    '\\n'          (4, 1) (4, 2)
+    """)
+
 class GenerateTokensTest(TokenizeTest):
     def check_tokenize(self, s, expected):
         # Format the tokens in s in a table format.

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
@@ -2007,6 +2007,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
         tok->atbol = 1;
         if (blankline || tok->level > 0) {
             if (tok->tok_extra_tokens) {
+                if (tok->comment_newline) {
+                    tok->comment_newline = 0;
+                }
                 p_start = tok->start;
                 p_end = tok->cur;
                 return MAKE_TOKEN(NL);
@@ -2015,9 +2018,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
         }
         if (tok->comment_newline && tok->tok_extra_tokens) {
             tok->comment_newline = 0;
-                p_start = tok->start;
-                p_end = tok->cur;
-                return MAKE_TOKEN(NL);
+            p_start = tok->start;
+            p_end = tok->cur;
+            return MAKE_TOKEN(NL);
         }
         p_start = tok->start;
         p_end = tok->cur - 1; /* Leave '\n' out of the string */