bpo-46542: test_lib2to3 uses support.infinite_recursion() (GH-31035)

* bpo-46542: test_lib2to3 uses support.infinite_recursion()

Fix a Python crash in test_lib2to3 when using Python built in debug
mode: limit the recursion limit.

The test_all_project_files() test of test_lib2to3 now uses the
test.support.infinite_recursion() context manager when processing the
infinite_recursion.py file to prevent a crash when Python is built in
debug mode.

The two test_all_project_files() tests now use subTest() and log the
refactored/parsed filename (if test_lib2to3 is run in verbose mode).

* Update Lib/lib2to3/tests/data/infinite_recursion.py

Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com>

Co-authored-by: Łukasz Langa <lukasz@langa.pl>
Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com>
(cherry picked from commit ee0ac32)

Co-authored-by: Victor Stinner <vstinner@python.org>

Lib/lib2to3/pytree.py

@@ -720,8 +720,8 @@ def generate_matches(self, nodes):
                         r[self.name] = nodes[:count]
                     yield count, r
             except RuntimeError:
-                # We fall back to the iterative pattern matching scheme if the recursive
-                # scheme hits the recursion limit.
+                # Fall back to the iterative pattern matching scheme if the
+                # recursive scheme hits the recursion limit (RecursionError).
                 for count, r in self._iterative_matches(nodes):
                     if self.name:
                         r[self.name] = nodes[:count]

Lib/lib2to3/tests/data/infinite_recursion.py

@@ -1,5 +1,5 @@
-# This file is used to verify that 2to3 falls back to a slower, iterative pattern matching
-# scheme in the event that the faster recursive system fails due to infinite recursion.
+# Verify that 2to3 falls back from the recursive pattern matching scheme to a
+# slower, iterative scheme in the event of a RecursionError.
 from ctypes import *
 STRING = c_char_p
 

Lib/lib2to3/tests/test_all_fixers.py

@@ -6,8 +6,10 @@
 # Author: Collin Winter
 
 # Python imports
-import unittest
+import os.path
+import sys
 import test.support
+import unittest
 
 # Local imports
 from . import support
@@ -19,9 +21,22 @@ class Test_all(support.TestCase):
     def setUp(self):
         self.refactor = support.get_refactorer()
 
+    def refactor_file(self, filepath):
+        if test.support.verbose:
+            print(f"Refactor file: {filepath}")
+        if os.path.basename(filepath) == 'infinite_recursion.py':
+            # bpo-46542: Processing infinite_recursion.py can crash Python
+            # if Python is built in debug mode: lower the recursion limit
+            # to prevent a crash.
+            with test.support.infinite_recursion(150):
+                self.refactor.refactor_file(filepath)
+        else:
+            self.refactor.refactor_file(filepath)
+
     def test_all_project_files(self):
         for filepath in support.all_project_files():
-            self.refactor.refactor_file(filepath)
+            with self.subTest(filepath=filepath):
+                self.refactor_file(filepath)
 
 if __name__ == '__main__':
     unittest.main()

Lib/lib2to3/tests/test_parser.py

@@ -20,6 +20,7 @@
 import subprocess
 import sys
 import tempfile
+import test.support
 import unittest
 
 # Local imports
@@ -582,25 +583,31 @@ class TestParserIdempotency(support.TestCase):
 
     """A cut-down version of pytree_idempotency.py."""
 
+    def parse_file(self, filepath):
+        if test.support.verbose:
+            print(f"Parse file: {filepath}")
+        with open(filepath, "rb") as fp:
+            encoding = tokenize.detect_encoding(fp.readline)[0]
+        self.assertIsNotNone(encoding,
+                             "can't detect encoding for %s" % filepath)
+        with open(filepath, "r", encoding=encoding) as fp:
+            source = fp.read()
+        try:
+            tree = driver.parse_string(source)
+        except ParseError:
+            try:
+                tree = driver_no_print_statement.parse_string(source)
+            except ParseError as err:
+                self.fail('ParseError on file %s (%s)' % (filepath, err))
+        new = str(tree)
+        if new != source:
+            print(diff_texts(source, new, filepath))
+            self.fail("Idempotency failed: %s" % filepath)
+
     def test_all_project_files(self):
         for filepath in support.all_project_files():
-            with open(filepath, "rb") as fp:
-                encoding = tokenize.detect_encoding(fp.readline)[0]
-            self.assertIsNotNone(encoding,
-                                 "can't detect encoding for %s" % filepath)
-            with open(filepath, "r", encoding=encoding) as fp:
-                source = fp.read()
-            try:
-                tree = driver.parse_string(source)
-            except ParseError:
-                try:
-                    tree = driver_no_print_statement.parse_string(source)
-                except ParseError as err:
-                    self.fail('ParseError on file %s (%s)' % (filepath, err))
-            new = str(tree)
-            if new != source:
-                print(diff_texts(source, new, filepath))
-                self.fail("Idempotency failed: %s" % filepath)
+            with self.subTest(filepath=filepath):
+                self.parse_file(filepath)
 
     def test_extended_unpacking(self):
         driver.parse_string("a, *b, c = x\n")

Misc/NEWS.d/next/Tests/2022-01-31-17-34-13.bpo-46542.RTMm1T.rst

@@ -0,0 +1,2 @@
+Fix a Python crash in test_lib2to3 when using Python built in debug mode:
+limit the recursion limit. Patch by Victor Stinner.