diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index f28a3a2632c1c52..4fe82e917d3e38d 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2242,6 +2242,34 @@ def check_disallow_instantiation(testcase, tp, *args, **kwds): msg = f"cannot create '{re.escape(qualname)}' instances" testcase.assertRaisesRegex(TypeError, msg, tp, *args, **kwds) +def get_recursion_depth(): + try: + import _testinternalcapi + except ImportError: + _testinternalcapi = None + + if _testinternalcapi is not None: + depth = _testinternalcapi.get_recursion_depth() + else: + # Implementation using sys._getframe() and frame.f_back + try: + depth = 0 + frame = sys._getframe() + while frame is not None: + depth += 1 + frame = frame.f_back + finally: + # Break reference cycle + frame = None + + # Ignore get_recursion_depth() frame + return max(depth - 1, 1) + +def get_recursion_available(): + limit = sys.getrecursionlimit() + depth = get_recursion_depth() + return limit - depth + @contextlib.contextmanager def set_recursion_limit(limit): """Temporarily change the recursion limit.""" @@ -2252,14 +2280,16 @@ def set_recursion_limit(limit): finally: sys.setrecursionlimit(original_limit) -def infinite_recursion(max_depth=75): +def infinite_recursion(max_depth=100): """Set a lower limit for tests that interact with infinite recursions (e.g test_ast.ASTHelpers_Test.test_recursion_direct) since on some debug windows builds, due to not enough functions being inlined the stack size might not handle the default recursion limit (1000). See bpo-11105 for details.""" - return set_recursion_limit(max_depth) - + if max_depth < 1: + raise ValueError("max_depth must be at least 1") + limit = get_recursion_depth() + max_depth + return set_recursion_limit(limit) def ignore_deprecations_from(module: str, *, like: str) -> object: token = object() diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 86d26b7e8df4d02..90b121529b1c358 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -685,6 +685,64 @@ def test_has_strftime_extensions(self): else: self.assertTrue(support.has_strftime_extensions) + def test_get_recursion_depth(self): + # test support.get_recursion_depth() + code = textwrap.dedent(""" + from test import support + import sys + + def check(cond): + if not cond: + raise AssertionError("test failed") + + check(support.get_recursion_depth() == 1) + + def test_func(): + check(support.get_recursion_depth() == 2) + test_func() + + def test_recursive(depth, limit): + if depth >= limit: + return + get_depth = support.get_recursion_depth() + print(f"test_recursive: {depth}/{limit}: get_recursion_depth() says {get_depth}") + check(get_depth == depth) + test_recursive(depth + 1, limit) + + with support.infinite_recursion(): + limit = sys.getrecursionlimit() + test_recursive(2, limit) + """) + script_helper.assert_python_ok("-c", code) + + def test_recursion(self): + # test infinite_recursion() and get_recursion_available() functions + def recursive_function(depth): + if depth: + recursive_function(depth - 1) + + # test also the bare minimum, max_depth=1: only allow one frame, one + # function call! + for max_depth in (1, 5, 25, 250): + with support.infinite_recursion(max_depth): + limit = sys.getrecursionlimit() + available = support.get_recursion_available() + try: + recursive_function(available) + # avoid self.assertRaises(RecursionError) which consumes + # more than one frame and so raises RecursionError + try: + recursive_function(available + 1) + except RecursionError: + pass + else: + self.fail("RecursionError was not raised") + except Exception as exc: + # avoid self.subTest() since it consumes more + # than one frame and so raises RecursionError! + raise AssertionError(f"test failed with {max_depth=}, " + f"{limit=}, {available=}") from exc + # XXX -follows a list of untested API # make_legacy_pyc # is_resource_enabled diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index d8b684c8a008f09..346a3778b351408 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -279,20 +279,29 @@ def test_switchinterval(self): finally: sys.setswitchinterval(orig) - def test_recursionlimit(self): + def test_getrecursionlimit(self): + limit = sys.getrecursionlimit() + self.assertIsInstance(limit, int) + self.assertGreater(limit, 1) + self.assertRaises(TypeError, sys.getrecursionlimit, 42) - oldlimit = sys.getrecursionlimit() - self.assertRaises(TypeError, sys.setrecursionlimit) - self.assertRaises(ValueError, sys.setrecursionlimit, -42) - sys.setrecursionlimit(10000) - self.assertEqual(sys.getrecursionlimit(), 10000) - sys.setrecursionlimit(oldlimit) + + def test_setrecursionlimit(self): + old_limit = sys.getrecursionlimit() + try: + sys.setrecursionlimit(10000) + self.assertEqual(sys.getrecursionlimit(), 10000) + + self.assertRaises(TypeError, sys.setrecursionlimit) + self.assertRaises(ValueError, sys.setrecursionlimit, -42) + finally: + sys.setrecursionlimit(old_limit) def test_recursionlimit_recovery(self): if hasattr(sys, 'gettrace') and sys.gettrace(): self.skipTest('fatal error if run with a trace function') - oldlimit = sys.getrecursionlimit() + old_limit = sys.getrecursionlimit() def f(): f() try: @@ -311,35 +320,63 @@ def f(): with self.assertRaises(RecursionError): f() finally: - sys.setrecursionlimit(oldlimit) + sys.setrecursionlimit(old_limit) @test.support.cpython_only - def test_setrecursionlimit_recursion_depth(self): + def test_setrecursionlimit_recursionerror(self): # Issue #25274: Setting a low recursion limit must be blocked if the # current recursion depth is already higher than limit. - from _testinternalcapi import get_recursion_depth - - def set_recursion_limit_at_depth(depth, limit): - recursion_depth = get_recursion_depth() - if recursion_depth >= depth: + def set_recursion_limit_at_depth(desired_depth, limit): + recursion_depth = support.get_recursion_depth() + if recursion_depth < desired_depth: + set_recursion_limit_at_depth(desired_depth, limit) + else: with self.assertRaises(RecursionError) as cm: sys.setrecursionlimit(limit) self.assertRegex(str(cm.exception), "cannot set the recursion limit to [0-9]+ " "at the recursion depth [0-9]+: " "the limit is too low") - else: - set_recursion_limit_at_depth(depth, limit) - oldlimit = sys.getrecursionlimit() + old_limit = sys.getrecursionlimit() + try: + with support.infinite_recursion(max_depth=500): + for set_limit in (10, 25, 100, 250): + # At depth limit+1, sys.setrecursionlimit(limit) must + # raises RecursionError + depth = set_limit + 1 + + # Add details if the test fails + with self.subTest(limit=sys.getrecursionlimit(), + available=support.get_recursion_depth(), + set_limit=set_limit): + set_recursion_limit_at_depth(depth, set_limit) + finally: + sys.setrecursionlimit(old_limit) + + @test.support.cpython_only + def test_setrecursionlimit_to_depth(self): + def func(): + pass + + old_limit = sys.getrecursionlimit() try: - sys.setrecursionlimit(1000) + depth = support.get_recursion_depth() - for limit in (10, 25, 50, 75, 100, 150, 200): - set_recursion_limit_at_depth(limit, limit) + # gh-108851: Corner case: set the limit to current recursion depth + # should be permitted. Calling any Python function raises + # RecursionError! Calling C functions is still ok, like the + # sys.setrecursionlimit() call below. + sys.setrecursionlimit(depth) + try: + func() + except RecursionError: + pass + else: + self.fail("RecursionError was not raised") finally: - sys.setrecursionlimit(oldlimit) + sys.setrecursionlimit(old_limit) def test_getwindowsversion(self): # Raise SkipTest if sys doesn't have getwindowsversion attribute diff --git a/Lib/test/test_tomllib/test_misc.py b/Lib/test/test_tomllib/test_misc.py index a477a219fd9ebd6..9e677a337a2835f 100644 --- a/Lib/test/test_tomllib/test_misc.py +++ b/Lib/test/test_tomllib/test_misc.py @@ -9,6 +9,7 @@ import sys import tempfile import unittest +from test import support from . import tomllib @@ -92,13 +93,23 @@ def test_deepcopy(self): self.assertEqual(obj_copy, expected_obj) def test_inline_array_recursion_limit(self): - # 465 with default recursion limit - nest_count = int(sys.getrecursionlimit() * 0.465) - recursive_array_toml = "arr = " + nest_count * "[" + nest_count * "]" - tomllib.loads(recursive_array_toml) + with support.infinite_recursion(max_depth=100): + available = support.get_recursion_available() + nest_count = (available // 2) - 2 + # Add details if the test fails + with self.subTest(limit=sys.getrecursionlimit(), + available=available, + nest_count=nest_count): + recursive_array_toml = "arr = " + nest_count * "[" + nest_count * "]" + tomllib.loads(recursive_array_toml) def test_inline_table_recursion_limit(self): - # 310 with default recursion limit - nest_count = int(sys.getrecursionlimit() * 0.31) - recursive_table_toml = nest_count * "key = {" + nest_count * "}" - tomllib.loads(recursive_table_toml) + with support.infinite_recursion(max_depth=100): + available = support.get_recursion_available() + nest_count = (available // 3) - 1 + # Add details if the test fails + with self.subTest(limit=sys.getrecursionlimit(), + available=available, + nest_count=nest_count): + recursive_table_toml = nest_count * "key = {" + nest_count * "}" + tomllib.loads(recursive_table_toml) diff --git a/Misc/NEWS.d/next/Library/2023-09-03-21-17-01.gh-issue-108851.XABIWw.rst b/Misc/NEWS.d/next/Library/2023-09-03-21-17-01.gh-issue-108851.XABIWw.rst new file mode 100644 index 000000000000000..6be4add11b4295f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-03-21-17-01.gh-issue-108851.XABIWw.rst @@ -0,0 +1,2 @@ +:func:`sys.setrecursionlimit` now allows setting the limit to the current +recursion depth. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2023-09-03-21-18-35.gh-issue-108851.CCuHyI.rst b/Misc/NEWS.d/next/Tests/2023-09-03-21-18-35.gh-issue-108851.CCuHyI.rst new file mode 100644 index 000000000000000..7a5b3052af22f27 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-03-21-18-35.gh-issue-108851.CCuHyI.rst @@ -0,0 +1,2 @@ +Add ``get_recursion_available()`` and ``get_recursion_depth()`` functions to +the :mod:`test.support` module. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2023-09-03-21-41-10.gh-issue-108851.xFTYOE.rst b/Misc/NEWS.d/next/Tests/2023-09-03-21-41-10.gh-issue-108851.xFTYOE.rst new file mode 100644 index 000000000000000..250e3a0b0576ea7 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-03-21-41-10.gh-issue-108851.xFTYOE.rst @@ -0,0 +1,3 @@ +Fix ``test_tomllib`` recursion tests: reduce the limit to pass tests on WASI +buildbots and compute the maximum nested array/dict depending on the current +available recursion limit. Patch by Victor Stinner. diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 0ec763c7aa7cf85..0eb210626d21fe8 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1271,7 +1271,7 @@ sys_setrecursionlimit_impl(PyObject *module, int new_limit) /* Reject too low new limit if the current recursion depth is higher than the new low-water mark. */ int depth = tstate->py_recursion_limit - tstate->py_recursion_remaining; - if (depth >= new_limit) { + if (depth > new_limit) { _PyErr_Format(tstate, PyExc_RecursionError, "cannot set the recursion limit to %i at " "the recursion depth %i: the limit is too low",