From ed99f4ca2949168243b55380443f1e5b5cae3244 Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Wed, 10 Sep 2025 19:45:48 +0100 Subject: [PATCH 1/9] Add tests for calling a custom annotate function that only implements VALUE annotations --- Lib/test/test_annotationlib.py | 69 ++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 88e0d611647f28..40cdff377b185c 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -10,6 +10,7 @@ from string.templatelib import Template import typing import unittest +import unittest.mock from annotationlib import ( Format, ForwardRef, @@ -1206,6 +1207,74 @@ def evaluate(format, exc=NotImplementedError): ) +class TestCallAnnotateFunction(unittest.TestCase): + def _annotate_mock(self): + def annotate(format, /): + if format == Format.VALUE: + return {"x": str} + else: + raise NotImplementedError(format) + + annotate_mock = unittest.mock.MagicMock( + wraps=annotate + ) + + # Add missing magic attributes needed + required_magic = [ + "__builtins__", + "__closure__", + "__code__", + "__defaults__", + "__globals__", + "__kwdefaults__", + ] + + for attrib in required_magic: + setattr(annotate_mock, attrib, getattr(annotate, attrib)) + + return annotate_mock + + def test_user_annotate_value(self): + annotate = self._annotate_mock() + + annotations = annotationlib.call_annotate_function( + annotate, + Format.VALUE, + ) + + self.assertEqual(annotations, {"x": str}) + annotate.assert_called_once_with(Format.VALUE) + + def test_user_annotate_forwardref(self): + annotate = self._annotate_mock() + + with self.assertRaises(NotImplementedError): + annotations = annotationlib.call_annotate_function( + annotate, + Format.FORWARDREF, + ) + + annotate.assert_has_calls([ + unittest.mock.Call(Format.FORWARDREF), + unittest.mock.Call(Format.VALUE_WITH_FAKE_GLOBALS), + ]) + + def test_user_annotate_string(self): + annotate = self._annotate_mock() + + with self.assertRaises(NotImplementedError): + annotations = annotationlib.call_annotate_function( + annotate, + Format.STRING, + ) + + annotate.assert_has_calls([ + unittest.mock.Call(Format.STRING), + unittest.mock.Call(Format.VALUE_WITH_FAKE_GLOBALS), + ]) + + + class MetaclassTests(unittest.TestCase): def test_annotated_meta(self): class Meta(type): From 4d88779eaeae2385cd5cf96dc7e7d4cb4477bd5a Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Wed, 10 Sep 2025 19:49:51 +0100 Subject: [PATCH 2/9] call is lower case --- Lib/test/test_annotationlib.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 40cdff377b185c..039c070ed33c0c 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -1255,8 +1255,8 @@ def test_user_annotate_forwardref(self): ) annotate.assert_has_calls([ - unittest.mock.Call(Format.FORWARDREF), - unittest.mock.Call(Format.VALUE_WITH_FAKE_GLOBALS), + unittest.mock.call(Format.FORWARDREF), + unittest.mock.call(Format.VALUE_WITH_FAKE_GLOBALS), ]) def test_user_annotate_string(self): @@ -1269,8 +1269,8 @@ def test_user_annotate_string(self): ) annotate.assert_has_calls([ - unittest.mock.Call(Format.STRING), - unittest.mock.Call(Format.VALUE_WITH_FAKE_GLOBALS), + unittest.mock.call(Format.STRING), + unittest.mock.call(Format.VALUE_WITH_FAKE_GLOBALS), ]) From 8296cd8d833c70e8d437fecc188ad71874c0256b Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Wed, 10 Sep 2025 20:01:13 +0100 Subject: [PATCH 3/9] actual function is only called once --- Lib/test/test_annotationlib.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 039c070ed33c0c..b8355a7cf5e21d 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -1254,10 +1254,9 @@ def test_user_annotate_forwardref(self): Format.FORWARDREF, ) - annotate.assert_has_calls([ - unittest.mock.call(Format.FORWARDREF), - unittest.mock.call(Format.VALUE_WITH_FAKE_GLOBALS), - ]) + # The annotate function itself is not called the second time + # A new function built from the code is called instead + annotate.assert_called_once_with(Format.FORWARDREF) def test_user_annotate_string(self): annotate = self._annotate_mock() @@ -1274,7 +1273,6 @@ def test_user_annotate_string(self): ]) - class MetaclassTests(unittest.TestCase): def test_annotated_meta(self): class Meta(type): From c19bcbdc71b7c3bb71079c3e6433d86cae0f925f Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Wed, 10 Sep 2025 20:04:49 +0100 Subject: [PATCH 4/9] Fail if NotImplementedError is raised for VALUE_WITH_FAKE_GLOBALS --- Lib/annotationlib.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index bee019cd51591e..13590c62bee2d3 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -664,6 +664,17 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): # possibly constants if the annotate function uses them directly). We then # convert each of those into a string to get an approximation of the # original source. + + # Attempt to call with VALUE_WITH_FAKE_GLOBALS to check if it is implemented + # See: https://github.com/python/cpython/issues/138764 + # Only fail on NotImplementedError + try: + annotate(Format.VALUE_WITH_FAKE_GLOBALS) + except NotImplementedError: + raise + except Exception: + pass + globals = _StringifierDict({}, format=format) is_class = isinstance(owner, type) closure = _build_closure( @@ -722,6 +733,10 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): ) try: result = func(Format.VALUE_WITH_FAKE_GLOBALS) + except NotImplementedError: + # If NotImplementedError is raised, don't try to call again with + # no globals. + raise except Exception: pass else: From 9166a870c6a804305ccab9c50dd8dd9290497d8b Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Thu, 11 Sep 2025 18:20:16 +0100 Subject: [PATCH 5/9] fallback to using VALUE annotations instead of failing --- Lib/annotationlib.py | 8 +++---- Lib/test/test_annotationlib.py | 38 ++++++++++++++++++++++++++-------- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 13590c62bee2d3..a1ff00dcc5319b 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -671,7 +671,8 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): try: annotate(Format.VALUE_WITH_FAKE_GLOBALS) except NotImplementedError: - raise + # Both STRING and VALUE_WITH_FAKE_GLOBALS are not implemented fallback to VALUE + return annotations_to_string(annotate(Format.VALUE)) except Exception: pass @@ -734,9 +735,8 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): try: result = func(Format.VALUE_WITH_FAKE_GLOBALS) except NotImplementedError: - # If NotImplementedError is raised, don't try to call again with - # no globals. - raise + # FORWARDREF and VALUE_WITH_FAKE_GLOBALS not supported, fall back to VALUE + return annotate(Format.VALUE) except Exception: pass else: diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index b8355a7cf5e21d..bd291e37bc08cc 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -8,6 +8,7 @@ import itertools import pickle from string.templatelib import Template +import types import typing import unittest import unittest.mock @@ -1248,30 +1249,49 @@ def test_user_annotate_value(self): def test_user_annotate_forwardref(self): annotate = self._annotate_mock() - with self.assertRaises(NotImplementedError): + new_annotate = None + functype = types.FunctionType + + def functiontype(*args, **kwargs): + nonlocal new_annotate + new_func = unittest.mock.MagicMock(wraps=functype(*args, **kwargs)) + new_annotate = new_func + return new_func + + with unittest.mock.patch("types.FunctionType", new=functiontype): annotations = annotationlib.call_annotate_function( annotate, Format.FORWARDREF, ) - # The annotate function itself is not called the second time - # A new function built from the code is called instead - annotate.assert_called_once_with(Format.FORWARDREF) + # The call with Format.VALUE_WITH_FAKE_GLOBALS is not + # on the original function. + annotate.assert_has_calls([ + unittest.mock.call(Format.FORWARDREF), + unittest.mock.call(Format.VALUE), + ]) + + new_annotate.assert_called_once_with(Format.VALUE_WITH_FAKE_GLOBALS) + + self.assertEqual(annotations, {"x": str}) + def test_user_annotate_string(self): annotate = self._annotate_mock() - with self.assertRaises(NotImplementedError): - annotations = annotationlib.call_annotate_function( - annotate, - Format.STRING, - ) + annotations = annotationlib.call_annotate_function( + annotate, + Format.STRING, + ) annotate.assert_has_calls([ unittest.mock.call(Format.STRING), unittest.mock.call(Format.VALUE_WITH_FAKE_GLOBALS), + unittest.mock.call(Format.VALUE), ]) + self.assertEqual(annotations, {"x": "str"}) + class MetaclassTests(unittest.TestCase): def test_annotated_meta(self): From 18bdc1589ea34fd71249395c64d21fbc255ce57f Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Fri, 12 Sep 2025 10:27:27 +0100 Subject: [PATCH 6/9] Test observable behaviour instead of internals --- Lib/test/test_annotationlib.py | 158 ++++++++++++++++++++++----------- 1 file changed, 106 insertions(+), 52 deletions(-) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index bd291e37bc08cc..db1b74c5708700 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -8,10 +8,8 @@ import itertools import pickle from string.templatelib import Template -import types import typing import unittest -import unittest.mock from annotationlib import ( Format, ForwardRef, @@ -1209,89 +1207,145 @@ def evaluate(format, exc=NotImplementedError): class TestCallAnnotateFunction(unittest.TestCase): - def _annotate_mock(self): + # Tests for user defined annotate functions. + + # Format and NotImplementedError are provided as arguments so they exist in + # the fake globals namespace. + # This avoids non-matching conditions passing by being converted to stringifiers. + # See: https://github.com/python/cpython/issues/138764 + + def test_user_annotate_value(self): def annotate(format, /): if format == Format.VALUE: return {"x": str} else: raise NotImplementedError(format) - annotate_mock = unittest.mock.MagicMock( - wraps=annotate + annotations = annotationlib.call_annotate_function( + annotate, + Format.VALUE, ) - # Add missing magic attributes needed - required_magic = [ - "__builtins__", - "__closure__", - "__code__", - "__defaults__", - "__globals__", - "__kwdefaults__", - ] + self.assertEqual(annotations, {"x": str}) + + def test_user_annotate_forwardref_supported(self): + # If Format.FORWARDREF is supported prefer it over Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + elif format == __Format.FORWARDREF: + return {'x': float} + else: + raise __NotImplementedError(format) - for attrib in required_magic: - setattr(annotate_mock, attrib, getattr(annotate, attrib)) + annotations = annotationlib.call_annotate_function( + annotate, + Format.FORWARDREF + ) - return annotate_mock + self.assertEqual(annotations, {"x": float}) - def test_user_annotate_value(self): - annotate = self._annotate_mock() + def test_user_annotate_forwardref_fakeglobals(self): + # If Format.FORWARDREF is not supported, use Format.VALUE_WITH_FAKE_GLOBALS + # before falling back to Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + else: + raise __NotImplementedError(format) annotations = annotationlib.call_annotate_function( annotate, - Format.VALUE, + Format.FORWARDREF ) - self.assertEqual(annotations, {"x": str}) - annotate.assert_called_once_with(Format.VALUE) + self.assertEqual(annotations, {"x": int}) - def test_user_annotate_forwardref(self): - annotate = self._annotate_mock() + def test_user_annotate_forwardref_value_fallback(self): + # If Format.FORWARDREF and Format.VALUE_WITH_FAKE_GLOBALS are not supported + # use Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {"x": str} + else: + raise __NotImplementedError(format) - new_annotate = None - functype = types.FunctionType + annotations = annotationlib.call_annotate_function( + annotate, + Format.FORWARDREF, + ) - def functiontype(*args, **kwargs): - nonlocal new_annotate - new_func = unittest.mock.MagicMock(wraps=functype(*args, **kwargs)) - new_annotate = new_func - return new_func + self.assertEqual(annotations, {"x": str}) - with unittest.mock.patch("types.FunctionType", new=functiontype): - annotations = annotationlib.call_annotate_function( - annotate, - Format.FORWARDREF, - ) + def test_user_annotate_string_supported(self): + # If Format.STRING is supported prefer it over Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + elif format == __Format.STRING: + return {'x': "float"} + else: + raise __NotImplementedError(format) - # The call with Format.VALUE_WITH_FAKE_GLOBALS is not - # on the original function. - annotate.assert_has_calls([ - unittest.mock.call(Format.FORWARDREF), - unittest.mock.call(Format.VALUE), - ]) + annotations = annotationlib.call_annotate_function( + annotate, + Format.STRING, + ) - new_annotate.assert_called_once_with(Format.VALUE_WITH_FAKE_GLOBALS) + self.assertEqual(annotations, {"x": "float"}) - self.assertEqual(annotations, {"x": str}) + def test_user_annotate_string_fakeglobals(self): + # If Format.STRING is not supported but Format.VALUE_WITH_FAKE_GLOBALS is + # prefer that over Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + else: + raise __NotImplementedError(format) + annotations = annotationlib.call_annotate_function( + annotate, + Format.STRING, + ) - def test_user_annotate_string(self): - annotate = self._annotate_mock() + self.assertEqual(annotations, {"x": "int"}) + + def test_user_annotate_string_value_fallback(self): + # If Format.STRING and Format.VALUE_WITH_FAKE_GLOBALS are not + # supported fall back to Format.VALUE and convert to strings + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {"x": str} + else: + raise __NotImplementedError(format) annotations = annotationlib.call_annotate_function( annotate, Format.STRING, ) - annotate.assert_has_calls([ - unittest.mock.call(Format.STRING), - unittest.mock.call(Format.VALUE_WITH_FAKE_GLOBALS), - unittest.mock.call(Format.VALUE), - ]) - self.assertEqual(annotations, {"x": "str"}) + def test_condition_not_stringified(self): + # Make sure the first condition isn't evaluated as True by being converted + # to a _Stringifier + def annotate(format, /): + if format == Format.FORWARDREF: + return {"x": str} + else: + raise NotImplementedError(format) + + with self.assertRaises(NotImplementedError): + _ = annotationlib.call_annotate_function(annotate, Format.STRING) + class MetaclassTests(unittest.TestCase): def test_annotated_meta(self): From b387bc3e38b848f1093f90aac0ca4afcd9249351 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Fri, 12 Sep 2025 09:34:43 +0000 Subject: [PATCH 7/9] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2025-09-12-09-34-37.gh-issue-138764.mokHoY.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-09-12-09-34-37.gh-issue-138764.mokHoY.rst diff --git a/Misc/NEWS.d/next/Library/2025-09-12-09-34-37.gh-issue-138764.mokHoY.rst b/Misc/NEWS.d/next/Library/2025-09-12-09-34-37.gh-issue-138764.mokHoY.rst new file mode 100644 index 00000000000000..f0b9737d9d5bf8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-12-09-34-37.gh-issue-138764.mokHoY.rst @@ -0,0 +1,2 @@ +Prevent ``annotationlib.call_annotate_function`` from calling ``annotate`` functions that don't support ``VALUE_WITH_FAKE_GLOBALS`` in a fake globals namespace with empty globals. +Make ``FORWARDREF`` and ``STRING`` annotations fall back to using ``VALUE`` annotations in the case that neither their own format, nor ``VALUE_WITH_FAKE_GLOBALS`` are supported. From 63e4bc996f6fff000849153044e7efb915866d78 Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Fri, 12 Sep 2025 11:53:41 +0100 Subject: [PATCH 8/9] Add a test for propagating the error from a failed call to `VALUE` annotations if they fail. --- Lib/test/test_annotationlib.py | 37 +++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index db1b74c5708700..91608bcd1f7eb4 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -1186,6 +1186,25 @@ class RaisesAttributeError: }, ) + def test_raises_error_from_value(self): + # test that if VALUE is the only supported format, but raises an error + # that error is propagated from get_annotations + class DemoException(Exception): ... + + def annotate(format, /): + if format == Format.VALUE: + raise DemoException() + else: + raise NotImplementedError(format) + + def f(): ... + + f.__annotate__ = annotate + + for fmt in [Format.VALUE, Format.FORWARDREF, Format.STRING]: + with self.assertRaises(DemoException): + get_annotations(f, format=fmt) + class TestCallEvaluateFunction(unittest.TestCase): def test_evaluation(self): @@ -1344,7 +1363,23 @@ def annotate(format, /): raise NotImplementedError(format) with self.assertRaises(NotImplementedError): - _ = annotationlib.call_annotate_function(annotate, Format.STRING) + annotationlib.call_annotate_function(annotate, Format.STRING) + + def test_error_from_value_raised(self): + # Test that the error from format.VALUE is raised + # if all formats fail + + class DemoException(Exception): ... + + def annotate(format, /): + if format == Format.VALUE: + raise DemoException() + else: + raise NotImplementedError(format) + + for fmt in [Format.VALUE, Format.FORWARDREF, Format.STRING]: + with self.assertRaises(DemoException): + annotationlib.call_annotate_function(annotate, format=fmt) class MetaclassTests(unittest.TestCase): From 0c7fefc2d8b228d48944e8f3644c37ff042b0f01 Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Fri, 12 Sep 2025 12:09:33 +0100 Subject: [PATCH 9/9] Extend the listing of how the formats work --- Doc/library/annotationlib.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index d6f5055955e8cf..612eb02fde06f6 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -340,14 +340,29 @@ Functions * VALUE: :attr:`!object.__annotations__` is tried first; if that does not exist, the :attr:`!object.__annotate__` function is called if it exists. + * FORWARDREF: If :attr:`!object.__annotations__` exists and can be evaluated successfully, it is used; otherwise, the :attr:`!object.__annotate__` function is called. If it does not exist either, :attr:`!object.__annotations__` is tried again and any error from accessing it is re-raised. + + * When calling :attr:`!object.__annotate__` it is first called with :attr:`~Format.FORWARDREF`, + if this is not implemented it will then check if :attr:`~Format.VALUE_WITH_FAKE_GLOBALS` + is supported and use that in the fake globals environment. + If neither of these formats are supported it will fall back to using :attr:`~Format.VALUE`. + If :attr:`~Format.VALUE` fails, the error from this call will be raised. + * STRING: If :attr:`!object.__annotate__` exists, it is called first; otherwise, :attr:`!object.__annotations__` is used and stringified using :func:`annotations_to_string`. + * When calling :attr:`!object.__annotate__` it is first called with :attr:`~Format.STRING`, + if this is not implemented it will then check if :attr:`~Format.VALUE_WITH_FAKE_GLOBALS` + is supported and use that in the fake globals environment. + If neither of these formats are supported it will fall back to using :attr:`~Format.VALUE` + with the result converted using :func:`annotations_to_string`. + If :attr:`~Format.VALUE` fails, the error from this call will be raised. + Returns a dict. :func:`!get_annotations` returns a new dict every time it's called; calling it twice on the same object will return two different but equivalent dicts.