Fix generics creation time and allow model name reusing (#2078)

* preserve progress

* make get_caller_module_name much faster
combine get_caller_module_name and is_call_from_module in get_caller_frame_info

* fix coverage

* add changes file

changes/2078-MrMrRobat.md

@@ -0,0 +1 @@
+fix slow `GenericModel` concrete model creation, allow `GenericModel` concrete name reusing in module 

pydantic/generics.py

@@ -1,9 +1,7 @@
 import sys
-from types import FrameType, ModuleType
 from typing import (
     TYPE_CHECKING,
     Any,
-    Callable,
     ClassVar,
     Dict,
     Generic,
@@ -62,26 +60,26 @@ def __class_getitem__(cls: Type[GenericModelT], params: Union[Type[Any], Tuple[T
         model_name = cls.__concrete_name__(params)
         validators = gather_all_validators(cls)
         fields = _build_generic_fields(cls.__fields__, concrete_type_hints, typevars_map)
-        model_module = get_caller_module_name() or cls.__module__
+        model_module, called_globally = get_caller_frame_info()
         created_model = cast(
             Type[GenericModel],  # casting ensures mypy is aware of the __concrete__ and __parameters__ attributes
             create_model(
                 model_name,
-                __module__=model_module,
+                __module__=model_module or cls.__module__,
                 __base__=cls,
                 __config__=None,
                 __validators__=validators,
                 **fields,
             ),
         )
 
-        if is_call_from_module():  # create global reference and therefore allow pickling
-            object_in_module = sys.modules[model_module].__dict__.setdefault(model_name, created_model)
-            if object_in_module is not created_model:
-                # this should not ever happen because of _generic_types_cache, but just in case
-                raise TypeError(f'{model_name!r} already defined above, please consider reusing it') from NameError(
-                    f'Name conflict: {model_name!r} in {model_module!r} is already used by {object_in_module!r}'
-                )
+        if called_globally:  # create global reference and therefore allow pickling
+            object_by_reference = None
+            reference_name = model_name
+            reference_module_globals = sys.modules[created_model.__module__].__dict__
+            while object_by_reference is not created_model:
+                object_by_reference = reference_module_globals.setdefault(reference_name, created_model)
+                reference_name += '_'
 
         created_model.Config = cls.Config
         concrete = all(not _is_typevar(v) for v in concrete_type_hints.values())
@@ -143,34 +141,19 @@ def _is_typevar(v: Any) -> bool:
     return isinstance(v, TypeVar)
 
 
-def get_caller_module_name() -> Optional[str]:
-    """
-    Used inside a function to get its caller module name
-
-    Will only work against non-compiled code, therefore used only in pydantic.generics
-    """
-    import inspect
-
-    try:
-        previous_caller_frame = inspect.stack()[2].frame
-    except IndexError as e:
-        raise RuntimeError('This function must be used inside another function') from e
-
-    getmodule = cast(Callable[[FrameType, str], Optional[ModuleType]], inspect.getmodule)
-    previous_caller_module = getmodule(previous_caller_frame, previous_caller_frame.f_code.co_filename)
-    return previous_caller_module.__name__ if previous_caller_module is not None else None
-
-
-def is_call_from_module() -> bool:
+def get_caller_frame_info() -> Tuple[Optional[str], bool]:
     """
     Used inside a function to check whether it was called globally
 
     Will only work against non-compiled code, therefore used only in pydantic.generics
-    """
-    import inspect
 
+    :returns Tuple[module_name, called_globally]
+    """
     try:
-        previous_caller_frame = inspect.stack()[2].frame
-    except IndexError as e:
+        previous_caller_frame = sys._getframe(2)
+    except ValueError as e:
         raise RuntimeError('This function must be used inside another function') from e
-    return previous_caller_frame.f_locals is previous_caller_frame.f_globals
+    except AttributeError:  # sys module does not have _getframe function, so there's nothing we can do about it
+        return None, False
+    frame_globals = previous_caller_frame.f_globals
+    return frame_globals.get('__name__'), previous_caller_frame.f_locals is frame_globals

tests/test_generics.py

@@ -5,7 +5,7 @@
 import pytest
 
 from pydantic import BaseModel, Field, ValidationError, root_validator, validator
-from pydantic.generics import GenericModel, _generic_types_cache, get_caller_module_name
+from pydantic.generics import GenericModel, _generic_types_cache
 
 skip_36 = pytest.mark.skipif(sys.version_info < (3, 7), reason='generics only supported for python 3.7 and above')
 
@@ -686,95 +686,73 @@ def test_generic_model_redefined_without_cache_fail(create_module):
     def module():
         from typing import Generic, TypeVar
 
-        import pytest
-
+        from pydantic import BaseModel
         from pydantic.generics import GenericModel, _generic_types_cache
 
         t = TypeVar('t')
 
         class MyGeneric(GenericModel, Generic[t]):
             value: t
 
-        concrete = MyGeneric[t]
-        _generic_types_cache.clear()
-        with pytest.raises(
-            TypeError, match=r"'MyGeneric\[t\]' already defined above, please consider reusing it"
-        ) as exc_info:
-            MyGeneric[t]
-
-        cause = exc_info.value.__cause__
-        assert isinstance(cause, NameError), cause
-        expected_message = f"Name conflict: 'MyGeneric[t]' in {__name__!r} is already used by {concrete!r}"
-        assert cause.args[0] == expected_message, f'{cause.args[0]} != {expected_message}'
-
-
-def test_get_caller_module_name(create_module):
-    @create_module
-    def module():
-        from pydantic.generics import get_caller_module_name
-
-        def get_current_module_name():
-            return get_caller_module_name()
+        class Model(BaseModel):
+            ...
 
-        assert get_current_module_name() == __name__
+        concrete = MyGeneric[Model]
+        _generic_types_cache.clear()
+        second_concrete = MyGeneric[Model]
 
+        class Model(BaseModel):  # same name, but type different, so it's not in cache
+            ...
 
-def test_get_caller_module_name_not_found(mocker):
-    mocker.patch('inspect.getmodule', return_value=None)
-    assert get_caller_module_name() is None
+        third_concrete = MyGeneric[Model]
+        assert concrete is not second_concrete
+        assert concrete is not third_concrete
+        assert second_concrete is not third_concrete
+        assert globals()['MyGeneric[Model]'] is concrete
+        assert globals()['MyGeneric[Model]_'] is second_concrete
+        assert globals()['MyGeneric[Model]__'] is third_concrete
 
 
-def test_is_call_from_module(create_module):
+def test_get_caller_frame_info(create_module):
     @create_module
     def module():
-        from pydantic.generics import is_call_from_module
+        from pydantic.generics import get_caller_frame_info
 
         def function():
-            assert is_call_from_module()
+            assert get_caller_frame_info() == (__name__, True)
 
             another_function()
 
         def another_function():
-            assert not is_call_from_module()
+            assert get_caller_frame_info() == (__name__, False)
             third_function()
 
         def third_function():
-            assert not is_call_from_module()
+            assert get_caller_frame_info() == (__name__, False)
 
         function()
 
 
-def test_is_call_from_module_called_in_module(create_module):
+def test_get_caller_frame_info_called_from_module(create_module):
     @create_module
     def module():
         from unittest.mock import patch
 
         import pytest
 
-        from pydantic.generics import is_call_from_module
-
-        with pytest.raises(RuntimeError, match='This function must be used inside another function') as exc_info:
-            with patch('inspect.stack', new=lambda: [..., ...]):
-                is_call_from_module()
-
-        e = exc_info.value
-        assert isinstance(e.__cause__, IndexError)
-        assert isinstance(e.__context__, IndexError)
-
+        from pydantic.generics import get_caller_frame_info
 
-def test_get_caller_module_called_from_module(create_module):
-    @create_module
-    def module():
-        from unittest.mock import patch
-
-        import pytest
+        with pytest.raises(RuntimeError, match='This function must be used inside another function'):
+            with patch('sys._getframe', side_effect=ValueError('getframe_exc')):
+                get_caller_frame_info()
 
-        from pydantic.generics import get_caller_module_name
 
-        with pytest.raises(RuntimeError, match='This function must be used inside another function') as exc_info:
-            with patch('inspect.stack', new=lambda: [..., ...]):
-                get_caller_module_name()
+def test_get_caller_frame_info_when_sys_getframe_undefined():
+    from pydantic.generics import get_caller_frame_info
 
-        e = exc_info.value
-        assert isinstance(e.__cause__, IndexError)
-        assert isinstance(e.__context__, IndexError)
+    getframe = sys._getframe
+    del sys._getframe
+    try:
+        assert get_caller_frame_info() == (None, False)
+    finally:  # just to make sure we always setting original attribute back
+        sys._getframe = getframe