From 75986a97930a1b529f664833b08e535cbf697d37 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sun, 29 Oct 2023 22:54:23 +0200 Subject: [PATCH 01/31] Add eval_type_backport to handle union operator in older Pythons --- pydantic/_internal/_generate_schema.py | 2 +- pydantic/_internal/_typing_extra.py | 79 +++++++++++++++++++------- 2 files changed, 60 insertions(+), 21 deletions(-) diff --git a/pydantic/_internal/_generate_schema.py b/pydantic/_internal/_generate_schema.py index e87e256f5e..3f4fa73b23 100644 --- a/pydantic/_internal/_generate_schema.py +++ b/pydantic/_internal/_generate_schema.py @@ -637,7 +637,7 @@ def _resolve_forward_ref(self, obj: Any) -> Any: # class Model(BaseModel): # x: SomeImportedTypeAliasWithAForwardReference try: - obj = _typing_extra.evaluate_fwd_ref(obj, globalns=self._types_namespace) + obj = _typing_extra.eval_type_backport(obj, globalns=self._types_namespace, localns=None) except NameError as e: raise PydanticUndefinedAnnotation.from_name_error(e) from e diff --git a/pydantic/_internal/_typing_extra.py b/pydantic/_internal/_typing_extra.py index 8b94d472d7..15c174fea3 100644 --- a/pydantic/_internal/_typing_extra.py +++ b/pydantic/_internal/_typing_extra.py @@ -1,14 +1,16 @@ """Logic for interacting with type annotations, mostly extensions, shims and hacks to wrap python's typing module.""" from __future__ import annotations as _annotations +import ast import dataclasses import sys import types import typing +import uuid from collections.abc import Callable from functools import partial from types import GetSetDescriptorType -from typing import TYPE_CHECKING, Any, ForwardRef +from typing import TYPE_CHECKING, Any from typing_extensions import Annotated, Final, Literal, TypeAliasType, TypeGuard, get_args, get_origin @@ -221,12 +223,64 @@ def eval_type_lenient(value: Any, globalns: dict[str, Any] | None, localns: dict value = _make_forward_ref(value, is_argument=False, is_class=True) try: - return typing._eval_type(value, globalns, localns) # type: ignore + return eval_type_backport(value, globalns, localns) # type: ignore except NameError: # the point of this function is to be tolerant to this case return value +def node_to_ref(node: ast.AST): + if not isinstance(node, ast.Expression): + node = ast.copy_location(ast.Expression(node), node) + ref = typing.ForwardRef(ast.dump(node)) + ref.__forward_code__ = compile(node, '', 'eval') + return ref + + +class UnionTransformer(ast.NodeTransformer): + def __init__(self, globalns, localns): + self.typing_name = f'typing_{uuid.uuid4().hex}' + self.globalns = globalns + self.localns = {**(localns or {}), self.typing_name: typing} + + def eval_type(self, node): + return typing._eval_type(node_to_ref(node), self.globalns, self.localns) # type: ignore + + def visit_BinOp(self, node): + if isinstance(node.op, ast.BitOr): + left = self.visit(node.left) + right = self.visit(node.right) + left_val = self.eval_type(left) + right_val = self.eval_type(right) + try: + left_val | right_val # type: ignore + except TypeError: + replacement = ast.Subscript( + value=ast.Attribute( + value=ast.Name(id=self.typing_name, ctx=ast.Load()), + attr='Union', + ctx=ast.Load(), + ), + slice=ast.Index(value=ast.Tuple(elts=[left, right], ctx=ast.Load())), + ctx=ast.Load(), + ) + return ast.fix_missing_locations(replacement) + + return node + + +def eval_type_backport(value: Any, globalns: dict[str, Any] | None, localns: dict[str, Any] | None): + try: + return typing._eval_type(value, globalns, localns) # type: ignore + except TypeError: + if not isinstance(value, typing.ForwardRef): + raise + tree = ast.parse(value.__forward_arg__, mode='eval') + transformer = UnionTransformer(globalns, localns) + tree = transformer.visit(tree) + return transformer.eval_type(tree) + + def get_function_type_hints( function: Callable[..., Any], *, include_keys: set[str] | None = None, types_namespace: dict[str, Any] | None = None ) -> dict[str, Any]: @@ -248,7 +302,7 @@ def get_function_type_hints( elif isinstance(value, str): value = _make_forward_ref(value) - type_hints[name] = typing._eval_type(value, globalns, types_namespace) # type: ignore + type_hints[name] = eval_type_backport(value, globalns, types_namespace) # type: ignore return type_hints @@ -363,7 +417,7 @@ def get_type_hints( # noqa: C901 if isinstance(value, str): value = _make_forward_ref(value, is_argument=False, is_class=True) - value = typing._eval_type(value, base_globals, base_locals) # type: ignore + value = eval_type_backport(value, base_globals, base_locals) # type: ignore hints[name] = value return ( hints if include_extras else {k: typing._strip_annotations(t) for k, t in hints.items()} # type: ignore @@ -403,28 +457,13 @@ def get_type_hints( # noqa: C901 is_argument=not isinstance(obj, types.ModuleType), is_class=False, ) - value = typing._eval_type(value, globalns, localns) # type: ignore + value = eval_type_backport(value, globalns, localns) # type: ignore if name in defaults and defaults[name] is None: value = typing.Optional[value] hints[name] = value return hints if include_extras else {k: typing._strip_annotations(t) for k, t in hints.items()} # type: ignore -if sys.version_info < (3, 9): - - def evaluate_fwd_ref( - ref: ForwardRef, globalns: dict[str, Any] | None = None, localns: dict[str, Any] | None = None - ) -> Any: - return ref._evaluate(globalns=globalns, localns=localns) - -else: - - def evaluate_fwd_ref( - ref: ForwardRef, globalns: dict[str, Any] | None = None, localns: dict[str, Any] | None = None - ) -> Any: - return ref._evaluate(globalns=globalns, localns=localns, recursive_guard=frozenset()) - - def is_dataclass(_cls: type[Any]) -> TypeGuard[type[StandardDataclass]]: # The dataclasses.is_dataclass function doesn't seem to provide TypeGuard functionality, # so I created this convenience function From ca60caaad6f412ce7b5dd994b5126a83f0c7f1cd Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sun, 29 Oct 2023 23:01:06 +0200 Subject: [PATCH 02/31] Use modified get_type_hints in test_config --- tests/test_config.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index 8111a0d261..b2e3488350 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -4,7 +4,7 @@ from contextlib import nullcontext as does_not_raise from decimal import Decimal from inspect import signature -from typing import Any, ContextManager, Iterable, NamedTuple, Optional, Type, Union, get_type_hints +from typing import Any, ContextManager, Iterable, NamedTuple, Optional, Type, Union from dirty_equals import HasRepr, IsPartialDict from pydantic_core import SchemaError, SchemaSerializer, SchemaValidator @@ -24,6 +24,7 @@ ) from pydantic._internal._config import ConfigWrapper, config_defaults from pydantic._internal._mock_val_ser import MockValSer +from pydantic._internal._typing_extra import get_type_hints from pydantic.config import ConfigDict, JsonValue from pydantic.dataclasses import dataclass as pydantic_dataclass from pydantic.errors import PydanticUserError @@ -523,7 +524,7 @@ class Child(Mixin, Parent): assert Child.model_config.get('use_enum_values') is True -@pytest.mark.skipif(sys.version_info < (3, 10), reason='different on older versions') +@pytest.mark.skipif(sys.version_info < (3, 9), reason='different on older versions') def test_config_wrapper_match(): localns = {'_GenerateSchema': GenerateSchema, 'GenerateSchema': GenerateSchema, 'JsonValue': JsonValue} config_dict_annotations = [(k, str(v)) for k, v in get_type_hints(ConfigDict, localns=localns).items()] @@ -567,7 +568,7 @@ def check_foo(cls, v): assert src_exc.__notes__[0] == '\nPydantic: cause of loc: foo' -@pytest.mark.skipif(sys.version_info < (3, 10), reason='different on older versions') +@pytest.mark.skipif(sys.version_info < (3, 9), reason='different on older versions') def test_config_defaults_match(): localns = {'_GenerateSchema': GenerateSchema, 'GenerateSchema': GenerateSchema} config_dict_keys = sorted(list(get_type_hints(ConfigDict, localns=localns).keys())) From a163ea3f8c75ff1f5bf372cd7b6466926df8c581 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sun, 29 Oct 2023 23:22:04 +0200 Subject: [PATCH 03/31] unskip a couple more tests in older pythons --- tests/test_forward_ref.py | 1 - tests/test_main.py | 16 +++++++--------- tests/test_model_validator.py | 4 ++-- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/tests/test_forward_ref.py b/tests/test_forward_ref.py index dff4d7c4f1..63592bd2cc 100644 --- a/tests/test_forward_ref.py +++ b/tests/test_forward_ref.py @@ -711,7 +711,6 @@ class Foobar(BaseModel): assert f.y.model_fields_set == {'x'} -@pytest.mark.skipif(sys.version_info < (3, 10), reason='needs 3.10 or newer') def test_recursive_models_union(create_module): module = create_module( # language=Python diff --git a/tests/test_main.py b/tests/test_main.py index fc2060bf40..a5aa63b76f 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,7 +1,6 @@ import json import platform import re -import sys from collections import defaultdict from copy import deepcopy from dataclasses import dataclass @@ -1841,22 +1840,21 @@ class Model(BaseModel, some_config='new_value'): a: int -@pytest.mark.skipif(sys.version_info < (3, 10), reason='need 3.10 version') def test_new_union_origin(): """On 3.10+, origin of `int | str` is `types.UnionType`, not `typing.Union`""" class Model(BaseModel): - x: int | str + x: 'int | str' assert Model(x=3).x == 3 assert Model(x='3').x == '3' assert Model(x='pika').x == 'pika' - # assert Model.model_json_schema() == { - # 'title': 'Model', - # 'type': 'object', - # 'properties': {'x': {'title': 'X', 'anyOf': [{'type': 'integer'}, {'type': 'string'}]}}, - # 'required': ['x'], - # } + assert Model.model_json_schema() == { + 'title': 'Model', + 'type': 'object', + 'properties': {'x': {'title': 'X', 'anyOf': [{'type': 'integer'}, {'type': 'string'}]}}, + 'required': ['x'], + } @pytest.mark.parametrize( diff --git a/tests/test_model_validator.py b/tests/test_model_validator.py index e34d18c693..4d152051d0 100644 --- a/tests/test_model_validator.py +++ b/tests/test_model_validator.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Dict, Union, cast +from typing import Any, Dict, cast import pytest @@ -116,7 +116,7 @@ def test_nested_models() -> None: calls: list[str] = [] class Model(BaseModel): - inner: Union[Model, None] # noqa + inner: Model | None @model_validator(mode='before') @classmethod From 80335569cb18d702947f3fe6c4238ad53a834d6c Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sun, 29 Oct 2023 23:45:54 +0200 Subject: [PATCH 04/31] Use pipe operator in a bunch of tests --- pydantic/_internal/_typing_extra.py | 2 +- tests/test_dataclasses.py | 4 ++-- tests/test_discriminated_union.py | 32 ++++++++++++++--------------- tests/test_edge_cases.py | 18 ++++++++-------- tests/test_generics.py | 4 ++-- tests/test_json_schema.py | 4 ++-- tests/test_root_model.py | 6 +++--- tests/test_types.py | 12 +++++------ tests/test_typing.py | 5 ++++- tests/test_validators.py | 4 ++-- 10 files changed, 47 insertions(+), 44 deletions(-) diff --git a/pydantic/_internal/_typing_extra.py b/pydantic/_internal/_typing_extra.py index 15c174fea3..e7974a95f5 100644 --- a/pydantic/_internal/_typing_extra.py +++ b/pydantic/_internal/_typing_extra.py @@ -269,7 +269,7 @@ def visit_BinOp(self, node): return node -def eval_type_backport(value: Any, globalns: dict[str, Any] | None, localns: dict[str, Any] | None): +def eval_type_backport(value: Any, globalns: dict[str, Any] | None = None, localns: dict[str, Any] | None = None): try: return typing._eval_type(value, globalns, localns) # type: ignore except TypeError: diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index 7b363ad7e1..127689e326 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -8,7 +8,7 @@ from dataclasses import InitVar from datetime import date, datetime from pathlib import Path -from typing import Any, Callable, ClassVar, Dict, FrozenSet, Generic, List, Optional, Set, TypeVar, Union +from typing import Any, Callable, ClassVar, Dict, FrozenSet, Generic, List, Optional, Set, TypeVar import pytest from dirty_equals import HasRepr @@ -1359,7 +1359,7 @@ class B: @pydantic.dataclasses.dataclass class Top: - sub: Union[A, B] = dataclasses.field(metadata=dict(discriminator='l')) + sub: 'A | B' = dataclasses.field(metadata=dict(discriminator='l')) t = Top(sub=A(l='a')) assert isinstance(t, Top) diff --git a/tests/test_discriminated_union.py b/tests/test_discriminated_union.py index 251fee3d31..8406f38219 100644 --- a/tests/test_discriminated_union.py +++ b/tests/test_discriminated_union.py @@ -66,7 +66,7 @@ def test_discriminated_union_invalid_type(): ): class Model(BaseModel): - x: Union[str, int] = Field(..., discriminator='qwe') + x: 'str | int' = Field(..., discriminator='qwe') def test_discriminated_union_defined_discriminator(): @@ -80,7 +80,7 @@ class Dog(BaseModel): with pytest.raises(PydanticUserError, match="Model 'Cat' needs a discriminator field for key 'pet_type'"): class Model(BaseModel): - pet: Union[Cat, Dog] = Field(..., discriminator='pet_type') + pet: 'Cat | Dog' = Field(..., discriminator='pet_type') number: int @@ -96,7 +96,7 @@ class Dog(BaseModel): with pytest.raises(PydanticUserError, match="Model 'Cat' needs field 'pet_type' to be of type `Literal`"): class Model(BaseModel): - pet: Union[Cat, Dog] = Field(..., discriminator='pet_type') + pet: 'Cat | Dog' = Field(..., discriminator='pet_type') number: int @@ -331,7 +331,7 @@ class B(BaseModel): foo: Literal['b'] class Top(BaseModel): - sub: Union[A, B] = Field(..., discriminator='foo') + sub: 'A | B' = Field(..., discriminator='foo') t = Top(sub=A(foo='a')) assert isinstance(t, Top) @@ -346,7 +346,7 @@ class B(BaseModel): literal: Literal['b'] = Field(alias='lit') class Top(BaseModel): - sub: Union[A, B] = Field(..., discriminator='literal') + sub: 'A | B' = Field(..., discriminator='literal') with pytest.raises(ValidationError) as exc_info: Top(sub=A(literal='a')) @@ -367,7 +367,7 @@ class B(BaseModel): m: Literal[2] class Top(BaseModel): - sub: Union[A, B] = Field(..., discriminator='m') + sub: 'A | B' = Field(..., discriminator='m') assert isinstance(Top.model_validate({'sub': {'m': 2}}).sub, B) with pytest.raises(ValidationError) as exc_info: @@ -416,7 +416,7 @@ class B(BaseModel): m: Literal[EnumValue.b] class Top(BaseModel): - sub: Union[A, B] = Field(..., discriminator='m') + sub: 'A | B' = Field(..., discriminator='m') assert isinstance(Top.model_validate({'sub': {'m': EnumValue.b}}).sub, B) if isinstance(EnumValue.b, (int, str)): @@ -449,7 +449,7 @@ class Dog(BaseModel): with pytest.raises(TypeError, match=re.escape("Aliases for discriminator 'pet_type' must be the same (got T, U)")): class Model(BaseModel): - pet: Union[Cat, Dog] = Field(discriminator='pet_type') + pet: 'Cat | Dog' = Field(discriminator='pet_type') def test_alias_same(): @@ -462,7 +462,7 @@ class Dog(BaseModel): d: str class Model(BaseModel): - pet: Union[Cat, Dog] = Field(discriminator='pet_type') + pet: 'Cat | Dog' = Field(discriminator='pet_type') assert Model(**{'pet': {'typeOfPet': 'dog', 'd': 'milou'}}).pet.pet_type == 'dog' @@ -483,7 +483,7 @@ class Lizard(BaseModel): name: str class Model(BaseModel): - pet: Union[CommonPet, Lizard] = Field(..., discriminator='pet_type') + pet: 'CommonPet | Lizard' = Field(..., discriminator='pet_type') n: int assert isinstance(Model(**{'pet': {'pet_type': 'dog', 'name': 'Milou'}, 'n': 5}).pet, Dog) @@ -501,7 +501,7 @@ class Failure(BaseModel): error_message: str class Container(BaseModel, Generic[T]): - result: Union[Success[T], Failure] = Field(discriminator='type') + result: 'Success[T] | Failure' = Field(discriminator='type') with pytest.raises(ValidationError, match="Unable to extract tag using discriminator 'type'"): Container[str].model_validate({'result': {}}) @@ -545,7 +545,7 @@ class Dog(BaseModel): name: str class Pet(BaseModel): - pet: Optional[Union[Cat, Dog]] = Field(discriminator='pet_type') + pet: 'Cat | Dog | None' = Field(discriminator='pet_type') assert Pet(pet={'pet_type': 'cat', 'name': 'Milo'}).model_dump() == {'pet': {'name': 'Milo', 'pet_type': 'cat'}} assert Pet(pet={'pet_type': 'dog', 'name': 'Otis'}).model_dump() == {'pet': {'name': 'Otis', 'pet_type': 'dog'}} @@ -592,7 +592,7 @@ class Dog(BaseModel): name: str class Pet(BaseModel): - pet: Optional[Union[Cat, Dog]] = Field(default=None, discriminator='pet_type') + pet: 'Cat | Dog | None' = Field(default=None, discriminator='pet_type') assert Pet(pet={'pet_type': 'cat', 'name': 'Milo'}).model_dump() == {'pet': {'name': 'Milo', 'pet_type': 'cat'}} assert Pet(pet={'pet_type': 'dog', 'name': 'Otis'}).model_dump() == {'pet': {'name': 'Otis', 'pet_type': 'dog'}} @@ -634,7 +634,7 @@ class Case2(BaseModel): with pytest.raises(PydanticUserError, match="Model 'Case1' needs a discriminator field for key 'kind'"): class TaggedParent(BaseModel): - tagged: Union[Case1, Case2] = Field(discriminator='kind') + tagged: 'Case1 | Case2' = Field(discriminator='kind') def test_nested_optional_unions() -> None: @@ -651,7 +651,7 @@ class Lizard(BaseModel): MaybeDogLizard = Annotated[Union[Dog, Lizard, None], Field(discriminator='pet_type')] class Pet(BaseModel): - pet: Union[MaybeCatDog, MaybeDogLizard] = Field(discriminator='pet_type') + pet: 'MaybeCatDog | MaybeDogLizard' = Field(discriminator='pet_type') Pet.model_validate({'pet': {'pet_type': 'dog'}}) Pet.model_validate({'pet': {'pet_type': 'cat'}}) @@ -740,7 +740,7 @@ class Lizard(BaseModel): MaybeDogLizard = Annotated[Optional[Union[Dog, Lizard]], 'some other annotation'] class Model(BaseModel): - maybe_pet: Union[MaybeCat, MaybeDogLizard] = Field(discriminator='pet_type') + maybe_pet: 'MaybeCat | MaybeDogLizard' = Field(discriminator='pet_type') assert Model(**{'maybe_pet': None}).maybe_pet is None assert Model(**{'maybe_pet': {'typeOfPet': 'dog', 'd': 'milou'}}).maybe_pet.pet_type == 'dog' diff --git a/tests/test_edge_cases.py b/tests/test_edge_cases.py index 6a990ff989..53e5db307e 100644 --- a/tests/test_edge_cases.py +++ b/tests/test_edge_cases.py @@ -52,7 +52,7 @@ def test_str_bytes(): class Model(BaseModel): - v: Union[str, bytes] + v: 'str | bytes' m = Model(v='s') assert m.v == 's' @@ -72,7 +72,7 @@ class Model(BaseModel): def test_str_bytes_none(): class Model(BaseModel): - v: Union[None, str, bytes] = ... + v: 'None | str | bytes' = ... m = Model(v='s') assert m.v == 's' @@ -496,7 +496,7 @@ class Model(BaseModel): def test_list_unions(): class Model(BaseModel): - v: List[Union[int, str]] = ... + v: 'List[int | str]' = ... assert Model(v=[123, '456', 'foobar']).v == [123, '456', 'foobar'] @@ -511,7 +511,7 @@ class Model(BaseModel): def test_recursive_lists(): class Model(BaseModel): - v: List[List[Union[int, float]]] = ... + v: 'List[List[int | float]]' = ... assert Model(v=[[1, 2], [3, '4', '4.1']]).v == [[1, 2], [3, 4, 4.1]] assert Model.model_fields['v'].annotation == List[List[Union[int, float]]] @@ -1209,7 +1209,7 @@ class InvalidDefinitionModel(BaseModel): def test_multiple_errors(): class Model(BaseModel): - a: Union[None, int, float, Decimal] + a: 'None | int | float | Decimal' with pytest.raises(ValidationError) as exc_info: Model(a='foobar') @@ -1373,8 +1373,8 @@ class Model(BaseModel): e: Type[FooBar] f: Type[FooBar] = FooBar g: Sequence[Type[FooBar]] = [FooBar] - h: Union[Type[FooBar], Sequence[Type[FooBar]]] = FooBar - i: Union[Type[FooBar], Sequence[Type[FooBar]]] = [FooBar] + h: 'Type[FooBar] | Sequence[Type[FooBar]]' = FooBar + i: 'Type[FooBar] | Sequence[Type[FooBar]]' = [FooBar] model_config = dict(arbitrary_types_allowed=True) @@ -2555,8 +2555,8 @@ class Model(BaseModel): def test_type_union(): class Model(BaseModel): - a: Type[Union[str, bytes]] - b: Type[Union[Any, str]] + a: 'Type[str | bytes]' + b: 'Type[Any | str]' m = Model(a=bytes, b=int) assert m.model_dump() == {'a': bytes, 'b': int} diff --git a/tests/test_generics.py b/tests/test_generics.py index cd03b0f85f..b82096e07b 100644 --- a/tests/test_generics.py +++ b/tests/test_generics.py @@ -595,7 +595,7 @@ def test_complex_nesting(): T = TypeVar('T') class MyModel(BaseModel, Generic[T]): - item: List[Dict[Union[int, T], str]] + item: 'List[Dict[int | T, str]]' item = [{1: 'a', 'a': 'a'}] model = MyModel[str](item=item) @@ -1869,7 +1869,7 @@ class M1(BaseModel, Generic[V1, V2]): m: 'M2[V1]' class M2(BaseModel, Generic[V3]): - m: Union[M1[V3, int], V3] + m: 'M1[V3, int] | V3' M1.model_rebuild() diff --git a/tests/test_json_schema.py b/tests/test_json_schema.py index 675b52a4a4..741f6eaad0 100644 --- a/tests/test_json_schema.py +++ b/tests/test_json_schema.py @@ -1278,7 +1278,7 @@ class ModelWithOverride(BaseModel): ) def test_callable_type_with_fallback(default_value, properties): class Model(BaseModel): - callback: Union[int, Callable[[int], int]] = default_value + callback: 'int | Callable[[int], int]' = default_value class MyGenerator(GenerateJsonSchema): ignored_warning_kinds = () @@ -1331,7 +1331,7 @@ class Model(BaseModel): ) def test_callable_fallback_with_non_serializable_default(warning_match): class Model(BaseModel): - callback: Union[int, Callable[[int], int]] = lambda x: x # noqa E731 + callback: 'int | Callable[[int], int]' = lambda x: x # noqa E731 class MyGenerator(GenerateJsonSchema): ignored_warning_kinds = () diff --git a/tests/test_root_model.py b/tests/test_root_model.py index b3916ca31b..bdd0926e76 100644 --- a/tests/test_root_model.py +++ b/tests/test_root_model.py @@ -474,12 +474,12 @@ class RModel(RootModel): if order == 'BR': class Model(RootModel): - root: List[Union[BModel, RModel]] + root: 'List[BModel | RModel]' elif order == 'RB': class Model(RootModel): - root: List[Union[RModel, BModel]] + root: 'List[RModel | BModel]' m = Model([1, 2, {'value': 'abc'}]) @@ -508,7 +508,7 @@ class SModel(BaseModel): str_value: str class Model(RootModel): - root: Union[SModel, RModel] = Field(discriminator='kind') + root: 'SModel | RModel' = Field(discriminator='kind') assert Model(data).model_dump() == data assert Model(**data).model_dump() == data diff --git a/tests/test_types.py b/tests/test_types.py index 6720f4bd7d..35ec9bada1 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -4803,7 +4803,7 @@ class Model(BaseModel): def test_default_union_types(): class DefaultModel(BaseModel): - v: Union[int, bool, str] + v: 'int | bool | str' # do it this way since `1 == True` assert repr(DefaultModel(v=True).v) == 'True' @@ -4884,7 +4884,7 @@ class B(BaseModel): x: str class Model(BaseModel): - y: Union[A, B] + y: 'A | B' assert isinstance(Model(y=A(x='a')).y, A) assert isinstance(Model(y=B(x='b')).y, B) @@ -4896,7 +4896,7 @@ class MyStr(str): ... class Model(BaseModel): - x: Union[int, Annotated[str, Field(max_length=max_length)]] + x: 'int | Annotated[str, Field(max_length=max_length)]' v = Model(x=MyStr('1')).x assert type(v) is str @@ -4905,7 +4905,7 @@ class Model(BaseModel): def test_union_compound_types(): class Model(BaseModel): - values: Union[Dict[str, str], List[str], Dict[str, List[str]]] + values: 'Dict[str, str] | List[str] | Dict[str, List[str]]' assert Model(values={'L': '1'}).model_dump() == {'values': {'L': '1'}} assert Model(values=['L1']).model_dump() == {'values': ['L1']} @@ -4939,7 +4939,7 @@ class Model(BaseModel): def test_smart_union_compounded_types_edge_case(): class Model(BaseModel): - x: Union[List[str], List[int]] + x: 'List[str] | List[int]' assert Model(x=[1, 2]).x == [1, 2] assert Model(x=['1', '2']).x == ['1', '2'] @@ -4954,7 +4954,7 @@ class Dict2(TypedDict): bar: str class M(BaseModel): - d: Union[Dict2, Dict1] + d: 'Dict2 | Dict1' assert M(d=dict(foo='baz')).d == {'foo': 'baz'} diff --git a/tests/test_typing.py b/tests/test_typing.py index 7bd20af1e7..215b4ac8c6 100644 --- a/tests/test_typing.py +++ b/tests/test_typing.py @@ -9,6 +9,7 @@ from pydantic import Field # noqa: F401 from pydantic._internal._typing_extra import ( NoneType, + eval_type_backport, get_function_type_hints, is_classvar, is_literal_type, @@ -65,7 +66,9 @@ def test_is_none_type(): assert is_none_type(Callable) is False -@pytest.mark.parametrize('union_gen', [lambda: typing.Union[int, str], lambda: int | str]) +@pytest.mark.parametrize( + 'union_gen', [lambda: typing.Union[int, str], lambda: int | str, eval_type_backport('int | str')] +) def test_is_union(union_gen): try: union = union_gen() diff --git a/tests/test_validators.py b/tests/test_validators.py index 0fa80c13c9..50ac8d1fba 100644 --- a/tests/test_validators.py +++ b/tests/test_validators.py @@ -7,7 +7,7 @@ from enum import Enum from functools import partial, partialmethod from itertools import product -from typing import Any, Callable, Deque, Dict, FrozenSet, List, NamedTuple, Optional, Tuple, Union +from typing import Any, Callable, Deque, Dict, FrozenSet, List, NamedTuple, Optional, Tuple from unittest.mock import MagicMock import pytest @@ -1670,7 +1670,7 @@ class Model(BaseModel): def test_union_literal_with_constraints(): class Model(BaseModel, validate_assignment=True): - x: Union[Literal[42], Literal['pika']] = Field(frozen=True) + x: "Literal[42] | Literal['pika']" = Field(frozen=True) m = Model(x=42) with pytest.raises(ValidationError) as exc_info: From 0495f0f029add3ed4ba4d4e5d3d0f83adf62811c Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sat, 4 Nov 2023 13:03:49 +0200 Subject: [PATCH 05/31] various misc tidying up: use default localns=None, handle None values properly, clean up type ignore comments --- pydantic/_internal/_generate_schema.py | 2 +- pydantic/_internal/_typing_extra.py | 20 ++++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/pydantic/_internal/_generate_schema.py b/pydantic/_internal/_generate_schema.py index 3f4fa73b23..4a7d8f324c 100644 --- a/pydantic/_internal/_generate_schema.py +++ b/pydantic/_internal/_generate_schema.py @@ -637,7 +637,7 @@ def _resolve_forward_ref(self, obj: Any) -> Any: # class Model(BaseModel): # x: SomeImportedTypeAliasWithAForwardReference try: - obj = _typing_extra.eval_type_backport(obj, globalns=self._types_namespace, localns=None) + obj = _typing_extra.eval_type_backport(obj, globalns=self._types_namespace) except NameError as e: raise PydanticUndefinedAnnotation.from_name_error(e) from e diff --git a/pydantic/_internal/_typing_extra.py b/pydantic/_internal/_typing_extra.py index e7974a95f5..5ea322827c 100644 --- a/pydantic/_internal/_typing_extra.py +++ b/pydantic/_internal/_typing_extra.py @@ -223,7 +223,7 @@ def eval_type_lenient(value: Any, globalns: dict[str, Any] | None, localns: dict value = _make_forward_ref(value, is_argument=False, is_class=True) try: - return eval_type_backport(value, globalns, localns) # type: ignore + return eval_type_backport(value, globalns, localns) except NameError: # the point of this function is to be tolerant to this case return value @@ -238,13 +238,25 @@ def node_to_ref(node: ast.AST): class UnionTransformer(ast.NodeTransformer): - def __init__(self, globalns, localns): + def __init__(self, globalns: dict[str, Any] | None, localns: dict[str, Any] | None): + # This logic for handling Nones is copied from typing.ForwardRef._evaluate + if globalns is None and localns is None: + globalns = localns = {} + elif globalns is None: + assert localns is not None + globalns = localns + elif localns is None: + assert globalns is not None + localns = globalns + self.typing_name = f'typing_{uuid.uuid4().hex}' self.globalns = globalns - self.localns = {**(localns or {}), self.typing_name: typing} + self.localns = {**localns, self.typing_name: typing} def eval_type(self, node): - return typing._eval_type(node_to_ref(node), self.globalns, self.localns) # type: ignore + return typing._eval_type( # type: ignore + node_to_ref(node), self.globalns, self.localns + ) def visit_BinOp(self, node): if isinstance(node.op, ast.BitOr): From 780b46afcd218cae54f662130d10ca1bcfd74c71 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sat, 4 Nov 2023 13:04:45 +0200 Subject: [PATCH 06/31] explain asserts --- pydantic/_internal/_typing_extra.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pydantic/_internal/_typing_extra.py b/pydantic/_internal/_typing_extra.py index 5ea322827c..f1624e3491 100644 --- a/pydantic/_internal/_typing_extra.py +++ b/pydantic/_internal/_typing_extra.py @@ -243,10 +243,10 @@ def __init__(self, globalns: dict[str, Any] | None, localns: dict[str, Any] | No if globalns is None and localns is None: globalns = localns = {} elif globalns is None: - assert localns is not None + assert localns is not None # apparently pyright doesn't infer this automatically globalns = localns elif localns is None: - assert globalns is not None + assert globalns is not None # apparently pyright doesn't infer this automatically localns = globalns self.typing_name = f'typing_{uuid.uuid4().hex}' From 4c536b6dd67f466a92719e1e24fbd886e92ce4e7 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sat, 4 Nov 2023 13:08:01 +0200 Subject: [PATCH 07/31] type hints --- pydantic/_internal/_typing_extra.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pydantic/_internal/_typing_extra.py b/pydantic/_internal/_typing_extra.py index f1624e3491..b8576261fd 100644 --- a/pydantic/_internal/_typing_extra.py +++ b/pydantic/_internal/_typing_extra.py @@ -229,7 +229,7 @@ def eval_type_lenient(value: Any, globalns: dict[str, Any] | None, localns: dict return value -def node_to_ref(node: ast.AST): +def node_to_ref(node: ast.AST) -> typing.ForwardRef: if not isinstance(node, ast.Expression): node = ast.copy_location(ast.Expression(node), node) ref = typing.ForwardRef(ast.dump(node)) @@ -253,12 +253,12 @@ def __init__(self, globalns: dict[str, Any] | None, localns: dict[str, Any] | No self.globalns = globalns self.localns = {**localns, self.typing_name: typing} - def eval_type(self, node): + def eval_type(self, node: ast.AST) -> Any: return typing._eval_type( # type: ignore node_to_ref(node), self.globalns, self.localns ) - def visit_BinOp(self, node): + def visit_BinOp(self, node) -> ast.BinOp | ast.Subscript: if isinstance(node.op, ast.BitOr): left = self.visit(node.left) right = self.visit(node.right) @@ -267,6 +267,7 @@ def visit_BinOp(self, node): try: left_val | right_val # type: ignore except TypeError: + # Replace `left | right` with `Union[left, right]` replacement = ast.Subscript( value=ast.Attribute( value=ast.Name(id=self.typing_name, ctx=ast.Load()), @@ -281,9 +282,13 @@ def visit_BinOp(self, node): return node -def eval_type_backport(value: Any, globalns: dict[str, Any] | None = None, localns: dict[str, Any] | None = None): +def eval_type_backport( + value: Any, globalns: dict[str, Any] | None = None, localns: dict[str, Any] | None = None +) -> Any: try: - return typing._eval_type(value, globalns, localns) # type: ignore + return typing._eval_type( # type: ignore + value, globalns, localns + ) except TypeError: if not isinstance(value, typing.ForwardRef): raise From d7f874b0cddc34afd350443f2c9271dc61a32537 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sat, 4 Nov 2023 13:15:04 +0200 Subject: [PATCH 08/31] inline node_to_ref --- pydantic/_internal/_typing_extra.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/pydantic/_internal/_typing_extra.py b/pydantic/_internal/_typing_extra.py index b8576261fd..55260f0129 100644 --- a/pydantic/_internal/_typing_extra.py +++ b/pydantic/_internal/_typing_extra.py @@ -229,14 +229,6 @@ def eval_type_lenient(value: Any, globalns: dict[str, Any] | None, localns: dict return value -def node_to_ref(node: ast.AST) -> typing.ForwardRef: - if not isinstance(node, ast.Expression): - node = ast.copy_location(ast.Expression(node), node) - ref = typing.ForwardRef(ast.dump(node)) - ref.__forward_code__ = compile(node, '', 'eval') - return ref - - class UnionTransformer(ast.NodeTransformer): def __init__(self, globalns: dict[str, Any] | None, localns: dict[str, Any] | None): # This logic for handling Nones is copied from typing.ForwardRef._evaluate @@ -254,8 +246,12 @@ def __init__(self, globalns: dict[str, Any] | None, localns: dict[str, Any] | No self.localns = {**localns, self.typing_name: typing} def eval_type(self, node: ast.AST) -> Any: + if not isinstance(node, ast.Expression): + node = ast.copy_location(ast.Expression(node), node) + ref = typing.ForwardRef(ast.dump(node)) + ref.__forward_code__ = compile(node, '', 'eval') return typing._eval_type( # type: ignore - node_to_ref(node), self.globalns, self.localns + ref, self.globalns, self.localns ) def visit_BinOp(self, node) -> ast.BinOp | ast.Subscript: From 09d2e93ecf576ca24fef8ee34bc1252817b03fed Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sat, 4 Nov 2023 13:17:55 +0200 Subject: [PATCH 09/31] is_unsupported_types_for_union_error --- pydantic/_internal/_typing_extra.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pydantic/_internal/_typing_extra.py b/pydantic/_internal/_typing_extra.py index 55260f0129..f5e0a6a37d 100644 --- a/pydantic/_internal/_typing_extra.py +++ b/pydantic/_internal/_typing_extra.py @@ -229,6 +229,10 @@ def eval_type_lenient(value: Any, globalns: dict[str, Any] | None, localns: dict return value +def is_unsupported_types_for_union_error(e: TypeError): + return str(e).startswith('unsupported operand type(s) for |: ') + + class UnionTransformer(ast.NodeTransformer): def __init__(self, globalns: dict[str, Any] | None, localns: dict[str, Any] | None): # This logic for handling Nones is copied from typing.ForwardRef._evaluate @@ -262,7 +266,9 @@ def visit_BinOp(self, node) -> ast.BinOp | ast.Subscript: right_val = self.eval_type(right) try: left_val | right_val # type: ignore - except TypeError: + except TypeError as e: + if not is_unsupported_types_for_union_error(e): + raise # Replace `left | right` with `Union[left, right]` replacement = ast.Subscript( value=ast.Attribute( @@ -285,8 +291,8 @@ def eval_type_backport( return typing._eval_type( # type: ignore value, globalns, localns ) - except TypeError: - if not isinstance(value, typing.ForwardRef): + except TypeError as e: + if not (isinstance(value, typing.ForwardRef) and is_unsupported_types_for_union_error(e)): raise tree = ast.parse(value.__forward_arg__, mode='eval') transformer = UnionTransformer(globalns, localns) From d7b14621203dacbaeefca13e1c9f4da8b73dad2c Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sat, 4 Nov 2023 13:18:51 +0200 Subject: [PATCH 10/31] tidying --- pydantic/_internal/_typing_extra.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pydantic/_internal/_typing_extra.py b/pydantic/_internal/_typing_extra.py index f5e0a6a37d..e3756db039 100644 --- a/pydantic/_internal/_typing_extra.py +++ b/pydantic/_internal/_typing_extra.py @@ -229,7 +229,7 @@ def eval_type_lenient(value: Any, globalns: dict[str, Any] | None, localns: dict return value -def is_unsupported_types_for_union_error(e: TypeError): +def is_unsupported_types_for_union_error(e: TypeError) -> bool: return str(e).startswith('unsupported operand type(s) for |: ') @@ -265,7 +265,7 @@ def visit_BinOp(self, node) -> ast.BinOp | ast.Subscript: left_val = self.eval_type(left) right_val = self.eval_type(right) try: - left_val | right_val # type: ignore + _ = left_val | right_val except TypeError as e: if not is_unsupported_types_for_union_error(e): raise From c65ae3dc92444c06236d9902d9542a0a8a31152f Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sat, 4 Nov 2023 13:19:54 +0200 Subject: [PATCH 11/31] remove more type: ignore comments --- pydantic/_internal/_typing_extra.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pydantic/_internal/_typing_extra.py b/pydantic/_internal/_typing_extra.py index e3756db039..c26771569a 100644 --- a/pydantic/_internal/_typing_extra.py +++ b/pydantic/_internal/_typing_extra.py @@ -321,7 +321,7 @@ def get_function_type_hints( elif isinstance(value, str): value = _make_forward_ref(value) - type_hints[name] = eval_type_backport(value, globalns, types_namespace) # type: ignore + type_hints[name] = eval_type_backport(value, globalns, types_namespace) return type_hints @@ -436,7 +436,7 @@ def get_type_hints( # noqa: C901 if isinstance(value, str): value = _make_forward_ref(value, is_argument=False, is_class=True) - value = eval_type_backport(value, base_globals, base_locals) # type: ignore + value = eval_type_backport(value, base_globals, base_locals) hints[name] = value return ( hints if include_extras else {k: typing._strip_annotations(t) for k, t in hints.items()} # type: ignore @@ -476,7 +476,7 @@ def get_type_hints( # noqa: C901 is_argument=not isinstance(obj, types.ModuleType), is_class=False, ) - value = eval_type_backport(value, globalns, localns) # type: ignore + value = eval_type_backport(value, globalns, localns) if name in defaults and defaults[name] is None: value = typing.Optional[value] hints[name] = value From 3aa88da7245f0b80b358ad1557ee69caceeb327a Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sat, 4 Nov 2023 13:33:52 +0200 Subject: [PATCH 12/31] docstrings and tidying --- pydantic/_internal/_typing_extra.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/pydantic/_internal/_typing_extra.py b/pydantic/_internal/_typing_extra.py index c26771569a..8c1dec985f 100644 --- a/pydantic/_internal/_typing_extra.py +++ b/pydantic/_internal/_typing_extra.py @@ -233,7 +233,9 @@ def is_unsupported_types_for_union_error(e: TypeError) -> bool: return str(e).startswith('unsupported operand type(s) for |: ') -class UnionTransformer(ast.NodeTransformer): +class _UnionTransformer(ast.NodeTransformer): + """Transforms `X | Y` into `Union[X, Y]` if `X | Y` is not supported.""" + def __init__(self, globalns: dict[str, Any] | None, localns: dict[str, Any] | None): # This logic for handling Nones is copied from typing.ForwardRef._evaluate if globalns is None and localns is None: @@ -260,23 +262,23 @@ def eval_type(self, node: ast.AST) -> Any: def visit_BinOp(self, node) -> ast.BinOp | ast.Subscript: if isinstance(node.op, ast.BitOr): - left = self.visit(node.left) - right = self.visit(node.right) - left_val = self.eval_type(left) - right_val = self.eval_type(right) + left_node = self.visit(node.left) + right_node = self.visit(node.right) + left_val = self.eval_type(left_node) + right_val = self.eval_type(right_node) try: _ = left_val | right_val except TypeError as e: if not is_unsupported_types_for_union_error(e): raise - # Replace `left | right` with `Union[left, right]` + # Replace `left | right` with `typing.Union[left, right]` replacement = ast.Subscript( value=ast.Attribute( value=ast.Name(id=self.typing_name, ctx=ast.Load()), attr='Union', ctx=ast.Load(), ), - slice=ast.Index(value=ast.Tuple(elts=[left, right], ctx=ast.Load())), + slice=ast.Index(value=ast.Tuple(elts=[left_node, right_node], ctx=ast.Load())), ctx=ast.Load(), ) return ast.fix_missing_locations(replacement) @@ -287,6 +289,10 @@ def visit_BinOp(self, node) -> ast.BinOp | ast.Subscript: def eval_type_backport( value: Any, globalns: dict[str, Any] | None = None, localns: dict[str, Any] | None = None ) -> Any: + """Like `typing._eval_type`, but lets older Python versions use newer typing features. + Currently this just means that `X | Y` is converted to `Union[X, Y]` if `X | Y` is not supported. + This would also be the place to add support for `list[int]` instead of `List[int]` etc. + """ try: return typing._eval_type( # type: ignore value, globalns, localns @@ -295,7 +301,7 @@ def eval_type_backport( if not (isinstance(value, typing.ForwardRef) and is_unsupported_types_for_union_error(e)): raise tree = ast.parse(value.__forward_arg__, mode='eval') - transformer = UnionTransformer(globalns, localns) + transformer = _UnionTransformer(globalns, localns) tree = transformer.visit(tree) return transformer.eval_type(tree) From ca09a03ba6c1ea22b98db40f815ac0f364473a93 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sat, 4 Nov 2023 13:39:12 +0200 Subject: [PATCH 13/31] fix and tighten test_is_union --- tests/test_typing.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_typing.py b/tests/test_typing.py index 215b4ac8c6..74a028bf2a 100644 --- a/tests/test_typing.py +++ b/tests/test_typing.py @@ -67,16 +67,16 @@ def test_is_none_type(): @pytest.mark.parametrize( - 'union_gen', [lambda: typing.Union[int, str], lambda: int | str, eval_type_backport('int | str')] + 'union', + [ + typing.Union[int, str], + eval_type_backport(typing.ForwardRef('int | str')), + *([int | str] if sys.version_info >= (3, 10) else []), + ], ) -def test_is_union(union_gen): - try: - union = union_gen() - except TypeError: - pytest.skip('not supported in this python version') - else: - origin = get_origin(union) - assert origin_is_union(origin) +def test_is_union(union): + origin = get_origin(union) + assert origin_is_union(origin) def test_is_literal_with_typing_extension_literal(): From c2d9d229542325fbbca9ff036396a24f1458cc20 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Wed, 22 Nov 2023 21:01:03 +0200 Subject: [PATCH 14/31] Use `eval_type_backport` package --- pydantic/_internal/_typing_extra.py | 65 +++-------------------------- 1 file changed, 5 insertions(+), 60 deletions(-) diff --git a/pydantic/_internal/_typing_extra.py b/pydantic/_internal/_typing_extra.py index d3c3af6f2f..f2de562c85 100644 --- a/pydantic/_internal/_typing_extra.py +++ b/pydantic/_internal/_typing_extra.py @@ -1,12 +1,10 @@ """Logic for interacting with type annotations, mostly extensions, shims and hacks to wrap python's typing module.""" from __future__ import annotations as _annotations -import ast import dataclasses import sys import types import typing -import uuid from collections.abc import Callable from functools import partial from types import GetSetDescriptorType @@ -233,63 +231,11 @@ def is_unsupported_types_for_union_error(e: TypeError) -> bool: return str(e).startswith('unsupported operand type(s) for |: ') -class _UnionTransformer(ast.NodeTransformer): - """Transforms `X | Y` into `Union[X, Y]` if `X | Y` is not supported.""" - - def __init__(self, globalns: dict[str, Any] | None, localns: dict[str, Any] | None): - # This logic for handling Nones is copied from typing.ForwardRef._evaluate - if globalns is None and localns is None: - globalns = localns = {} - elif globalns is None: - assert localns is not None # apparently pyright doesn't infer this automatically - globalns = localns - elif localns is None: - assert globalns is not None # apparently pyright doesn't infer this automatically - localns = globalns - - self.typing_name = f'typing_{uuid.uuid4().hex}' - self.globalns = globalns - self.localns = {**localns, self.typing_name: typing} - - def eval_type(self, node: ast.AST) -> Any: - if not isinstance(node, ast.Expression): - node = ast.copy_location(ast.Expression(node), node) - ref = typing.ForwardRef(ast.dump(node)) - ref.__forward_code__ = compile(node, '', 'eval') - return typing._eval_type( # type: ignore - ref, self.globalns, self.localns - ) - - def visit_BinOp(self, node) -> ast.BinOp | ast.Subscript: - if isinstance(node.op, ast.BitOr): - left_node = self.visit(node.left) - right_node = self.visit(node.right) - left_val = self.eval_type(left_node) - right_val = self.eval_type(right_node) - try: - _ = left_val | right_val - except TypeError as e: - if not is_unsupported_types_for_union_error(e): - raise - # Replace `left | right` with `typing.Union[left, right]` - replacement = ast.Subscript( - value=ast.Attribute( - value=ast.Name(id=self.typing_name, ctx=ast.Load()), - attr='Union', - ctx=ast.Load(), - ), - slice=ast.Index(value=ast.Tuple(elts=[left_node, right_node], ctx=ast.Load())), - ctx=ast.Load(), - ) - return ast.fix_missing_locations(replacement) - - return node - - def eval_type_backport( value: Any, globalns: dict[str, Any] | None = None, localns: dict[str, Any] | None = None ) -> Any: - """Like `typing._eval_type`, but lets older Python versions use newer typing features. + """Like `typing._eval_type`, but falls back to the `eval_type_backport` package if it's + installed to let older Python versions use newer typing features. Currently this just means that `X | Y` is converted to `Union[X, Y]` if `X | Y` is not supported. This would also be the place to add support for `list[int]` instead of `List[int]` etc. """ @@ -300,10 +246,9 @@ def eval_type_backport( except TypeError as e: if not (isinstance(value, typing.ForwardRef) and is_unsupported_types_for_union_error(e)): raise - tree = ast.parse(value.__forward_arg__, mode='eval') - transformer = _UnionTransformer(globalns, localns) - tree = transformer.visit(tree) - return transformer.eval_type(tree) + from eval_type_backport import eval_type_backport + + return eval_type_backport(value, globalns, localns, try_default=False) def get_function_type_hints( From 40a41fb745c854ee35d1f651e007553403cdb13b Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Wed, 22 Nov 2023 21:06:27 +0200 Subject: [PATCH 15/31] Add test dependency --- pdm.lock | 12 +++++- pyproject.toml | 105 +++++++++++++++++++++++++------------------------ 2 files changed, 65 insertions(+), 52 deletions(-) diff --git a/pdm.lock b/pdm.lock index d37751aac0..a09de9af52 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "docs", "email", "linting", "memray", "mypy", "testing", "testing-extra"] strategy = ["cross_platform"] lock_version = "4.4" -content_hash = "sha256:bfa326e0a46425e1cdaf0d38b8442a10a0a70771a0a7251918eb4b83fecb2298" +content_hash = "sha256:b7cfab0c90511e67f931fe8b7ee13ea82e396405edbd11057bde7f0f41fa0082" [[package]] name = "annotated-types" @@ -461,6 +461,16 @@ files = [ {file = "email_validator-2.1.0.post1.tar.gz", hash = "sha256:a4b0bd1cf55f073b924258d19321b1f3aa74b4b5a71a42c305575dba920e1a44"}, ] +[[package]] +name = "eval-type-backport" +version = "0.0.1" +requires_python = ">=3.7" +summary = "Like `typing._eval_type`, but lets older Python versions use newer typing features." +files = [ + {file = "eval_type_backport-0.0.1-py3-none-any.whl", hash = "sha256:3aa0d990e642af9d8f9480c84dcaeabaaef83207bdcf5cb018706db6bb77d692"}, + {file = "eval_type_backport-0.0.1.tar.gz", hash = "sha256:3328c532f0d159f84756b11bd932ff2b5f93a7c605a603af02de0e38566c86df"}, +] + [[package]] name = "exceptiongroup" version = "1.1.3" diff --git a/pyproject.toml b/pyproject.toml index 9e50d85c19..50af2fc4bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,57 +20,6 @@ include = [ '/requirements', ] -[project] -name = 'pydantic' -description = 'Data validation using Python type hints' -authors = [ - {name = 'Samuel Colvin', email = 's@muelcolvin.com'}, - {name = 'Eric Jolibois', email = 'em.jolibois@gmail.com'}, - {name = 'Hasan Ramezani', email = 'hasan.r67@gmail.com'}, - {name = 'Adrian Garcia Badaracco', email = '1755071+adriangb@users.noreply.github.com'}, - {name = 'Terrence Dorsey', email = 'terry@pydantic.dev'}, - {name = 'David Montague', email = 'david@pydantic.dev'}, - {name = 'Serge Matveenko', email = 'lig@countzero.co'}, - {name = 'Marcelo Trylesinski', email = 'marcelotryle@gmail.com'}, - {name = 'Sydney Runkle', email = 'sydneymarierunkle@gmail.com'}, - {name = 'David Hewitt', email = 'mail@davidhewitt.io'}, -] -license = 'MIT' -classifiers = [ - 'Development Status :: 5 - Production/Stable', - 'Programming Language :: Python', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3 :: Only', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Intended Audience :: Developers', - 'Intended Audience :: Information Technology', - 'Intended Audience :: System Administrators', - 'License :: OSI Approved :: MIT License', - 'Operating System :: Unix', - 'Operating System :: POSIX :: Linux', - 'Environment :: Console', - 'Environment :: MacOS X', - 'Framework :: Hypothesis', - 'Framework :: Pydantic', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Topic :: Internet', -] -requires-python = '>=3.8' -dependencies = [ - 'typing-extensions>=4.6.1', - 'annotated-types>=0.4.0', - "pydantic-core==2.14.3", -] -dynamic = ['version', 'readme'] - -[project.optional-dependencies] -email = ['email-validator>=2.0.0'] [tool.pdm.dev-dependencies] docs = [ @@ -105,6 +54,7 @@ testing = [ "faker>=18.13.0", "pytest-benchmark>=4.0.0", "pytest-codspeed~=2.2.0", + "eval-type-backport>=0.0.1", ] testing-extra = [ # used when generate devtools docs example @@ -126,6 +76,58 @@ memray = [ # requires Python > 3.8, we only test with 3.8 in CI but because of it won't lock properly pytest-memray = "1.5.0" +[project] +name = 'pydantic' +description = 'Data validation using Python type hints' +authors = [ + {name = 'Samuel Colvin', email = 's@muelcolvin.com'}, + {name = 'Eric Jolibois', email = 'em.jolibois@gmail.com'}, + {name = 'Hasan Ramezani', email = 'hasan.r67@gmail.com'}, + {name = 'Adrian Garcia Badaracco', email = '1755071+adriangb@users.noreply.github.com'}, + {name = 'Terrence Dorsey', email = 'terry@pydantic.dev'}, + {name = 'David Montague', email = 'david@pydantic.dev'}, + {name = 'Serge Matveenko', email = 'lig@countzero.co'}, + {name = 'Marcelo Trylesinski', email = 'marcelotryle@gmail.com'}, + {name = 'Sydney Runkle', email = 'sydneymarierunkle@gmail.com'}, + {name = 'David Hewitt', email = 'mail@davidhewitt.io'}, +] +license = 'MIT' +classifiers = [ + 'Development Status :: 5 - Production/Stable', + 'Programming Language :: Python', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Intended Audience :: Developers', + 'Intended Audience :: Information Technology', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: MIT License', + 'Operating System :: Unix', + 'Operating System :: POSIX :: Linux', + 'Environment :: Console', + 'Environment :: MacOS X', + 'Framework :: Hypothesis', + 'Framework :: Pydantic', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Topic :: Internet', +] +requires-python = '>=3.8' +dependencies = [ + 'typing-extensions>=4.6.1', + 'annotated-types>=0.4.0', + "pydantic-core==2.14.3", +] +dynamic = ['version', 'readme'] + +[project.optional-dependencies] +email = ['email-validator>=2.0.0'] + [project.urls] Homepage = 'https://github.com/pydantic/pydantic' Documentation = 'https://docs.pydantic.dev' @@ -133,6 +135,7 @@ Funding = 'https://github.com/sponsors/samuelcolvin' Source = 'https://github.com/pydantic/pydantic' Changelog = 'https://docs.pydantic.dev/latest/changelog/' + [tool.hatch.metadata.hooks.fancy-pypi-readme] content-type = 'text/markdown' # construct the PyPI readme from README.md and HISTORY.md From a998d14016beb413d76446981e21fd3969e34c08 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sat, 16 Dec 2023 20:41:06 +0200 Subject: [PATCH 16/31] upgrade eval_type_backport --- pdm.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pdm.lock b/pdm.lock index cc6881fb78..fb9ab50530 100644 --- a/pdm.lock +++ b/pdm.lock @@ -4,8 +4,8 @@ [metadata] groups = ["default", "docs", "email", "linting", "memray", "mypy", "testing", "testing-extra"] strategy = ["cross_platform"] -lock_version = "4.4" -content_hash = "sha256:66e5649881b398968ef855f6b7d41831abd2d7436ded8dbb8ee7f95a1c11513d" +lock_version = "4.4.1" +content_hash = "sha256:9bbdb835acfbe55679d45362084a31c74c2acc1ddcde7dcaacafc941f27f431f" [[package]] name = "annotated-types" @@ -463,12 +463,12 @@ files = [ [[package]] name = "eval-type-backport" -version = "0.0.1" +version = "0.1.0" requires_python = ">=3.7" summary = "Like `typing._eval_type`, but lets older Python versions use newer typing features." files = [ - {file = "eval_type_backport-0.0.1-py3-none-any.whl", hash = "sha256:3aa0d990e642af9d8f9480c84dcaeabaaef83207bdcf5cb018706db6bb77d692"}, - {file = "eval_type_backport-0.0.1.tar.gz", hash = "sha256:3328c532f0d159f84756b11bd932ff2b5f93a7c605a603af02de0e38566c86df"}, + {file = "eval_type_backport-0.1.0-py3-none-any.whl", hash = "sha256:9e63ef53a47b7d3d7236a75954fe55bb283564b6c5daccc76792090800862cd2"}, + {file = "eval_type_backport-0.1.0.tar.gz", hash = "sha256:fd811b6d52ec526adb777391ebd525b156b126883ff85af1ac1719c624ad38f6"}, ] [[package]] @@ -1633,7 +1633,7 @@ version = "2.0.23" requires_python = ">=3.7" summary = "Database Abstraction Library" dependencies = [ - "greenlet!=0.4.17; platform_machine == \"aarch64\" or (platform_machine == \"ppc64le\" or (platform_machine == \"x86_64\" or (platform_machine == \"amd64\" or (platform_machine == \"AMD64\" or (platform_machine == \"win32\" or platform_machine == \"WIN32\")))))", + "greenlet!=0.4.17; platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\"", "typing-extensions>=4.2.0", ] files = [ From 4f465a3cd6019867125c2e088b6a4469d2651833 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sat, 16 Dec 2023 20:49:20 +0200 Subject: [PATCH 17/31] fix pdm.lock --- pdm.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pdm.lock b/pdm.lock index fb9ab50530..151650daf4 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "docs", "email", "linting", "memray", "mypy", "testing", "testing-extra"] strategy = ["cross_platform"] lock_version = "4.4.1" -content_hash = "sha256:9bbdb835acfbe55679d45362084a31c74c2acc1ddcde7dcaacafc941f27f431f" +content_hash = "sha256:a38392aee7393d62239cad56e4fe7326d4dd669424a704719bd22ca959a6bdd1" [[package]] name = "annotated-types" @@ -1633,7 +1633,7 @@ version = "2.0.23" requires_python = ">=3.7" summary = "Database Abstraction Library" dependencies = [ - "greenlet!=0.4.17; platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\"", + "greenlet!=0.4.17; platform_machine == \"aarch64\" or (platform_machine == \"ppc64le\" or (platform_machine == \"x86_64\" or (platform_machine == \"amd64\" or (platform_machine == \"AMD64\" or (platform_machine == \"win32\" or platform_machine == \"WIN32\")))))", "typing-extensions>=4.2.0", ] files = [ From 71ca7cf58bb98916cff5cdff8c08e84d0acf75e3 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sat, 16 Dec 2023 21:28:02 +0200 Subject: [PATCH 18/31] upgrade eval_type_backport to handle fussy typing._type_check --- pdm.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pdm.lock b/pdm.lock index 151650daf4..f7c56ca736 100644 --- a/pdm.lock +++ b/pdm.lock @@ -463,12 +463,12 @@ files = [ [[package]] name = "eval-type-backport" -version = "0.1.0" +version = "0.1.1" requires_python = ">=3.7" summary = "Like `typing._eval_type`, but lets older Python versions use newer typing features." files = [ - {file = "eval_type_backport-0.1.0-py3-none-any.whl", hash = "sha256:9e63ef53a47b7d3d7236a75954fe55bb283564b6c5daccc76792090800862cd2"}, - {file = "eval_type_backport-0.1.0.tar.gz", hash = "sha256:fd811b6d52ec526adb777391ebd525b156b126883ff85af1ac1719c624ad38f6"}, + {file = "eval_type_backport-0.1.1-py3-none-any.whl", hash = "sha256:34e49d3a2ed0b24ed329a8a8753748dc8ea04884360743adb119673ddd7133ce"}, + {file = "eval_type_backport-0.1.1.tar.gz", hash = "sha256:8bd16b4612c0ce86560260c93b0c7f58701a07a8129670c0b52c184b74ff3906"}, ] [[package]] From f0608761b64a5d559ddd353fe0d88746696ae363 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sat, 16 Dec 2023 21:34:35 +0200 Subject: [PATCH 19/31] update is_backport_fixable_error and move down, update eval_type_backport function docstring --- pydantic/_internal/_typing_extra.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/pydantic/_internal/_typing_extra.py b/pydantic/_internal/_typing_extra.py index f2de562c85..f0959411c6 100644 --- a/pydantic/_internal/_typing_extra.py +++ b/pydantic/_internal/_typing_extra.py @@ -227,30 +227,32 @@ def eval_type_lenient(value: Any, globalns: dict[str, Any] | None, localns: dict return value -def is_unsupported_types_for_union_error(e: TypeError) -> bool: - return str(e).startswith('unsupported operand type(s) for |: ') - - def eval_type_backport( value: Any, globalns: dict[str, Any] | None = None, localns: dict[str, Any] | None = None ) -> Any: """Like `typing._eval_type`, but falls back to the `eval_type_backport` package if it's installed to let older Python versions use newer typing features. - Currently this just means that `X | Y` is converted to `Union[X, Y]` if `X | Y` is not supported. - This would also be the place to add support for `list[int]` instead of `List[int]` etc. + Specifically, this transforms `X | Y` into `typing.Union[X, Y]` + and `list[X]` into `typing.List[X]` etc. (for all the types made generic in PEP 585) + if the original syntax is not supported in the current Python version. """ try: return typing._eval_type( # type: ignore value, globalns, localns ) except TypeError as e: - if not (isinstance(value, typing.ForwardRef) and is_unsupported_types_for_union_error(e)): + if not (isinstance(value, typing.ForwardRef) and is_backport_fixable_error(e)): raise from eval_type_backport import eval_type_backport return eval_type_backport(value, globalns, localns, try_default=False) +def is_backport_fixable_error(e: TypeError) -> bool: + msg = str(e) + return msg.startswith('unsupported operand type(s) for |: ') or "' object is not subscriptable" in msg + + def get_function_type_hints( function: Callable[..., Any], *, include_keys: set[str] | None = None, types_namespace: dict[str, Any] | None = None ) -> dict[str, Any]: From 2b42d65c166db999be3dd75ecdb72cbf00e42c65 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sun, 17 Dec 2023 15:53:49 +0200 Subject: [PATCH 20/31] raise helpful error if eval_type_backport isn't installed. ensure test_recursive_models_union passes without backport --- pydantic/_internal/_typing_extra.py | 8 +++++++- tests/test_forward_ref.py | 18 +++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/pydantic/_internal/_typing_extra.py b/pydantic/_internal/_typing_extra.py index f0959411c6..318233fb04 100644 --- a/pydantic/_internal/_typing_extra.py +++ b/pydantic/_internal/_typing_extra.py @@ -243,7 +243,13 @@ def eval_type_backport( except TypeError as e: if not (isinstance(value, typing.ForwardRef) and is_backport_fixable_error(e)): raise - from eval_type_backport import eval_type_backport + try: + from eval_type_backport import eval_type_backport + except ImportError: + raise RuntimeError( + 'In order to use newer typing features on older Python versions, ' + 'you need to install the `eval_type_backport` package.' + ) from e return eval_type_backport(value, globalns, localns, try_default=False) diff --git a/tests/test_forward_ref.py b/tests/test_forward_ref.py index 63592bd2cc..4f30ba9c46 100644 --- a/tests/test_forward_ref.py +++ b/tests/test_forward_ref.py @@ -712,9 +712,14 @@ class Foobar(BaseModel): def test_recursive_models_union(create_module): - module = create_module( - # language=Python - """ + # This test should pass because PydanticRecursiveRef.__or__ is implemented, + # not because `eval_type_backport` magically makes `|` work, + # since it's installed for tests but otherwise optional. + sys.modules['eval_type_backport'] = None # type: ignore + try: + module = create_module( + # language=Python + """ from __future__ import annotations from pydantic import BaseModel @@ -728,8 +733,11 @@ class Foo(BaseModel): class Bar(BaseModel, Generic[T]): foo: Foo -""" - ) + """ + ) + finally: + del sys.modules['eval_type_backport'] + assert module.Foo.model_fields['bar'].annotation == typing.Optional[module.Bar[str]] assert module.Foo.model_fields['bar2'].annotation == typing.Union[int, module.Bar[float]] assert module.Bar.model_fields['foo'].annotation == module.Foo From dcbdd289a5ec80b8b2d7c999a7f0118621b2e26c Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sun, 17 Dec 2023 16:04:41 +0200 Subject: [PATCH 21/31] Restore skip, add another test for combination of backport and PydanticRecursiveRef.__or__ --- tests/test_forward_ref.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/test_forward_ref.py b/tests/test_forward_ref.py index 4f30ba9c46..2431c26a9a 100644 --- a/tests/test_forward_ref.py +++ b/tests/test_forward_ref.py @@ -711,6 +711,7 @@ class Foobar(BaseModel): assert f.y.model_fields_set == {'x'} +@pytest.mark.skipif(sys.version_info < (3, 10), reason='needs 3.10 or newer') def test_recursive_models_union(create_module): # This test should pass because PydanticRecursiveRef.__or__ is implemented, # not because `eval_type_backport` magically makes `|` work, @@ -743,6 +744,34 @@ class Bar(BaseModel, Generic[T]): assert module.Bar.model_fields['foo'].annotation == module.Foo +def test_recursive_models_union_backport(create_module): + module = create_module( + # language=Python + """ +from __future__ import annotations + +from pydantic import BaseModel +from typing import TypeVar, Generic + +T = TypeVar("T") + +class Foo(BaseModel): + bar: Bar[str] | None = None + # The `int | str` here differs from the previous test and requires the backport. + # At the same time, `PydanticRecursiveRef.__or__` means that the second `|` works normally, + # which actually triggered a bug in the backport that needed fixing. + bar2: int | str | Bar[float] + +class Bar(BaseModel, Generic[T]): + foo: Foo +""" + ) + + assert module.Foo.model_fields['bar'].annotation == typing.Optional[module.Bar[str]] + assert module.Foo.model_fields['bar2'].annotation == typing.Union[int, str, module.Bar[float]] + assert module.Bar.model_fields['foo'].annotation == module.Foo + + def test_force_rebuild(): class Foobar(BaseModel): b: int From 959c755ddfe7d228d575bbe93613009149a9f5b5 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sun, 17 Dec 2023 17:56:35 +0200 Subject: [PATCH 22/31] Test that eval_type_backport is being called in the right places --- tests/test_main.py | 36 ++++++++++++++++++++++++++++++++++++ tests/test_type_adapter.py | 28 ++++++++++++++++++++++++++++ tests/test_validate_call.py | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+) diff --git a/tests/test_main.py b/tests/test_main.py index a144338c76..895dca617f 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -3125,3 +3125,39 @@ class Three(One): assert getattr(One, 'foo', None) == ' edited!' assert getattr(Two, 'foo', None) == ' edited! edited!' assert getattr(Three, 'foo', None) == ' edited! edited!' + + +def test_eval_type_backport(): + class Model(BaseModel): + foo: 'list[int | str]' + + assert Model(foo=[1, '2']).model_dump() == {'foo': [1, '2']} + + with pytest.raises(ValidationError) as exc_info: + Model(foo='not a list') + # insert_assert(exc_info.value.errors(include_url=False)) + assert exc_info.value.errors(include_url=False) == [ + { + 'type': 'list_type', + 'loc': ('foo',), + 'msg': 'Input should be a valid list', + 'input': 'not a list', + } + ] + with pytest.raises(ValidationError) as exc_info: + Model(foo=[{'not a str or int'}]) + # insert_assert(exc_info.value.errors(include_url=False)) + assert exc_info.value.errors(include_url=False) == [ + { + 'type': 'int_type', + 'loc': ('foo', 0, 'int'), + 'msg': 'Input should be a valid integer', + 'input': {'not a str or int'}, + }, + { + 'type': 'string_type', + 'loc': ('foo', 0, 'str'), + 'msg': 'Input should be a valid string', + 'input': {'not a str or int'}, + }, + ] diff --git a/tests/test_type_adapter.py b/tests/test_type_adapter.py index 13c5b5d02f..04e38d4f14 100644 --- a/tests/test_type_adapter.py +++ b/tests/test_type_adapter.py @@ -344,3 +344,31 @@ class TestSerializer(BaseModel): assert TypeAdapter(Annotated[list[TestSerializer], ...]).dump_python([result], mode='json') == [ {'some_bytes': 'qg=='} ] + + +def test_eval_type_backport(): + v = TypeAdapter('list[int | str]').validate_python + assert v([1, '2']) == [1, '2'] + with pytest.raises(ValidationError) as exc_info: + v([{'not a str or int'}]) + # insert_assert(exc_info.value.errors(include_url=False)) + assert exc_info.value.errors(include_url=False) == [ + { + 'type': 'int_type', + 'loc': (0, 'int'), + 'msg': 'Input should be a valid integer', + 'input': {'not a str or int'}, + }, + { + 'type': 'string_type', + 'loc': (0, 'str'), + 'msg': 'Input should be a valid string', + 'input': {'not a str or int'}, + }, + ] + with pytest.raises(ValidationError) as exc_info: + v('not a list') + # insert_assert(exc_info.value.errors(include_url=False)) + assert exc_info.value.errors(include_url=False) == [ + {'type': 'list_type', 'loc': (), 'msg': 'Input should be a valid list', 'input': 'not a list'} + ] diff --git a/tests/test_validate_call.py b/tests/test_validate_call.py index 16f4de2654..a30f1f8b29 100644 --- a/tests/test_validate_call.py +++ b/tests/test_validate_call.py @@ -758,3 +758,39 @@ def some_static_method(x: str) -> str: assert c.some_instance_method == c.some_instance_method assert c.some_class_method == c.some_class_method assert c.some_static_method == c.some_static_method + + +def test_eval_type_backport(): + @validate_call + def foo(bar: 'list[int | str]') -> 'list[int | str]': + return bar + + assert foo([1, '2']) == [1, '2'] + with pytest.raises(ValidationError) as exc_info: + foo('not a list') # type: ignore + # insert_assert(exc_info.value.errors(include_url=False)) + assert exc_info.value.errors(include_url=False) == [ + { + 'type': 'list_type', + 'loc': (0,), + 'msg': 'Input should be a valid list', + 'input': 'not a list', + } + ] + with pytest.raises(ValidationError) as exc_info: + foo([{'not a str or int'}]) # type: ignore + # insert_assert(exc_info.value.errors(include_url=False)) + assert exc_info.value.errors(include_url=False) == [ + { + 'type': 'int_type', + 'loc': (0, 0, 'int'), + 'msg': 'Input should be a valid integer', + 'input': {'not a str or int'}, + }, + { + 'type': 'string_type', + 'loc': (0, 0, 'str'), + 'msg': 'Input should be a valid string', + 'input': {'not a str or int'}, + }, + ] From 71c912e450b1c55339a73d3bbb5871c25813b9f3 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sun, 17 Dec 2023 18:17:57 +0200 Subject: [PATCH 23/31] test calling backport from get_type_hints --- tests/test_types_namedtuple.py | 39 ++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/test_types_namedtuple.py b/tests/test_types_namedtuple.py index 92302f4bca..3774558275 100644 --- a/tests/test_types_namedtuple.py +++ b/tests/test_types_namedtuple.py @@ -208,3 +208,42 @@ class NT(NamedTuple): assert TypeAdapter(NT).validate_python([1]) == (1, 33) assert TypeAdapter(NT).validate_python({'x': 22}) == (22, 33) + + +def test_eval_type_backport(): + class MyNamedTuple(NamedTuple): + foo: 'list[int | str]' + + class Model(BaseModel): + t: MyNamedTuple + + assert Model(t=([1, '2'],)).model_dump() == {'t': ([1, '2'],)} + + with pytest.raises(ValidationError) as exc_info: + Model(t=('not a list',)) + # insert_assert(exc_info.value.errors(include_url=False)) + assert exc_info.value.errors(include_url=False) == [ + { + 'type': 'list_type', + 'loc': ('t', 0), + 'msg': 'Input should be a valid list', + 'input': 'not a list', + } + ] + with pytest.raises(ValidationError) as exc_info: + Model(t=([{'not a str or int'}],)) + # insert_assert(exc_info.value.errors(include_url=False)) + assert exc_info.value.errors(include_url=False) == [ + { + 'type': 'int_type', + 'loc': ('t', 0, 0, 'int'), + 'msg': 'Input should be a valid integer', + 'input': {'not a str or int'}, + }, + { + 'type': 'string_type', + 'loc': ('t', 0, 0, 'str'), + 'msg': 'Input should be a valid string', + 'input': {'not a str or int'}, + }, + ] From 984705898a500b1d5dabdb7cfc32306f11c2a760 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sun, 17 Dec 2023 18:25:42 +0200 Subject: [PATCH 24/31] upgrade eval_type_backport to handle working union operator --- pdm.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pdm.lock b/pdm.lock index f7c56ca736..bcd3908eb0 100644 --- a/pdm.lock +++ b/pdm.lock @@ -463,12 +463,12 @@ files = [ [[package]] name = "eval-type-backport" -version = "0.1.1" +version = "0.1.2" requires_python = ">=3.7" summary = "Like `typing._eval_type`, but lets older Python versions use newer typing features." files = [ - {file = "eval_type_backport-0.1.1-py3-none-any.whl", hash = "sha256:34e49d3a2ed0b24ed329a8a8753748dc8ea04884360743adb119673ddd7133ce"}, - {file = "eval_type_backport-0.1.1.tar.gz", hash = "sha256:8bd16b4612c0ce86560260c93b0c7f58701a07a8129670c0b52c184b74ff3906"}, + {file = "eval_type_backport-0.1.2-py3-none-any.whl", hash = "sha256:c9bd0d8b057581c91c0cee3c5c3848eebe7e97ec1d12f7d316c98070eb0006d8"}, + {file = "eval_type_backport-0.1.2.tar.gz", hash = "sha256:4c81b0cbd848037be6ba4d82e5b0e021e38a8ed988b8c96b8c591b35b1f10266"}, ] [[package]] From 6beab5bfcfd43bcdd60ba5375efc9d85021aebb6 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sun, 17 Dec 2023 18:44:02 +0200 Subject: [PATCH 25/31] unskip tests that can now pass in 3.8 --- pydantic/_internal/_typing_extra.py | 10 +++++++--- tests/test_config.py | 2 -- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pydantic/_internal/_typing_extra.py b/pydantic/_internal/_typing_extra.py index 318233fb04..2369a7f74a 100644 --- a/pydantic/_internal/_typing_extra.py +++ b/pydantic/_internal/_typing_extra.py @@ -397,9 +397,13 @@ def get_type_hints( # noqa: C901 value = eval_type_backport(value, base_globals, base_locals) hints[name] = value - return ( - hints if include_extras else {k: typing._strip_annotations(t) for k, t in hints.items()} # type: ignore - ) + if not include_extras and hasattr(typing, '_strip_annotations'): + return { + k: typing._strip_annotations(t) # type: ignore + for k, t in hints.items() + } + else: + return hints if globalns is None: if isinstance(obj, types.ModuleType): diff --git a/tests/test_config.py b/tests/test_config.py index b2e3488350..a9a46432ee 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -524,7 +524,6 @@ class Child(Mixin, Parent): assert Child.model_config.get('use_enum_values') is True -@pytest.mark.skipif(sys.version_info < (3, 9), reason='different on older versions') def test_config_wrapper_match(): localns = {'_GenerateSchema': GenerateSchema, 'GenerateSchema': GenerateSchema, 'JsonValue': JsonValue} config_dict_annotations = [(k, str(v)) for k, v in get_type_hints(ConfigDict, localns=localns).items()] @@ -568,7 +567,6 @@ def check_foo(cls, v): assert src_exc.__notes__[0] == '\nPydantic: cause of loc: foo' -@pytest.mark.skipif(sys.version_info < (3, 9), reason='different on older versions') def test_config_defaults_match(): localns = {'_GenerateSchema': GenerateSchema, 'GenerateSchema': GenerateSchema} config_dict_keys = sorted(list(get_type_hints(ConfigDict, localns=localns).keys())) From b79692b7f65b1d39cec544a8d5f29dfe98111ba7 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sun, 17 Dec 2023 18:52:17 +0200 Subject: [PATCH 26/31] revert scattered test changes --- tests/test_dataclasses.py | 4 ++-- tests/test_discriminated_union.py | 32 +++++++++++++++---------------- tests/test_edge_cases.py | 18 ++++++++--------- tests/test_generics.py | 4 ++-- tests/test_json_schema.py | 4 ++-- tests/test_root_model.py | 6 +++--- tests/test_types.py | 12 ++++++------ tests/test_validators.py | 4 ++-- 8 files changed, 42 insertions(+), 42 deletions(-) diff --git a/tests/test_dataclasses.py b/tests/test_dataclasses.py index f8ef00197c..a1e4291a44 100644 --- a/tests/test_dataclasses.py +++ b/tests/test_dataclasses.py @@ -8,7 +8,7 @@ from dataclasses import InitVar from datetime import date, datetime from pathlib import Path -from typing import Any, Callable, ClassVar, Dict, FrozenSet, Generic, List, Optional, Set, TypeVar +from typing import Any, Callable, ClassVar, Dict, FrozenSet, Generic, List, Optional, Set, TypeVar, Union import pytest from dirty_equals import HasRepr @@ -1346,7 +1346,7 @@ class B: @pydantic.dataclasses.dataclass class Top: - sub: 'A | B' = dataclasses.field(metadata=dict(discriminator='l')) + sub: Union[A, B] = dataclasses.field(metadata=dict(discriminator='l')) t = Top(sub=A(l='a')) assert isinstance(t, Top) diff --git a/tests/test_discriminated_union.py b/tests/test_discriminated_union.py index 8406f38219..251fee3d31 100644 --- a/tests/test_discriminated_union.py +++ b/tests/test_discriminated_union.py @@ -66,7 +66,7 @@ def test_discriminated_union_invalid_type(): ): class Model(BaseModel): - x: 'str | int' = Field(..., discriminator='qwe') + x: Union[str, int] = Field(..., discriminator='qwe') def test_discriminated_union_defined_discriminator(): @@ -80,7 +80,7 @@ class Dog(BaseModel): with pytest.raises(PydanticUserError, match="Model 'Cat' needs a discriminator field for key 'pet_type'"): class Model(BaseModel): - pet: 'Cat | Dog' = Field(..., discriminator='pet_type') + pet: Union[Cat, Dog] = Field(..., discriminator='pet_type') number: int @@ -96,7 +96,7 @@ class Dog(BaseModel): with pytest.raises(PydanticUserError, match="Model 'Cat' needs field 'pet_type' to be of type `Literal`"): class Model(BaseModel): - pet: 'Cat | Dog' = Field(..., discriminator='pet_type') + pet: Union[Cat, Dog] = Field(..., discriminator='pet_type') number: int @@ -331,7 +331,7 @@ class B(BaseModel): foo: Literal['b'] class Top(BaseModel): - sub: 'A | B' = Field(..., discriminator='foo') + sub: Union[A, B] = Field(..., discriminator='foo') t = Top(sub=A(foo='a')) assert isinstance(t, Top) @@ -346,7 +346,7 @@ class B(BaseModel): literal: Literal['b'] = Field(alias='lit') class Top(BaseModel): - sub: 'A | B' = Field(..., discriminator='literal') + sub: Union[A, B] = Field(..., discriminator='literal') with pytest.raises(ValidationError) as exc_info: Top(sub=A(literal='a')) @@ -367,7 +367,7 @@ class B(BaseModel): m: Literal[2] class Top(BaseModel): - sub: 'A | B' = Field(..., discriminator='m') + sub: Union[A, B] = Field(..., discriminator='m') assert isinstance(Top.model_validate({'sub': {'m': 2}}).sub, B) with pytest.raises(ValidationError) as exc_info: @@ -416,7 +416,7 @@ class B(BaseModel): m: Literal[EnumValue.b] class Top(BaseModel): - sub: 'A | B' = Field(..., discriminator='m') + sub: Union[A, B] = Field(..., discriminator='m') assert isinstance(Top.model_validate({'sub': {'m': EnumValue.b}}).sub, B) if isinstance(EnumValue.b, (int, str)): @@ -449,7 +449,7 @@ class Dog(BaseModel): with pytest.raises(TypeError, match=re.escape("Aliases for discriminator 'pet_type' must be the same (got T, U)")): class Model(BaseModel): - pet: 'Cat | Dog' = Field(discriminator='pet_type') + pet: Union[Cat, Dog] = Field(discriminator='pet_type') def test_alias_same(): @@ -462,7 +462,7 @@ class Dog(BaseModel): d: str class Model(BaseModel): - pet: 'Cat | Dog' = Field(discriminator='pet_type') + pet: Union[Cat, Dog] = Field(discriminator='pet_type') assert Model(**{'pet': {'typeOfPet': 'dog', 'd': 'milou'}}).pet.pet_type == 'dog' @@ -483,7 +483,7 @@ class Lizard(BaseModel): name: str class Model(BaseModel): - pet: 'CommonPet | Lizard' = Field(..., discriminator='pet_type') + pet: Union[CommonPet, Lizard] = Field(..., discriminator='pet_type') n: int assert isinstance(Model(**{'pet': {'pet_type': 'dog', 'name': 'Milou'}, 'n': 5}).pet, Dog) @@ -501,7 +501,7 @@ class Failure(BaseModel): error_message: str class Container(BaseModel, Generic[T]): - result: 'Success[T] | Failure' = Field(discriminator='type') + result: Union[Success[T], Failure] = Field(discriminator='type') with pytest.raises(ValidationError, match="Unable to extract tag using discriminator 'type'"): Container[str].model_validate({'result': {}}) @@ -545,7 +545,7 @@ class Dog(BaseModel): name: str class Pet(BaseModel): - pet: 'Cat | Dog | None' = Field(discriminator='pet_type') + pet: Optional[Union[Cat, Dog]] = Field(discriminator='pet_type') assert Pet(pet={'pet_type': 'cat', 'name': 'Milo'}).model_dump() == {'pet': {'name': 'Milo', 'pet_type': 'cat'}} assert Pet(pet={'pet_type': 'dog', 'name': 'Otis'}).model_dump() == {'pet': {'name': 'Otis', 'pet_type': 'dog'}} @@ -592,7 +592,7 @@ class Dog(BaseModel): name: str class Pet(BaseModel): - pet: 'Cat | Dog | None' = Field(default=None, discriminator='pet_type') + pet: Optional[Union[Cat, Dog]] = Field(default=None, discriminator='pet_type') assert Pet(pet={'pet_type': 'cat', 'name': 'Milo'}).model_dump() == {'pet': {'name': 'Milo', 'pet_type': 'cat'}} assert Pet(pet={'pet_type': 'dog', 'name': 'Otis'}).model_dump() == {'pet': {'name': 'Otis', 'pet_type': 'dog'}} @@ -634,7 +634,7 @@ class Case2(BaseModel): with pytest.raises(PydanticUserError, match="Model 'Case1' needs a discriminator field for key 'kind'"): class TaggedParent(BaseModel): - tagged: 'Case1 | Case2' = Field(discriminator='kind') + tagged: Union[Case1, Case2] = Field(discriminator='kind') def test_nested_optional_unions() -> None: @@ -651,7 +651,7 @@ class Lizard(BaseModel): MaybeDogLizard = Annotated[Union[Dog, Lizard, None], Field(discriminator='pet_type')] class Pet(BaseModel): - pet: 'MaybeCatDog | MaybeDogLizard' = Field(discriminator='pet_type') + pet: Union[MaybeCatDog, MaybeDogLizard] = Field(discriminator='pet_type') Pet.model_validate({'pet': {'pet_type': 'dog'}}) Pet.model_validate({'pet': {'pet_type': 'cat'}}) @@ -740,7 +740,7 @@ class Lizard(BaseModel): MaybeDogLizard = Annotated[Optional[Union[Dog, Lizard]], 'some other annotation'] class Model(BaseModel): - maybe_pet: 'MaybeCat | MaybeDogLizard' = Field(discriminator='pet_type') + maybe_pet: Union[MaybeCat, MaybeDogLizard] = Field(discriminator='pet_type') assert Model(**{'maybe_pet': None}).maybe_pet is None assert Model(**{'maybe_pet': {'typeOfPet': 'dog', 'd': 'milou'}}).maybe_pet.pet_type == 'dog' diff --git a/tests/test_edge_cases.py b/tests/test_edge_cases.py index acaf6baff7..84127dbefe 100644 --- a/tests/test_edge_cases.py +++ b/tests/test_edge_cases.py @@ -53,7 +53,7 @@ def test_str_bytes(): class Model(BaseModel): - v: 'str | bytes' + v: Union[str, bytes] m = Model(v='s') assert m.v == 's' @@ -73,7 +73,7 @@ class Model(BaseModel): def test_str_bytes_none(): class Model(BaseModel): - v: 'None | str | bytes' = ... + v: Union[None, str, bytes] = ... m = Model(v='s') assert m.v == 's' @@ -497,7 +497,7 @@ class Model(BaseModel): def test_list_unions(): class Model(BaseModel): - v: 'List[int | str]' = ... + v: List[Union[int, str]] = ... assert Model(v=[123, '456', 'foobar']).v == [123, '456', 'foobar'] @@ -512,7 +512,7 @@ class Model(BaseModel): def test_recursive_lists(): class Model(BaseModel): - v: 'List[List[int | float]]' = ... + v: List[List[Union[int, float]]] = ... assert Model(v=[[1, 2], [3, '4', '4.1']]).v == [[1, 2], [3, 4, 4.1]] assert Model.model_fields['v'].annotation == List[List[Union[int, float]]] @@ -1210,7 +1210,7 @@ class InvalidDefinitionModel(BaseModel): def test_multiple_errors(): class Model(BaseModel): - a: 'None | int | float | Decimal' + a: Union[None, int, float, Decimal] with pytest.raises(ValidationError) as exc_info: Model(a='foobar') @@ -1382,8 +1382,8 @@ class Model(BaseModel): e: Type[FooBar] f: Type[FooBar] = FooBar g: Sequence[Type[FooBar]] = [FooBar] - h: 'Type[FooBar] | Sequence[Type[FooBar]]' = FooBar - i: 'Type[FooBar] | Sequence[Type[FooBar]]' = [FooBar] + h: Union[Type[FooBar], Sequence[Type[FooBar]]] = FooBar + i: Union[Type[FooBar], Sequence[Type[FooBar]]] = [FooBar] model_config = dict(arbitrary_types_allowed=True) @@ -2564,8 +2564,8 @@ class Model(BaseModel): def test_type_union(): class Model(BaseModel): - a: 'Type[str | bytes]' - b: 'Type[Any | str]' + a: Type[Union[str, bytes]] + b: Type[Union[Any, str]] m = Model(a=bytes, b=int) assert m.model_dump() == {'a': bytes, 'b': int} diff --git a/tests/test_generics.py b/tests/test_generics.py index 521cf5a19e..001e818b1a 100644 --- a/tests/test_generics.py +++ b/tests/test_generics.py @@ -595,7 +595,7 @@ def test_complex_nesting(): T = TypeVar('T') class MyModel(BaseModel, Generic[T]): - item: 'List[Dict[int | T, str]]' + item: List[Dict[Union[int, T], str]] item = [{1: 'a', 'a': 'a'}] model = MyModel[str](item=item) @@ -1869,7 +1869,7 @@ class M1(BaseModel, Generic[V1, V2]): m: 'M2[V1]' class M2(BaseModel, Generic[V3]): - m: 'M1[V3, int] | V3' + m: Union[M1[V3, int], V3] M1.model_rebuild() diff --git a/tests/test_json_schema.py b/tests/test_json_schema.py index 56d53dafdb..c8042e1615 100644 --- a/tests/test_json_schema.py +++ b/tests/test_json_schema.py @@ -1279,7 +1279,7 @@ class ModelWithOverride(BaseModel): ) def test_callable_type_with_fallback(default_value, properties): class Model(BaseModel): - callback: 'int | Callable[[int], int]' = default_value + callback: Union[int, Callable[[int], int]] = default_value class MyGenerator(GenerateJsonSchema): ignored_warning_kinds = () @@ -1332,7 +1332,7 @@ class Model(BaseModel): ) def test_callable_fallback_with_non_serializable_default(warning_match): class Model(BaseModel): - callback: 'int | Callable[[int], int]' = lambda x: x # noqa E731 + callback: Union[int, Callable[[int], int]] = lambda x: x # noqa E731 class MyGenerator(GenerateJsonSchema): ignored_warning_kinds = () diff --git a/tests/test_root_model.py b/tests/test_root_model.py index 1b16a2a48a..d25a3706a8 100644 --- a/tests/test_root_model.py +++ b/tests/test_root_model.py @@ -474,12 +474,12 @@ class RModel(RootModel): if order == 'BR': class Model(RootModel): - root: 'List[BModel | RModel]' + root: List[Union[BModel, RModel]] elif order == 'RB': class Model(RootModel): - root: 'List[RModel | BModel]' + root: List[Union[RModel, BModel]] m = Model([1, 2, {'value': 'abc'}]) @@ -508,7 +508,7 @@ class SModel(BaseModel): str_value: str class Model(RootModel): - root: 'SModel | RModel' = Field(discriminator='kind') + root: Union[SModel, RModel] = Field(discriminator='kind') assert Model(data).model_dump() == data assert Model(**data).model_dump() == data diff --git a/tests/test_types.py b/tests/test_types.py index 6bb68b6945..780a681a7f 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -4828,7 +4828,7 @@ class Model(BaseModel): def test_default_union_types(): class DefaultModel(BaseModel): - v: 'int | bool | str' + v: Union[int, bool, str] # do it this way since `1 == True` assert repr(DefaultModel(v=True).v) == 'True' @@ -4909,7 +4909,7 @@ class B(BaseModel): x: str class Model(BaseModel): - y: 'A | B' + y: Union[A, B] assert isinstance(Model(y=A(x='a')).y, A) assert isinstance(Model(y=B(x='b')).y, B) @@ -4921,7 +4921,7 @@ class MyStr(str): ... class Model(BaseModel): - x: 'int | Annotated[str, Field(max_length=max_length)]' + x: Union[int, Annotated[str, Field(max_length=max_length)]] v = Model(x=MyStr('1')).x assert type(v) is str @@ -4930,7 +4930,7 @@ class Model(BaseModel): def test_union_compound_types(): class Model(BaseModel): - values: 'Dict[str, str] | List[str] | Dict[str, List[str]]' + values: Union[Dict[str, str], List[str], Dict[str, List[str]]] assert Model(values={'L': '1'}).model_dump() == {'values': {'L': '1'}} assert Model(values=['L1']).model_dump() == {'values': ['L1']} @@ -4964,7 +4964,7 @@ class Model(BaseModel): def test_smart_union_compounded_types_edge_case(): class Model(BaseModel): - x: 'List[str] | List[int]' + x: Union[List[str], List[int]] assert Model(x=[1, 2]).x == [1, 2] assert Model(x=['1', '2']).x == ['1', '2'] @@ -4979,7 +4979,7 @@ class Dict2(TypedDict): bar: str class M(BaseModel): - d: 'Dict2 | Dict1' + d: Union[Dict2, Dict1] assert M(d=dict(foo='baz')).d == {'foo': 'baz'} diff --git a/tests/test_validators.py b/tests/test_validators.py index f38891c376..96c7941d5d 100644 --- a/tests/test_validators.py +++ b/tests/test_validators.py @@ -7,7 +7,7 @@ from enum import Enum from functools import partial, partialmethod from itertools import product -from typing import Any, Callable, Deque, Dict, FrozenSet, List, NamedTuple, Optional, Tuple +from typing import Any, Callable, Deque, Dict, FrozenSet, List, NamedTuple, Optional, Tuple, Union from unittest.mock import MagicMock import pytest @@ -1670,7 +1670,7 @@ class Model(BaseModel): def test_union_literal_with_constraints(): class Model(BaseModel, validate_assignment=True): - x: "Literal[42] | Literal['pika']" = Field(frozen=True) + x: Union[Literal[42], Literal['pika']] = Field(frozen=True) m = Model(x=42) with pytest.raises(ValidationError) as exc_info: From 6bc0ee45347bc168fa6629a7c8efd1170afc661f Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sun, 17 Dec 2023 19:20:44 +0200 Subject: [PATCH 27/31] unskip more tests --- pydantic/_internal/_generate_schema.py | 4 ++-- pydantic/_internal/_typing_extra.py | 2 +- pydantic/fields.py | 2 +- tests/test_edge_cases.py | 27 ++++++++++++-------------- tests/test_forward_ref.py | 23 ++++++++++------------ tests/test_typing.py | 4 ++-- tests/test_validate_call.py | 4 ---- tests/test_validators.py | 5 ----- 8 files changed, 28 insertions(+), 43 deletions(-) diff --git a/pydantic/_internal/_generate_schema.py b/pydantic/_internal/_generate_schema.py index 999d77e5d1..6ef5bd2be6 100644 --- a/pydantic/_internal/_generate_schema.py +++ b/pydantic/_internal/_generate_schema.py @@ -989,7 +989,7 @@ def _common_field_schema( # C901 # Ensure that typevars get mapped to their concrete types: types_namespace.update({k.__name__: v for k, v in self._typevars_map.items()}) - evaluated = _typing_extra.eval_type_lenient(field_info.annotation, types_namespace, None) + evaluated = _typing_extra.eval_type_lenient(field_info.annotation, types_namespace) if evaluated is not field_info.annotation and not has_instance_in_type(evaluated, PydanticRecursiveRef): field_info.annotation = evaluated @@ -1118,7 +1118,7 @@ def _type_alias_type_schema( self._types_namespace = new_namespace typevars_map = get_standard_typevars_map(obj) - annotation = _typing_extra.eval_type_lenient(annotation, self._types_namespace, None) + annotation = _typing_extra.eval_type_lenient(annotation, self._types_namespace) annotation = replace_types(annotation, typevars_map) schema = self.generate_schema(annotation) assert schema['type'] != 'definitions' diff --git a/pydantic/_internal/_typing_extra.py b/pydantic/_internal/_typing_extra.py index 2369a7f74a..c4b456c954 100644 --- a/pydantic/_internal/_typing_extra.py +++ b/pydantic/_internal/_typing_extra.py @@ -213,7 +213,7 @@ def get_cls_type_hints_lenient(obj: Any, globalns: dict[str, Any] | None = None) return hints -def eval_type_lenient(value: Any, globalns: dict[str, Any] | None, localns: dict[str, Any] | None) -> Any: +def eval_type_lenient(value: Any, globalns: dict[str, Any] | None = None, localns: dict[str, Any] | None = None) -> Any: """Behaves like typing._eval_type, except it won't raise an error if a forward reference can't be resolved.""" if value is None: value = NoneType diff --git a/pydantic/fields.py b/pydantic/fields.py index 849215b3a1..bb11ec7b77 100644 --- a/pydantic/fields.py +++ b/pydantic/fields.py @@ -539,7 +539,7 @@ def apply_typevars_map(self, typevars_map: dict[Any, Any] | None, types_namespac pydantic._internal._generics.replace_types is used for replacing the typevars with their concrete types. """ - annotation = _typing_extra.eval_type_lenient(self.annotation, types_namespace, None) + annotation = _typing_extra.eval_type_lenient(self.annotation, types_namespace) self.annotation = _generics.replace_types(annotation, typevars_map) def __repr_args__(self) -> ReprArgs: diff --git a/tests/test_edge_cases.py b/tests/test_edge_cases.py index 84127dbefe..306867cbc3 100644 --- a/tests/test_edge_cases.py +++ b/tests/test_edge_cases.py @@ -303,7 +303,6 @@ class Model(BaseModel): (dict, frozenset, list, set, tuple, type), ], ) -@pytest.mark.skipif(sys.version_info < (3, 9), reason='PEP585 generics only supported for python 3.9 and above') def test_pep585_generic_types(dict_cls, frozenset_cls, list_cls, set_cls, tuple_cls, type_cls): class Type1: pass @@ -313,19 +312,19 @@ class Type2: class Model(BaseModel, arbitrary_types_allowed=True): a: dict_cls - a1: dict_cls[str, int] + a1: 'dict_cls[str, int]' b: frozenset_cls - b1: frozenset_cls[int] + b1: 'frozenset_cls[int]' c: list_cls - c1: list_cls[int] + c1: 'list_cls[int]' d: set_cls - d1: set_cls[int] + d1: 'set_cls[int]' e: tuple_cls - e1: tuple_cls[int] - e2: tuple_cls[int, ...] - e3: tuple_cls[()] + e1: 'tuple_cls[int]' + e2: 'tuple_cls[int, ...]' + e3: 'tuple_cls[()]' f: type_cls - f1: type_cls[Type1] + f1: 'type_cls[Type1]' default_model_kwargs = dict( a={}, @@ -361,7 +360,7 @@ class Model(BaseModel, arbitrary_types_allowed=True): assert m.f1 == Type1 with pytest.raises(ValidationError) as exc_info: - Model(**(default_model_kwargs | {'e3': (1,)})) + Model(**{**default_model_kwargs, 'e3': (1,)}) # insert_assert(exc_info.value.errors(include_url=False)) assert exc_info.value.errors(include_url=False) == [ { @@ -373,10 +372,10 @@ class Model(BaseModel, arbitrary_types_allowed=True): } ] - Model(**(default_model_kwargs | {'f': Type2})) + Model(**{**default_model_kwargs, 'f': Type2}) with pytest.raises(ValidationError) as exc_info: - Model(**(default_model_kwargs | {'f1': Type2})) + Model(**{**default_model_kwargs, 'f1': Type2}) # insert_assert(exc_info.value.errors(include_url=False)) assert exc_info.value.errors(include_url=False) == [ { @@ -2382,10 +2381,9 @@ class Square(AbstractSquare): Square(side=1.0) -@pytest.mark.skipif(sys.version_info < (3, 9), reason='cannot use list.__class_getitem__ before 3.9') def test_generic_wrapped_forwardref(): class Operation(BaseModel): - callbacks: list['PathItem'] + callbacks: 'list[PathItem]' class PathItem(BaseModel): pass @@ -2483,7 +2481,6 @@ class C(BaseModel): ] -@pytest.mark.skipif(sys.version_info < (3, 9), reason='cannot parametrize types before 3.9') @pytest.mark.parametrize( ('sequence_type', 'input_data', 'expected_error_type', 'expected_error_msg', 'expected_error_ctx'), [ diff --git a/tests/test_forward_ref.py b/tests/test_forward_ref.py index 2431c26a9a..8fb6471b59 100644 --- a/tests/test_forward_ref.py +++ b/tests/test_forward_ref.py @@ -297,12 +297,11 @@ def test_self_reference_json_schema_with_future_annotations(create_module): # language=Python """ from __future__ import annotations -from typing import List from pydantic import BaseModel class Account(BaseModel): name: str - subaccounts: List[Account] = [] + subaccounts: list[Account] = [] """ ) Account = module.Account @@ -376,7 +375,6 @@ def test_circular_reference_json_schema_with_future_annotations(create_module): # language=Python """ from __future__ import annotations -from typing import List from pydantic import BaseModel class Owner(BaseModel): @@ -385,7 +383,7 @@ class Owner(BaseModel): class Account(BaseModel): name: str owner: Owner - subaccounts: List[Account] = [] + subaccounts: list[Account] = [] """ ) @@ -440,28 +438,27 @@ def test_forward_ref_optional(create_module): # language=Python """ from __future__ import annotations -from pydantic import BaseModel, Field, ConfigDict -from typing import List, Optional +from pydantic import BaseModel, Field class Spec(BaseModel): - spec_fields: List[str] = Field(..., alias="fields") - filter: Optional[str] = None - sort: Optional[str] + spec_fields: list[str] = Field(..., alias="fields") + filter: str | None = None + sort: str | None class PSpec(Spec): - g: Optional[GSpec] = None + g: GSpec | None = None class GSpec(Spec): - p: Optional[PSpec] + p: PSpec | None # PSpec.model_rebuild() class Filter(BaseModel): - g: Optional[GSpec] = None - p: Optional[PSpec] + g: GSpec | None = None + p: PSpec | None """ ) Filter = module.Filter diff --git a/tests/test_typing.py b/tests/test_typing.py index eb0e44b39d..62bb8c6e3a 100644 --- a/tests/test_typing.py +++ b/tests/test_typing.py @@ -9,7 +9,7 @@ from pydantic import Field # noqa: F401 from pydantic._internal._typing_extra import ( NoneType, - eval_type_backport, + eval_type_lenient, get_function_type_hints, is_classvar, is_literal_type, @@ -64,7 +64,7 @@ def test_is_none_type(): 'union', [ typing.Union[int, str], - eval_type_backport(typing.ForwardRef('int | str')), + eval_type_lenient('int | str'), *([int | str] if sys.version_info >= (3, 10) else []), ], ) diff --git a/tests/test_validate_call.py b/tests/test_validate_call.py index a30f1f8b29..491190e7d3 100644 --- a/tests/test_validate_call.py +++ b/tests/test_validate_call.py @@ -1,7 +1,6 @@ import asyncio import inspect import re -import sys from datetime import datetime, timezone from functools import partial from typing import Any, List, Tuple @@ -13,8 +12,6 @@ from pydantic import Field, PydanticInvalidForJsonSchema, TypeAdapter, ValidationError, validate_call from pydantic.main import BaseModel -skip_pre_39 = pytest.mark.skipif(sys.version_info < (3, 9), reason='testing >= 3.9 behaviour only') - def test_args(): @validate_call @@ -372,7 +369,6 @@ def foo(self, a: int, b: int): ] -@skip_pre_39 def test_class_method(): class X: @classmethod diff --git a/tests/test_validators.py b/tests/test_validators.py index 96c7941d5d..6a8db047ae 100644 --- a/tests/test_validators.py +++ b/tests/test_validators.py @@ -1,6 +1,5 @@ import contextlib import re -import sys from collections import deque from dataclasses import dataclass from datetime import date, datetime @@ -1602,7 +1601,6 @@ def check_foo(cls, value): assert validator_calls == 2 -@pytest.mark.skipif(sys.version_info[:2] == (3, 8), reason='https://github.com/python/cpython/issues/103592') def test_literal_validator(): class Model(BaseModel): a: Literal['foo'] @@ -1643,9 +1641,6 @@ class Foo(BaseModel): assert my_foo.fizfuz is Bar.FUZ -@pytest.mark.skipif( - sys.version_info[:2] == (3, 8), reason='https://github.com/python/cpython/issues/103592', strict=False -) def test_nested_literal_validator(): L1 = Literal['foo'] L2 = Literal['bar'] From f423659657422606694eebeedc87b4d7e5e63d2f Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sun, 17 Dec 2023 20:14:11 +0200 Subject: [PATCH 28/31] upgrade eval_type_backport to copy ForwardRef attributes, allowing unskipping one more test --- pdm.lock | 6 +++--- tests/test_forward_ref.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pdm.lock b/pdm.lock index bcd3908eb0..77ba7a7622 100644 --- a/pdm.lock +++ b/pdm.lock @@ -463,12 +463,12 @@ files = [ [[package]] name = "eval-type-backport" -version = "0.1.2" +version = "0.1.3" requires_python = ">=3.7" summary = "Like `typing._eval_type`, but lets older Python versions use newer typing features." files = [ - {file = "eval_type_backport-0.1.2-py3-none-any.whl", hash = "sha256:c9bd0d8b057581c91c0cee3c5c3848eebe7e97ec1d12f7d316c98070eb0006d8"}, - {file = "eval_type_backport-0.1.2.tar.gz", hash = "sha256:4c81b0cbd848037be6ba4d82e5b0e021e38a8ed988b8c96b8c591b35b1f10266"}, + {file = "eval_type_backport-0.1.3-py3-none-any.whl", hash = "sha256:519d2a993b3da286df9f90e17f503f66435106ad870cf26620c5720e2158ddf2"}, + {file = "eval_type_backport-0.1.3.tar.gz", hash = "sha256:d83ee225331dfa009493cec1f3608a71550b515ee4749abe78da14e3c5e314f5"}, ] [[package]] diff --git a/tests/test_forward_ref.py b/tests/test_forward_ref.py index 8fb6471b59..1ddcff327c 100644 --- a/tests/test_forward_ref.py +++ b/tests/test_forward_ref.py @@ -673,7 +673,6 @@ class Hero(BaseModel): assert h.model_dump() == {'name': 'Ivan', 'teams': [{'name': 'TheBest', 'heroes': []}]} -@pytest.mark.skipif(sys.version_info < (3, 9), reason='needs 3.9 or newer') def test_class_var_forward_ref(create_module): # see #3679 create_module( From aa21092288dd0350c681732ed6e9dd5516077534 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Mon, 15 Jan 2024 20:14:35 +0200 Subject: [PATCH 29/31] revert moving part of pyproject.toml --- pyproject.toml | 104 ++++++++++++++++++++++++------------------------- 1 file changed, 51 insertions(+), 53 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e8d2d279e2..ba9dc7eaeb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,57 @@ include = [ '/requirements', ] +[project] +name = 'pydantic' +description = 'Data validation using Python type hints' +authors = [ + {name = 'Samuel Colvin', email = 's@muelcolvin.com'}, + {name = 'Eric Jolibois', email = 'em.jolibois@gmail.com'}, + {name = 'Hasan Ramezani', email = 'hasan.r67@gmail.com'}, + {name = 'Adrian Garcia Badaracco', email = '1755071+adriangb@users.noreply.github.com'}, + {name = 'Terrence Dorsey', email = 'terry@pydantic.dev'}, + {name = 'David Montague', email = 'david@pydantic.dev'}, + {name = 'Serge Matveenko', email = 'lig@countzero.co'}, + {name = 'Marcelo Trylesinski', email = 'marcelotryle@gmail.com'}, + {name = 'Sydney Runkle', email = 'sydneymarierunkle@gmail.com'}, + {name = 'David Hewitt', email = 'mail@davidhewitt.io'}, +] +license = 'MIT' +classifiers = [ + 'Development Status :: 5 - Production/Stable', + 'Programming Language :: Python', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Intended Audience :: Developers', + 'Intended Audience :: Information Technology', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: MIT License', + 'Operating System :: Unix', + 'Operating System :: POSIX :: Linux', + 'Environment :: Console', + 'Environment :: MacOS X', + 'Framework :: Hypothesis', + 'Framework :: Pydantic', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Topic :: Internet', +] +requires-python = '>=3.8' +dependencies = [ + 'typing-extensions>=4.6.1', + 'annotated-types>=0.4.0', + "pydantic-core==2.15.0", +] +dynamic = ['version', 'readme'] + +[project.optional-dependencies] +email = ['email-validator>=2.0.0'] [tool.pdm.dev-dependencies] docs = [ @@ -77,58 +128,6 @@ memray = [ # requires Python > 3.8, we only test with 3.8 in CI but because of it won't lock properly pytest-memray = "1.5.0" -[project] -name = 'pydantic' -description = 'Data validation using Python type hints' -authors = [ - {name = 'Samuel Colvin', email = 's@muelcolvin.com'}, - {name = 'Eric Jolibois', email = 'em.jolibois@gmail.com'}, - {name = 'Hasan Ramezani', email = 'hasan.r67@gmail.com'}, - {name = 'Adrian Garcia Badaracco', email = '1755071+adriangb@users.noreply.github.com'}, - {name = 'Terrence Dorsey', email = 'terry@pydantic.dev'}, - {name = 'David Montague', email = 'david@pydantic.dev'}, - {name = 'Serge Matveenko', email = 'lig@countzero.co'}, - {name = 'Marcelo Trylesinski', email = 'marcelotryle@gmail.com'}, - {name = 'Sydney Runkle', email = 'sydneymarierunkle@gmail.com'}, - {name = 'David Hewitt', email = 'mail@davidhewitt.io'}, -] -license = 'MIT' -classifiers = [ - 'Development Status :: 5 - Production/Stable', - 'Programming Language :: Python', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3 :: Only', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Intended Audience :: Developers', - 'Intended Audience :: Information Technology', - 'Intended Audience :: System Administrators', - 'License :: OSI Approved :: MIT License', - 'Operating System :: Unix', - 'Operating System :: POSIX :: Linux', - 'Environment :: Console', - 'Environment :: MacOS X', - 'Framework :: Hypothesis', - 'Framework :: Pydantic', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Topic :: Internet', -] -requires-python = '>=3.8' -dependencies = [ - 'typing-extensions>=4.6.1', - 'annotated-types>=0.4.0', - "pydantic-core==2.15.0", -] -dynamic = ['version', 'readme'] - -[project.optional-dependencies] -email = ['email-validator>=2.0.0'] - [project.urls] Homepage = 'https://github.com/pydantic/pydantic' Documentation = 'https://docs.pydantic.dev' @@ -136,7 +135,6 @@ Funding = 'https://github.com/sponsors/samuelcolvin' Source = 'https://github.com/pydantic/pydantic' Changelog = 'https://docs.pydantic.dev/latest/changelog/' - [tool.hatch.metadata.hooks.fancy-pypi-readme] content-type = 'text/markdown' # construct the PyPI readme from README.md and HISTORY.md From 8da4294371a04ad8082c2470f36cd96fdd37dad5 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Tue, 16 Jan 2024 19:22:38 +0200 Subject: [PATCH 30/31] Refine and test error raised when eval_type_backport isn't installed --- pydantic/_internal/_typing_extra.py | 8 +++++--- tests/test_typing.py | 22 +++++++++++++++++++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/pydantic/_internal/_typing_extra.py b/pydantic/_internal/_typing_extra.py index c4b456c954..1d5d3b3f85 100644 --- a/pydantic/_internal/_typing_extra.py +++ b/pydantic/_internal/_typing_extra.py @@ -246,9 +246,11 @@ def eval_type_backport( try: from eval_type_backport import eval_type_backport except ImportError: - raise RuntimeError( - 'In order to use newer typing features on older Python versions, ' - 'you need to install the `eval_type_backport` package.' + raise TypeError( + f'You have a type annotation {value.__forward_arg__!r} ' + f'which makes use of newer typing features than are supported in your version of Python. ' + f'To handle this error, you should either remove the use of new syntax ' + f'or install the `eval_type_backport` package.' ) from e return eval_type_backport(value, globalns, localns, try_default=False) diff --git a/tests/test_typing.py b/tests/test_typing.py index 62bb8c6e3a..dc1881d752 100644 --- a/tests/test_typing.py +++ b/tests/test_typing.py @@ -6,7 +6,7 @@ import pytest from typing_extensions import Literal, get_origin -from pydantic import Field # noqa: F401 +from pydantic import BaseModel, Field # noqa: F401 from pydantic._internal._typing_extra import ( NoneType, eval_type_lenient, @@ -117,3 +117,23 @@ def f(x: int, y: None) -> int: return x assert get_function_type_hints(f) == {'return': int, 'x': int, 'y': NoneType} + + +@pytest.mark.skipif(sys.version_info[:2] > (3, 9), reason='testing using a feature not supported by older Python') +def test_eval_type_backport_not_installed(): + sys.modules['eval_type_backport'] = None + try: + with pytest.raises(TypeError) as exc_info: + + class _Model(BaseModel): + foo: 'list[int]' + + assert str(exc_info.value) == ( + "You have a type annotation 'list[int]' which makes use of newer typing " + 'features than are supported in your version of Python. To handle this error, ' + 'you should either remove the use of new syntax or install the ' + '`eval_type_backport` package.' + ) + + finally: + del sys.modules['eval_type_backport'] From 60aa70fdfaaa6edf3b6bba94148f60bbcf980cc0 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Tue, 16 Jan 2024 19:26:41 +0200 Subject: [PATCH 31/31] use a type annotation that's unsupported in 3.9, not just 3.8 --- tests/test_typing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_typing.py b/tests/test_typing.py index dc1881d752..849dd1552a 100644 --- a/tests/test_typing.py +++ b/tests/test_typing.py @@ -126,10 +126,10 @@ def test_eval_type_backport_not_installed(): with pytest.raises(TypeError) as exc_info: class _Model(BaseModel): - foo: 'list[int]' + foo: 'int | str' assert str(exc_info.value) == ( - "You have a type annotation 'list[int]' which makes use of newer typing " + "You have a type annotation 'int | str' which makes use of newer typing " 'features than are supported in your version of Python. To handle this error, ' 'you should either remove the use of new syntax or install the ' '`eval_type_backport` package.'