From 509aaab0026a85e01bdd91b8b0988bec76a0ac42 Mon Sep 17 00:00:00 2001 From: David Montague <35119617+dmontagu@users.noreply.github.com> Date: Thu, 2 Nov 2023 16:20:50 -0600 Subject: [PATCH 1/5] Make union case tags affect union error messages --- pydantic/_internal/_generate_schema.py | 18 ++++++++++----- pydantic/types.py | 2 ++ tests/test_types.py | 31 ++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/pydantic/_internal/_generate_schema.py b/pydantic/_internal/_generate_schema.py index 276dd12b2e..cdd82a9a11 100644 --- a/pydantic/_internal/_generate_schema.py +++ b/pydantic/_internal/_generate_schema.py @@ -42,7 +42,7 @@ from ..json_schema import JsonSchemaValue from ..version import version_short from ..warnings import PydanticDeprecatedSince20 -from . import _decorators, _discriminated_union, _known_annotated_metadata, _typing_extra +from . import _core_utils, _decorators, _discriminated_union, _known_annotated_metadata, _typing_extra from ._config import ConfigWrapper, ConfigWrapperStack from ._core_metadata import CoreMetadataHandler, build_metadata_dict from ._core_utils import ( @@ -1033,7 +1033,7 @@ def json_schema_update_func(schema: CoreSchemaOrField, handler: GetJsonSchemaHan def _union_schema(self, union_type: Any) -> core_schema.CoreSchema: """Generate schema for a Union.""" args = self._get_args_resolving_forward_refs(union_type, required=True) - choices: list[CoreSchema | tuple[CoreSchema, str]] = [] + choices: list[CoreSchema] = [] nullable = False for arg in args: if arg is None or arg is _typing_extra.NoneType: @@ -1042,10 +1042,18 @@ def _union_schema(self, union_type: Any) -> core_schema.CoreSchema: choices.append(self.generate_schema(arg)) if len(choices) == 1: - first_choice = choices[0] - s = first_choice[0] if isinstance(first_choice, tuple) else first_choice + s = choices[0] else: - s = core_schema.union_schema(choices) + choices_with_tags: list[CoreSchema | tuple[CoreSchema, str]] = [] + for choice in choices: + metadata = choice.get('metadata') + if isinstance(metadata, dict): + tag = metadata.get(_core_utils.TAGGED_UNION_TAG_KEY) + if tag is not None: + choices_with_tags.append((choice, tag)) + else: + choices_with_tags.append(choice) + s = core_schema.union_schema(choices_with_tags) if nullable: s = core_schema.nullable_schema(s) diff --git a/pydantic/types.py b/pydantic/types.py index 1c4cb93224..19a05c4ba0 100644 --- a/pydantic/types.py +++ b/pydantic/types.py @@ -2444,6 +2444,8 @@ def __getattr__(self, item: str) -> Any: class Tag: """Provides a way to specify the expected tag to use for a case with a callable discriminated union. + Also provides a way to label a union case in error messages. + When using a `CallableDiscriminator`, attach a `Tag` to each case in the `Union` to specify the tag that should be used to identify that case. For example, in the below example, the `Tag` is used to specify that if `get_discriminator_value` returns `'apple'`, the input should be validated as an `ApplePie`, and if it diff --git a/tests/test_types.py b/tests/test_types.py index 02ff55adfc..638aa6d394 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -15,6 +15,7 @@ from numbers import Number from pathlib import Path from typing import ( + Annotated, Any, Callable, Counter, @@ -47,6 +48,7 @@ UUID3, UUID4, UUID5, + AfterValidator, AwareDatetime, Base64Bytes, Base64Str, @@ -88,6 +90,7 @@ StrictFloat, StrictInt, StrictStr, + Tag, TypeAdapter, ValidationError, conbytes, @@ -6025,3 +6028,31 @@ class Model(BaseModel): value: str assert Model.model_validate_json(f'{{"value": {number}}}').model_dump() == {'value': expected_str} + + +def test_union_tags_in_errors(): + DoubledList = Annotated[list[int], AfterValidator(lambda x: x * 2), Tag('DoubledList')] + StringsMap = Annotated[dict[str, str], Tag('StringsMap')] + + adapter = TypeAdapter(Union[DoubledList, StringsMap]) + + with pytest.raises(ValidationError) as exc_info: + adapter.validate_python(['a']) + + assert '2 validation errors for union[DoubledList,StringsMap]' in str(exc_info) + assert exc_info.value.errors() == [ + { + 'input': 'a', + 'loc': ('DoubledList', 0), + 'msg': 'Input should be a valid integer, unable to parse string as an ' 'integer', + 'type': 'int_parsing', + 'url': 'https://errors.pydantic.dev/2.4/v/int_parsing', + }, + { + 'input': ['a'], + 'loc': ('StringsMap',), + 'msg': 'Input should be a valid dictionary', + 'type': 'dict_type', + 'url': 'https://errors.pydantic.dev/2.4/v/dict_type', + }, + ] From c6736eb48d160340191e37c696b043e8217fcf14 Mon Sep 17 00:00:00 2001 From: David Montague <35119617+dmontagu@users.noreply.github.com> Date: Thu, 2 Nov 2023 16:24:27 -0600 Subject: [PATCH 2/5] Make test clearer --- tests/test_types.py | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/tests/test_types.py b/tests/test_types.py index 638aa6d394..7a8ca4d89f 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -6031,15 +6031,41 @@ class Model(BaseModel): def test_union_tags_in_errors(): - DoubledList = Annotated[list[int], AfterValidator(lambda x: x * 2), Tag('DoubledList')] - StringsMap = Annotated[dict[str, str], Tag('StringsMap')] + DoubledList = Annotated[list[int], AfterValidator(lambda x: x * 2)] + StringsMap = dict[str, str] adapter = TypeAdapter(Union[DoubledList, StringsMap]) with pytest.raises(ValidationError) as exc_info: adapter.validate_python(['a']) - assert '2 validation errors for union[DoubledList,StringsMap]' in str(exc_info) + assert '2 validation errors for union[function-after[(), list[int]],dict[str,str]]' in str(exc_info) # yuck + # the loc's are bad here: + assert exc_info.value.errors() == [ + { + 'input': 'a', + 'loc': ('function-after[(), list[int]]', 0), + 'msg': 'Input should be a valid integer, unable to parse string as an ' 'integer', + 'type': 'int_parsing', + 'url': 'https://errors.pydantic.dev/2.4/v/int_parsing', + }, + { + 'input': ['a'], + 'loc': ('dict[str,str]',), + 'msg': 'Input should be a valid dictionary', + 'type': 'dict_type', + 'url': 'https://errors.pydantic.dev/2.4/v/dict_type', + }, + ] + + tag_adapter = TypeAdapter( + Union[Annotated[DoubledList, Tag('DoubledList')], Annotated[StringsMap, Tag('StringsMap')]] + ) + with pytest.raises(ValidationError) as exc_info: + tag_adapter.validate_python(['a']) + + assert '2 validation errors for union[DoubledList,StringsMap]' in str(exc_info) # nice + # the loc's are good here: assert exc_info.value.errors() == [ { 'input': 'a', From b93dc21014483d9078b2891e0b33ab5cd3703a41 Mon Sep 17 00:00:00 2001 From: sydney-runkle Date: Mon, 6 Nov 2023 13:41:28 -0600 Subject: [PATCH 3/5] test fixes and docs additions --- docs/concepts/unions.md | 79 ++++++++++++++++++++++++++++++++++++++++- pydantic/types.py | 4 +-- tests/test_types.py | 5 ++- 3 files changed, 82 insertions(+), 6 deletions(-) diff --git a/docs/concepts/unions.md b/docs/concepts/unions.md index 93625cb94f..59c407519b 100644 --- a/docs/concepts/unions.md +++ b/docs/concepts/unions.md @@ -513,4 +513,81 @@ assert m == DiscriminatedModel( assert m.model_dump() == data ``` -You can also simplify error messages with a custom error, like this: +You can also simplify error messages by labeling each case with a `Tag`. This is especially useful +when you have complex types like those in this example: + +```py +from typing import Dict, List, Union + +from typing_extensions import Annotated + +from pydantic import AfterValidator, Tag, TypeAdapter, ValidationError + +DoubledList = Annotated[List[int], AfterValidator(lambda x: x * 2)] +StringsMap = Dict[str, str] + + +# Not using any `Tag`s for each union case, the errors are not so nice to look at +adapter = TypeAdapter(Union[DoubledList, StringsMap]) + +try: + adapter.validate_python(['a']) +except ValidationError as exc_info: + assert ( + '2 validation errors for union[function-after[(), list[int]],dict[str,str]]' + in str(exc_info) + ) + + # the loc's are bad here: + assert exc_info.errors() == [ + { + 'input': 'a', + 'loc': ('function-after[(), list[int]]', 0), + 'msg': 'Input should be a valid integer, unable to parse string as an ' + 'integer', + 'type': 'int_parsing', + 'url': 'https://errors.pydantic.dev/2.4/v/int_parsing', + }, + { + 'input': ['a'], + 'loc': ('dict[str,str]',), + 'msg': 'Input should be a valid dictionary', + 'type': 'dict_type', + 'url': 'https://errors.pydantic.dev/2.4/v/dict_type', + }, + ] + + +tag_adapter = TypeAdapter( + Union[ + Annotated[DoubledList, Tag('DoubledList')], + Annotated[StringsMap, Tag('StringsMap')], + ] +) + +try: + tag_adapter.validate_python(['a']) +except ValidationError as exc_info: + assert '2 validation errors for union[DoubledList,StringsMap]' in str( + exc_info + ) + + # the loc's are good here: + assert exc_info.errors() == [ + { + 'input': 'a', + 'loc': ('DoubledList', 0), + 'msg': 'Input should be a valid integer, unable to parse string as an ' + 'integer', + 'type': 'int_parsing', + 'url': 'https://errors.pydantic.dev/2.4/v/int_parsing', + }, + { + 'input': ['a'], + 'loc': ('StringsMap',), + 'msg': 'Input should be a valid dictionary', + 'type': 'dict_type', + 'url': 'https://errors.pydantic.dev/2.4/v/dict_type', + }, + ] +``` diff --git a/pydantic/types.py b/pydantic/types.py index 9bf1e8c8fa..23cab0f6ec 100644 --- a/pydantic/types.py +++ b/pydantic/types.py @@ -2511,7 +2511,7 @@ class ThanksgivingDinner(BaseModel): Failing to do so will result in a `PydanticUserError` with code [`callable-discriminator-no-tag`](../errors/usage_errors.md#callable-discriminator-no-tag). - See the [Discriminated Unions](../api/standard_library_types.md#discriminated-unions-aka-tagged-unions) + See the [Discriminated Unions](../concepts/unions.md#discriminated-unions) docs for more details on how to use `Tag`s. """ @@ -2592,7 +2592,7 @@ class ThanksgivingDinner(BaseModel): ''' ``` - See the [Discriminated Unions](../api/standard_library_types.md#discriminated-unions-aka-tagged-unions) + See the [Discriminated Unions](../concepts/unions.md#discriminated-unions) docs for more details on how to use `CallableDiscriminator`s. """ diff --git a/tests/test_types.py b/tests/test_types.py index bf706f6c0e..3bd75c9101 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -15,7 +15,6 @@ from numbers import Number from pathlib import Path from typing import ( - Annotated, Any, Callable, Counter, @@ -6034,8 +6033,8 @@ class Model(BaseModel): def test_union_tags_in_errors(): - DoubledList = Annotated[list[int], AfterValidator(lambda x: x * 2)] - StringsMap = dict[str, str] + DoubledList = Annotated[List[int], AfterValidator(lambda x: x * 2)] + StringsMap = Dict[str, str] adapter = TypeAdapter(Union[DoubledList, StringsMap]) From 3b989d2596697aeb90783b5593206ad2c17beee6 Mon Sep 17 00:00:00 2001 From: sydney-runkle Date: Mon, 6 Nov 2023 13:47:48 -0600 Subject: [PATCH 4/5] fixing references to union docs in migration guide --- docs/concepts/unions.md | 6 ++++-- docs/migration.md | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/concepts/unions.md b/docs/concepts/unions.md index 59c407519b..429c57fafd 100644 --- a/docs/concepts/unions.md +++ b/docs/concepts/unions.md @@ -11,7 +11,9 @@ To solve these problems, Pydantic supports three fundamental approaches to valid 2. [smart mode](#smart-mode) - as with "left to right mode" all members are tried, but strict validation is used to try to find the best match 3. [discriminated unions]() - only one member of the union is tried, based on a discriminator -## Left to Right Mode +## Union Modes + +### Left to Right Mode !!! note Because this mode often leads to unexpected validation results, it is not the default in Pydantic >=2, instead `union_mode='smart'` is the default. @@ -72,7 +74,7 @@ print(User(id='456')) # (2) 2. We're in lax mode and the numeric string `'123'` is valid as input to the first member of the union, `int`. Since that is tried first, we get the surprising result of `id` being an `int` instead of a `str`. -## Smart Mode +### Smart Mode Because of the surprising side effects of `union_mode='left_to_right'`, in Pydantic >=2 the default mode for `Union` validation is `union_mode='smart'`. diff --git a/docs/migration.md b/docs/migration.md index c5f85b0daf..d6be675a6e 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -521,7 +521,7 @@ In Pydantic V1, the printed result would have been `x=1`, since the value would In Pydantic V2, we recognize that the value is an instance of one of the cases and short-circuit the standard union validation. To revert to the non-short-circuiting left-to-right behavior of V1, annotate the union with `Field(union_mode='left_to_right')`. -See [Union Mode](./api/standard_library_types.md#union-mode) for more details. +See [Union Mode](./concepts/unions.md#union-modes) for more details. #### Required, optional, and nullable fields From ba117c3cec875218b3df5f39d8c506cf2cf10078 Mon Sep 17 00:00:00 2001 From: sydney-runkle Date: Mon, 6 Nov 2023 14:04:30 -0600 Subject: [PATCH 5/5] adding in marcelo's suggestions --- docs/concepts/unions.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/concepts/unions.md b/docs/concepts/unions.md index 429c57fafd..80339f6e6a 100644 --- a/docs/concepts/unions.md +++ b/docs/concepts/unions.md @@ -9,7 +9,7 @@ Validating unions feels like adding another orthogonal dimension to the validati To solve these problems, Pydantic supports three fundamental approaches to validating unions: 1. [left to right mode](#left-to-right-mode) - the simplest approach, each member of the union is tried in order 2. [smart mode](#smart-mode) - as with "left to right mode" all members are tried, but strict validation is used to try to find the best match -3. [discriminated unions]() - only one member of the union is tried, based on a discriminator +3. [discriminated unions](#discriminated-unions) - only one member of the union is tried, based on a discriminator ## Union Modes @@ -378,7 +378,7 @@ except ValidationError as e: """ ``` -### Interpreting Error Messages +## Union Validation Errors When validation fails, error messages can be quite verbose, especially when you're not using discriminated unions. The below example shows the benefits of using discriminated unions in terms of error message simplicity. @@ -515,8 +515,8 @@ assert m == DiscriminatedModel( assert m.model_dump() == data ``` -You can also simplify error messages by labeling each case with a `Tag`. This is especially useful -when you have complex types like those in this example: +You can also simplify error messages by labeling each case with a [`Tag`][pydantic.types.Tag]. +This is especially useful when you have complex types like those in this example: ```py from typing import Dict, List, Union