From 233ce07239c20cd17248df8f4689ba645ebc7c38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Mon, 6 Jan 2020 20:29:22 +0100 Subject: [PATCH 01/12] :sparkles: Implement support for infinite generators with Iterable --- pydantic/errors.py | 4 ++++ pydantic/fields.py | 28 ++++++++++++++++++++++++++++ pydantic/schema.py | 3 ++- 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/pydantic/errors.py b/pydantic/errors.py index 9fd57f5912..fe26244188 100644 --- a/pydantic/errors.py +++ b/pydantic/errors.py @@ -238,6 +238,10 @@ class SequenceError(PydanticTypeError): msg_template = 'value is not a valid sequence' +class IterableError(PydanticTypeError): + msg_template = 'value is not a valid iterable' + + class ListError(PydanticTypeError): msg_template = 'value is not a valid list' diff --git a/pydantic/fields.py b/pydantic/fields.py index b5a56f8262..5d9e0c3c4e 100644 --- a/pydantic/fields.py +++ b/pydantic/fields.py @@ -1,10 +1,12 @@ import warnings +from collections import abc from typing import ( TYPE_CHECKING, Any, Dict, FrozenSet, Generator, + Iterable, Iterator, List, Mapping, @@ -174,12 +176,14 @@ def Schema(default: Any, **kwargs: Any) -> Any: SHAPE_TUPLE_ELLIPSIS = 6 SHAPE_SEQUENCE = 7 SHAPE_FROZENSET = 8 +SHAPE_ITERABLE = 9 SHAPE_NAME_LOOKUP = { SHAPE_LIST: 'List[{}]', SHAPE_SET: 'Set[{}]', SHAPE_TUPLE_ELLIPSIS: 'Tuple[{}, ...]', SHAPE_SEQUENCE: 'Sequence[{}]', SHAPE_FROZENSET: 'FrozenSet[{}]', + SHAPE_ITERABLE: 'Iterable[{}]', } @@ -416,6 +420,11 @@ def _type_analysis(self) -> None: # noqa: C901 (ignore complexity) self.key_field = self._create_sub_type(self.type_.__args__[0], 'key_' + self.name, for_keys=True) self.type_ = self.type_.__args__[1] self.shape = SHAPE_MAPPING + # Equality check as almost everything inherit form Iterable, including str + # check for typing.Iterable and abc.Iterable, as it could receive one even when declared with the other + elif origin == Iterable or origin == abc.Iterable: + self.type_ = self.type_.__args__[0] + self.shape = SHAPE_ITERABLE elif issubclass(origin, Type): # type: ignore return else: @@ -489,6 +498,8 @@ def validate( v, errors = self._validate_mapping(v, values, loc, cls) elif self.shape == SHAPE_TUPLE: v, errors = self._validate_tuple(v, values, loc, cls) + elif self.shape == SHAPE_ITERABLE: + v, errors = self._validate_iterable(v, values, loc, cls) else: # sequence, list, set, generator, tuple with ellipsis, frozen set v, errors = self._validate_sequence_like(v, values, loc, cls) @@ -548,6 +559,23 @@ def _validate_sequence_like( # noqa: C901 (ignore complexity) converted = iter(result) return converted, None + def _validate_iterable( + self, v: Any, values: Dict[str, Any], loc: 'LocStr', cls: Optional['ModelOrDc'] + ) -> 'ValidateReturn': + """ + Validate Iterables. + + This intentionally doesn't validate values to allow infinite generators. + """ + + e: Optional[Exception] = None + try: + iterable = iter(v) + except TypeError: + e = errors_.IterableError() + return v, ErrorWrapper(e, loc) + return iterable, None + def _validate_tuple( self, v: Any, values: Dict[str, Any], loc: 'LocStr', cls: Optional['ModelOrDc'] ) -> 'ValidateReturn': diff --git a/pydantic/schema.py b/pydantic/schema.py index 60b2f23dec..2f1197e3b1 100644 --- a/pydantic/schema.py +++ b/pydantic/schema.py @@ -26,6 +26,7 @@ from .class_validators import ROOT_KEY from .fields import ( SHAPE_FROZENSET, + SHAPE_ITERABLE, SHAPE_LIST, SHAPE_MAPPING, SHAPE_SEQUENCE, @@ -375,7 +376,7 @@ def field_type_schema( nested_models: Set[str] = set() f_schema: Dict[str, Any] ref_prefix = ref_prefix or default_prefix - if field.shape in {SHAPE_LIST, SHAPE_TUPLE_ELLIPSIS, SHAPE_SEQUENCE, SHAPE_SET, SHAPE_FROZENSET}: + if field.shape in {SHAPE_LIST, SHAPE_TUPLE_ELLIPSIS, SHAPE_SEQUENCE, SHAPE_SET, SHAPE_FROZENSET, SHAPE_ITERABLE}: items_schema, f_definitions, f_nested_models = field_singleton_schema( field, by_alias=by_alias, model_name_map=model_name_map, ref_prefix=ref_prefix, known_models=known_models ) From e2de0104f53c7fb1b13a2eb7e0472bf622296fa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Mon, 6 Jan 2020 20:31:28 +0100 Subject: [PATCH 02/12] :white_check_mark: Add tests for infinite generators --- tests/test_schema.py | 14 ++++++++++++- tests/test_types.py | 49 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/tests/test_schema.py b/tests/test_schema.py index dd54c91f51..c109f268f2 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -6,7 +6,7 @@ from enum import Enum, IntEnum from ipaddress import IPv4Address, IPv4Interface, IPv4Network, IPv6Address, IPv6Interface, IPv6Network from pathlib import Path -from typing import Any, Callable, Dict, FrozenSet, List, NewType, Optional, Set, Tuple, Union +from typing import Any, Callable, Dict, FrozenSet, Iterable, List, NewType, Optional, Set, Tuple, Union from uuid import UUID import pytest @@ -1746,3 +1746,15 @@ class Model(BaseModel): } }, } + + +def test_iterable(): + class Model(BaseModel): + a: Iterable[int] + + assert Model.schema() == { + 'title': 'Model', + 'type': 'object', + 'properties': {'a': {'title': 'A', 'type': 'array', 'items': {'type': 'integer'},}}, + 'required': ['a'], + } diff --git a/tests/test_types.py b/tests/test_types.py index 63d0d19e9d..057fd8e585 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -6,7 +6,20 @@ from decimal import Decimal from enum import Enum, IntEnum from pathlib import Path -from typing import Dict, FrozenSet, Iterator, List, MutableSet, NewType, Optional, Pattern, Sequence, Set, Tuple +from typing import ( + Dict, + FrozenSet, + Iterable, + Iterator, + List, + MutableSet, + NewType, + Optional, + Pattern, + Sequence, + Set, + Tuple, +) from uuid import UUID import pytest @@ -775,6 +788,40 @@ class Model(BaseModel): assert list(validated) == list(result) +def test_infinite_iterable(): + class Model(BaseModel): + it: Iterable[int] + b: int + + def iterable(): + i = 0 + while True: + i += 1 + yield i + + m = Model(it=iterable(), b=3) + + assert m.b == 3 + assert m.it + + for i in m.it: + assert i + if i == 10: + break + + +def test_invalid_iterable(): + class Model(BaseModel): + it: Iterable[int] + b: int + + with pytest.raises(ValidationError) as exc_info: + m = Model(it=3, b=3) + assert exc_info.value.errors() == [ + {'loc': ('it',), 'msg': 'value is not a valid iterable', 'type': 'type_error.iterable'} + ] + + @pytest.mark.parametrize( 'cls,value,errors', ( From 5291456f6ebde9dc2812bc2d02f97c8e9ebb16c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Mon, 6 Jan 2020 20:32:58 +0100 Subject: [PATCH 03/12] :art: Fix format --- tests/test_schema.py | 2 +- tests/test_types.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_schema.py b/tests/test_schema.py index c109f268f2..4d7cfbe0e2 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -1755,6 +1755,6 @@ class Model(BaseModel): assert Model.schema() == { 'title': 'Model', 'type': 'object', - 'properties': {'a': {'title': 'A', 'type': 'array', 'items': {'type': 'integer'},}}, + 'properties': {'a': {'title': 'A', 'type': 'array', 'items': {'type': 'integer'}}}, 'required': ['a'], } diff --git a/tests/test_types.py b/tests/test_types.py index 057fd8e585..f3834a7f5d 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -816,7 +816,7 @@ class Model(BaseModel): b: int with pytest.raises(ValidationError) as exc_info: - m = Model(it=3, b=3) + Model(it=3, b=3) assert exc_info.value.errors() == [ {'loc': ('it',), 'msg': 'value is not a valid iterable', 'type': 'type_error.iterable'} ] From 1829d57bae444fe69cc251cc024e846b66ddbb6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Mon, 6 Jan 2020 20:33:23 +0100 Subject: [PATCH 04/12] :memo: Add docs for infinite generators --- docs/examples/types_infinite_generator.py | 19 ++++++++++++++++++ docs/usage/types.md | 24 +++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 docs/examples/types_infinite_generator.py diff --git a/docs/examples/types_infinite_generator.py b/docs/examples/types_infinite_generator.py new file mode 100644 index 0000000000..7ceabf5939 --- /dev/null +++ b/docs/examples/types_infinite_generator.py @@ -0,0 +1,19 @@ +from typing import Iterable, Sequence +from pydantic import BaseModel + +class Model(BaseModel): + infinite: Iterable[int] + +def infinite_ints(): + i = 0 + while True: + yield i + i += 1 + +m = Model(infinite=infinite_ints()) +print(m) + +for i in m.infinite: + print(i) + if i == 10: + break diff --git a/docs/usage/types.md b/docs/usage/types.md index 3a92b7e626..2974850483 100644 --- a/docs/usage/types.md +++ b/docs/usage/types.md @@ -93,6 +93,9 @@ with custom properties and validation. `typing.Sequence` : see [Typing Iterables](#typing-iterables) below for more detail on parsing and validation +`typing.Iterable` +: this is reserved for iterables that shouldn't be consumed. See [Infinite Generators](#infinite-generators) below for more detail on parsing and validation + `typing.Type` : see [Type](#type) below for more detail on parsing and validation @@ -157,6 +160,27 @@ with custom properties and validation. ``` _(This script is complete, it should run "as is")_ +### Infinite Generators + +If you have a generator you can use `Sequence` as described above. In that case, the generator will be consumed and its values will be validated with the sub-type of `Sequence` (e.g. `int` in `Sequence[int]`). + +But if you have a generator that you don't want to be consumed, e.g. an infinite generator or a remote data loader, you can define its type with `Iterable`: + +```py +{!.tmp_examples/types_infinite_generator.py!} +``` +_(This script is complete, it should run "as is")_ + +!!! warning + `Iterable` fields only perform a simple check that the argument is iterable and won't be consumed. + + No validation of their values is performed as it cannot be done without consuming the iterable. + +!!! tip + If you want to validate the values of an infinite generator you can create a separate model and use it while consuming the generator, reporting the validation errors as appropriate. + + pydantic can't validate the values automatically for you because it would require consuming the infinite generator. + ### Unions The `Union` type allows a model attribute to accept different types, e.g.: From b08ec2e1ecc7b93b37ca970f4f9ea74e60878c92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Mon, 6 Jan 2020 21:06:29 +0100 Subject: [PATCH 05/12] :memo: Add changes file --- changes/1152-tiangolo.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/1152-tiangolo.md diff --git a/changes/1152-tiangolo.md b/changes/1152-tiangolo.md new file mode 100644 index 0000000000..8a004a6e4e --- /dev/null +++ b/changes/1152-tiangolo.md @@ -0,0 +1 @@ +add support for infinite generators with `Iterable` From aadfe1c7177b14007a91b4fc821d1a3eac078d29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Fri, 10 Jan 2020 13:52:26 +0100 Subject: [PATCH 06/12] :sparkles: Store sub_field with original type parameter to allow custom validation --- pydantic/fields.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pydantic/fields.py b/pydantic/fields.py index 5d9e0c3c4e..d98a347d06 100644 --- a/pydantic/fields.py +++ b/pydantic/fields.py @@ -425,6 +425,7 @@ def _type_analysis(self) -> None: # noqa: C901 (ignore complexity) elif origin == Iterable or origin == abc.Iterable: self.type_ = self.type_.__args__[0] self.shape = SHAPE_ITERABLE + self.sub_fields = [self._create_sub_type(self.type_, f'{self.name}_type')] elif issubclass(origin, Type): # type: ignore return else: From 4d5401453b6689e96e15a86482ecd1ceba2914bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Fri, 10 Jan 2020 13:53:23 +0100 Subject: [PATCH 07/12] :memo: Add example for validating first value in infinite generators --- ...types_infinite_generator_validate_first.py | 47 +++++++++++++++++++ docs/usage/types.md | 9 ++++ 2 files changed, 56 insertions(+) create mode 100644 docs/examples/types_infinite_generator_validate_first.py diff --git a/docs/examples/types_infinite_generator_validate_first.py b/docs/examples/types_infinite_generator_validate_first.py new file mode 100644 index 0000000000..cd0006b8bf --- /dev/null +++ b/docs/examples/types_infinite_generator_validate_first.py @@ -0,0 +1,47 @@ +import itertools +from typing import Iterable +from pydantic import BaseModel, validator, ValidationError +from pydantic.fields import ModelField + +class Model(BaseModel): + infinite: Iterable[int] + + @validator('infinite') + # You don't need to add the "ModelField", but it will help your + # editor give you completion and catch errors + def infinite_first_int(cls, iterable, field: ModelField): + first_value = next(iterable) + if field.sub_fields: + # The Iterable had a parameter type, in this case it's int + # We use it to validate the first value + sub_field = field.sub_fields[0] + v, error = sub_field.validate(first_value, {}, loc='first_value') + if error: + raise ValidationError([error], cls) + # This creates a new generator that returns the first value and then + # the rest of the values from the (already started) iterable + return itertools.chain([first_value], iterable) + +def infinite_ints(): + i = 0 + while True: + yield i + i += 1 + +m = Model(infinite=infinite_ints()) +print(m) + +for i in m.infinite: + print(i) + if i == 10: + break + +def infinite_strs(): + while True: + for letter in 'allthesingleladies': + yield letter + +try: + Model(infinite=infinite_strs()) +except ValidationError as e: + print(e) diff --git a/docs/usage/types.md b/docs/usage/types.md index 2974850483..2006beb6d0 100644 --- a/docs/usage/types.md +++ b/docs/usage/types.md @@ -181,6 +181,15 @@ _(This script is complete, it should run "as is")_ pydantic can't validate the values automatically for you because it would require consuming the infinite generator. +#### Infinite Generators with Validation for First Value + +You can create a [Validator](validators.md) to validate the first value in an infinite generator and still not consume it entirely. + +```py +{!.tmp_examples/types_infinite_generator_validate_first.py!} +``` +_(This script is complete, it should run "as is")_ + ### Unions The `Union` type allows a model attribute to accept different types, e.g.: From 286e14dd4ff3245d899aa32749d231f162331a00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Fri, 10 Jan 2020 13:53:44 +0100 Subject: [PATCH 08/12] :fire: Remove unused import in example --- docs/examples/types_infinite_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/examples/types_infinite_generator.py b/docs/examples/types_infinite_generator.py index 7ceabf5939..fb7a8becff 100644 --- a/docs/examples/types_infinite_generator.py +++ b/docs/examples/types_infinite_generator.py @@ -1,4 +1,4 @@ -from typing import Iterable, Sequence +from typing import Iterable from pydantic import BaseModel class Model(BaseModel): From ef146e774fb634b696c89e7acc5fbef83dc4af54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Fri, 10 Jan 2020 13:54:11 +0100 Subject: [PATCH 09/12] :white_check_mark: Add test for infinite generator with first-value validation --- tests/test_types.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tests/test_types.py b/tests/test_types.py index f3834a7f5d..2693a59ad9 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -1,3 +1,4 @@ +import itertools import os import sys import uuid @@ -822,6 +823,49 @@ class Model(BaseModel): ] +def test_infinite_iterable_validate_first(): + class Model(BaseModel): + it: Iterable[int] + b: int + + @validator('it') + def infinite_first_int(cls, it, field): + first_value = next(it) + if field.sub_fields: + sub_field = field.sub_fields[0] + v, error = sub_field.validate(first_value, {}, loc='first_value') + if error: + raise ValidationError([error], cls) + return itertools.chain([first_value], it) + + def int_iterable(): + i = 0 + while True: + i += 1 + yield i + + m = Model(it=int_iterable(), b=3) + + assert m.b == 3 + assert m.it + + for i in m.it: + assert i + if i == 10: + break + + def str_iterable(): + while True: + for c in 'foobarbaz': + yield c + + with pytest.raises(ValidationError) as exc_info: + Model(it=str_iterable(), b=3) + assert exc_info.value.errors() == [ + {'loc': ('it', 'first_value'), 'msg': 'value is not a valid integer', 'type': 'type_error.integer'} + ] + + @pytest.mark.parametrize( 'cls,value,errors', ( From c7db26c692e87b579fddc6f2fc0dab7a27f6c590 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Sat, 11 Jan 2020 16:04:40 +0100 Subject: [PATCH 10/12] :recycle: Update fields with code review --- pydantic/fields.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pydantic/fields.py b/pydantic/fields.py index d98a347d06..7229fa5aba 100644 --- a/pydantic/fields.py +++ b/pydantic/fields.py @@ -1,5 +1,5 @@ import warnings -from collections import abc +from collections.abc import Iterable as CollectionsIterable from typing import ( TYPE_CHECKING, Any, @@ -420,9 +420,9 @@ def _type_analysis(self) -> None: # noqa: C901 (ignore complexity) self.key_field = self._create_sub_type(self.type_.__args__[0], 'key_' + self.name, for_keys=True) self.type_ = self.type_.__args__[1] self.shape = SHAPE_MAPPING - # Equality check as almost everything inherit form Iterable, including str - # check for typing.Iterable and abc.Iterable, as it could receive one even when declared with the other - elif origin == Iterable or origin == abc.Iterable: + # Equality check as almost everything inherits form Iterable, including str + # check for Iterable and CollectionsIterable, as it could receive one even when declared with the other + elif origin in {Iterable, CollectionsIterable}: self.type_ = self.type_.__args__[0] self.shape = SHAPE_ITERABLE self.sub_fields = [self._create_sub_type(self.type_, f'{self.name}_type')] @@ -569,12 +569,10 @@ def _validate_iterable( This intentionally doesn't validate values to allow infinite generators. """ - e: Optional[Exception] = None try: iterable = iter(v) except TypeError: - e = errors_.IterableError() - return v, ErrorWrapper(e, loc) + return v, ErrorWrapper(errors_.IterableError(), loc) return iterable, None def _validate_tuple( From fa75fd15bd5d4cbce26bd28ae342f734dd4c358a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Sat, 11 Jan 2020 16:05:09 +0100 Subject: [PATCH 11/12] :memo: Update example from code review --- docs/examples/types_infinite_generator_validate_first.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/examples/types_infinite_generator_validate_first.py b/docs/examples/types_infinite_generator_validate_first.py index cd0006b8bf..aaf548504b 100644 --- a/docs/examples/types_infinite_generator_validate_first.py +++ b/docs/examples/types_infinite_generator_validate_first.py @@ -31,11 +31,6 @@ def infinite_ints(): m = Model(infinite=infinite_ints()) print(m) -for i in m.infinite: - print(i) - if i == 10: - break - def infinite_strs(): while True: for letter in 'allthesingleladies': From dbe25ccb577203d3bae3dfef3d21580658f6f124 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Sat, 11 Jan 2020 16:05:29 +0100 Subject: [PATCH 12/12] :memo: Update docs and format from code review --- docs/usage/types.md | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/docs/usage/types.md b/docs/usage/types.md index 2006beb6d0..6bc88c0cd4 100644 --- a/docs/usage/types.md +++ b/docs/usage/types.md @@ -162,9 +162,12 @@ _(This script is complete, it should run "as is")_ ### Infinite Generators -If you have a generator you can use `Sequence` as described above. In that case, the generator will be consumed and its values will be validated with the sub-type of `Sequence` (e.g. `int` in `Sequence[int]`). +If you have a generator you can use `Sequence` as described above. In that case, the +generator will be consumed and stored on the model as a list and its values will be +validated with the sub-type of `Sequence` (e.g. `int` in `Sequence[int]`). -But if you have a generator that you don't want to be consumed, e.g. an infinite generator or a remote data loader, you can define its type with `Iterable`: +But if you have a generator that you don't want to be consumed, e.g. an infinite +generator or a remote data loader, you can define its type with `Iterable`: ```py {!.tmp_examples/types_infinite_generator.py!} @@ -172,18 +175,23 @@ But if you have a generator that you don't want to be consumed, e.g. an infinite _(This script is complete, it should run "as is")_ !!! warning - `Iterable` fields only perform a simple check that the argument is iterable and won't be consumed. + `Iterable` fields only perform a simple check that the argument is iterable and + won't be consumed. - No validation of their values is performed as it cannot be done without consuming the iterable. + No validation of their values is performed as it cannot be done without consuming + the iterable. !!! tip - If you want to validate the values of an infinite generator you can create a separate model and use it while consuming the generator, reporting the validation errors as appropriate. + If you want to validate the values of an infinite generator you can create a + separate model and use it while consuming the generator, reporting the validation + errors as appropriate. - pydantic can't validate the values automatically for you because it would require consuming the infinite generator. + pydantic can't validate the values automatically for you because it would require + consuming the infinite generator. -#### Infinite Generators with Validation for First Value +## Validating the first value -You can create a [Validator](validators.md) to validate the first value in an infinite generator and still not consume it entirely. +You can create a [validator](validators.md) to validate the first value in an infinite generator and still not consume it entirely. ```py {!.tmp_examples/types_infinite_generator_validate_first.py!}