Implement support for declaring infinite generators

changes/1152-tiangolo.md

@@ -0,0 +1 @@
+add support for infinite generators with `Iterable`

docs/examples/types_infinite_generator.py

@@ -0,0 +1,19 @@
+from typing import Iterable
+from pydantic import BaseModel
+
+class Model(BaseModel):
+    infinite: Iterable[int]
+
+def infinite_ints():
+    i = 0
+    while True:
+        yield i
+        i += 1
+
+m = Model(infinite=infinite_ints())
+print(m)
+
+for i in m.infinite:
+    print(i)
+    if i == 10:
+        break

docs/examples/types_infinite_generator_validate_first.py

@@ -0,0 +1,42 @@
+import itertools
+from typing import Iterable
+from pydantic import BaseModel, validator, ValidationError
+from pydantic.fields import ModelField
+
+class Model(BaseModel):
+    infinite: Iterable[int]
+
+    @validator('infinite')
+    # You don't need to add the "ModelField", but it will help your
+    # editor give you completion and catch errors
+    def infinite_first_int(cls, iterable, field: ModelField):
+        first_value = next(iterable)
+        if field.sub_fields:
+            # The Iterable had a parameter type, in this case it's int
+            # We use it to validate the first value
+            sub_field = field.sub_fields[0]
+            v, error = sub_field.validate(first_value, {}, loc='first_value')
+            if error:
+                raise ValidationError([error], cls)
+        # This creates a new generator that returns the first value and then
+        # the rest of the values from the (already started) iterable
+        return itertools.chain([first_value], iterable)
+
+def infinite_ints():
+    i = 0
+    while True:
+        yield i
+        i += 1
+
+m = Model(infinite=infinite_ints())
+print(m)
+
+def infinite_strs():
+    while True:
+        for letter in 'allthesingleladies':
+            yield letter
+
+try:
+    Model(infinite=infinite_strs())
+except ValidationError as e:
+    print(e)

docs/usage/types.md

@@ -93,6 +93,9 @@ with custom properties and validation.
 `typing.Sequence`
 : see [Typing Iterables](#typing-iterables) below for more detail on parsing and validation
 
+`typing.Iterable`
+: this is reserved for iterables that shouldn't be consumed. See [Infinite Generators](#infinite-generators) below for more detail on parsing and validation
+
 `typing.Type`
 : see [Type](#type) below for more detail on parsing and validation
 
@@ -157,6 +160,44 @@ with custom properties and validation.
 ```
 _(This script is complete, it should run "as is")_
 
+### Infinite Generators
+
+If you have a generator you can use `Sequence` as described above. In that case, the
+generator will be consumed and stored on the model as a list and its values will be
+validated with the sub-type of `Sequence` (e.g. `int` in `Sequence[int]`).
+
+But if you have a generator that you don't want to be consumed, e.g. an infinite
+generator or a remote data loader, you can define its type with `Iterable`:
+
+```py
+{!.tmp_examples/types_infinite_generator.py!}
+```
+_(This script is complete, it should run "as is")_
+
+!!! warning
+    `Iterable` fields only perform a simple check that the argument is iterable and
+    won't be consumed.
+
+    No validation of their values is performed as it cannot be done without consuming
+    the iterable.
+
+!!! tip
+    If you want to validate the values of an infinite generator you can create a
+    separate model and use it while consuming the generator, reporting the validation
+    errors as appropriate.
+
+    pydantic can't validate the values automatically for you because it would require
+    consuming the infinite generator.
+
+## Validating the first value
+
+You can create a [validator](validators.md) to validate the first value in an infinite generator and still not consume it entirely.
+
+```py
+{!.tmp_examples/types_infinite_generator_validate_first.py!}
+```
+_(This script is complete, it should run "as is")_
+
 ### Unions
 
 The `Union` type allows a model attribute to accept different types, e.g.:

pydantic/errors.py

@@ -238,6 +238,10 @@ class SequenceError(PydanticTypeError):
     msg_template = 'value is not a valid sequence'
 
 
+class IterableError(PydanticTypeError):
+    msg_template = 'value is not a valid iterable'
+
+
 class ListError(PydanticTypeError):
     msg_template = 'value is not a valid list'
 

pydantic/fields.py

@@ -1,10 +1,12 @@
 import warnings
+from collections.abc import Iterable as CollectionsIterable
 from typing import (
     TYPE_CHECKING,
     Any,
     Dict,
     FrozenSet,
     Generator,
+    Iterable,
     Iterator,
     List,
     Mapping,
@@ -174,12 +176,14 @@ def Schema(default: Any, **kwargs: Any) -> Any:
 SHAPE_TUPLE_ELLIPSIS = 6
 SHAPE_SEQUENCE = 7
 SHAPE_FROZENSET = 8
+SHAPE_ITERABLE = 9
 SHAPE_NAME_LOOKUP = {
     SHAPE_LIST: 'List[{}]',
     SHAPE_SET: 'Set[{}]',
     SHAPE_TUPLE_ELLIPSIS: 'Tuple[{}, ...]',
     SHAPE_SEQUENCE: 'Sequence[{}]',
     SHAPE_FROZENSET: 'FrozenSet[{}]',
+    SHAPE_ITERABLE: 'Iterable[{}]',
 }
 
 
@@ -416,6 +420,12 @@ def _type_analysis(self) -> None:  # noqa: C901 (ignore complexity)
             self.key_field = self._create_sub_type(self.type_.__args__[0], 'key_' + self.name, for_keys=True)
             self.type_ = self.type_.__args__[1]
             self.shape = SHAPE_MAPPING
+        # Equality check as almost everything inherits form Iterable, including str
+        # check for Iterable and CollectionsIterable, as it could receive one even when declared with the other
+        elif origin in {Iterable, CollectionsIterable}:
+            self.type_ = self.type_.__args__[0]
+            self.shape = SHAPE_ITERABLE
+            self.sub_fields = [self._create_sub_type(self.type_, f'{self.name}_type')]
         elif issubclass(origin, Type):  # type: ignore
             return
         else:
@@ -489,6 +499,8 @@ def validate(
             v, errors = self._validate_mapping(v, values, loc, cls)
         elif self.shape == SHAPE_TUPLE:
             v, errors = self._validate_tuple(v, values, loc, cls)
+        elif self.shape == SHAPE_ITERABLE:
+            v, errors = self._validate_iterable(v, values, loc, cls)
         else:
             #  sequence, list, set, generator, tuple with ellipsis, frozen set
             v, errors = self._validate_sequence_like(v, values, loc, cls)
@@ -548,6 +560,21 @@ def _validate_sequence_like(  # noqa: C901 (ignore complexity)
                 converted = iter(result)
         return converted, None
 
+    def _validate_iterable(
+        self, v: Any, values: Dict[str, Any], loc: 'LocStr', cls: Optional['ModelOrDc']
+    ) -> 'ValidateReturn':
+        """
+        Validate Iterables.
+
+        This intentionally doesn't validate values to allow infinite generators.
+        """
+
+        try:
+            iterable = iter(v)
+        except TypeError:
+            return v, ErrorWrapper(errors_.IterableError(), loc)
+        return iterable, None
+
     def _validate_tuple(
         self, v: Any, values: Dict[str, Any], loc: 'LocStr', cls: Optional['ModelOrDc']
     ) -> 'ValidateReturn':

pydantic/schema.py

@@ -26,6 +26,7 @@
 from .class_validators import ROOT_KEY
 from .fields import (
     SHAPE_FROZENSET,
+    SHAPE_ITERABLE,
     SHAPE_LIST,
     SHAPE_MAPPING,
     SHAPE_SEQUENCE,
@@ -375,7 +376,7 @@ def field_type_schema(
     nested_models: Set[str] = set()
     f_schema: Dict[str, Any]
     ref_prefix = ref_prefix or default_prefix
-    if field.shape in {SHAPE_LIST, SHAPE_TUPLE_ELLIPSIS, SHAPE_SEQUENCE, SHAPE_SET, SHAPE_FROZENSET}:
+    if field.shape in {SHAPE_LIST, SHAPE_TUPLE_ELLIPSIS, SHAPE_SEQUENCE, SHAPE_SET, SHAPE_FROZENSET, SHAPE_ITERABLE}:
         items_schema, f_definitions, f_nested_models = field_singleton_schema(
             field, by_alias=by_alias, model_name_map=model_name_map, ref_prefix=ref_prefix, known_models=known_models
         )

tests/test_schema.py

@@ -6,7 +6,7 @@
 from enum import Enum, IntEnum
 from ipaddress import IPv4Address, IPv4Interface, IPv4Network, IPv6Address, IPv6Interface, IPv6Network
 from pathlib import Path
-from typing import Any, Callable, Dict, FrozenSet, List, NewType, Optional, Set, Tuple, Union
+from typing import Any, Callable, Dict, FrozenSet, Iterable, List, NewType, Optional, Set, Tuple, Union
 from uuid import UUID
 
 import pytest
@@ -1746,3 +1746,15 @@ class Model(BaseModel):
             }
         },
     }
+
+
+def test_iterable():
+    class Model(BaseModel):
+        a: Iterable[int]
+
+    assert Model.schema() == {
+        'title': 'Model',
+        'type': 'object',
+        'properties': {'a': {'title': 'A', 'type': 'array', 'items': {'type': 'integer'}}},
+        'required': ['a'],
+    }

tests/test_types.py

@@ -1,3 +1,4 @@
+import itertools
 import os
 import sys
 import uuid
@@ -6,7 +7,20 @@
 from decimal import Decimal
 from enum import Enum, IntEnum
 from pathlib import Path
-from typing import Dict, FrozenSet, Iterator, List, MutableSet, NewType, Optional, Pattern, Sequence, Set, Tuple
+from typing import (
+    Dict,
+    FrozenSet,
+    Iterable,
+    Iterator,
+    List,
+    MutableSet,
+    NewType,
+    Optional,
+    Pattern,
+    Sequence,
+    Set,
+    Tuple,
+)
 from uuid import UUID
 
 import pytest
@@ -775,6 +789,83 @@ class Model(BaseModel):
     assert list(validated) == list(result)
 
 
+def test_infinite_iterable():
+    class Model(BaseModel):
+        it: Iterable[int]
+        b: int
+
+    def iterable():
+        i = 0
+        while True:
+            i += 1
+            yield i
+
+    m = Model(it=iterable(), b=3)
+
+    assert m.b == 3
+    assert m.it
+
+    for i in m.it:
+        assert i
+        if i == 10:
+            break
+
+
+def test_invalid_iterable():
+    class Model(BaseModel):
+        it: Iterable[int]
+        b: int
+
+    with pytest.raises(ValidationError) as exc_info:
+        Model(it=3, b=3)
+    assert exc_info.value.errors() == [
+        {'loc': ('it',), 'msg': 'value is not a valid iterable', 'type': 'type_error.iterable'}
+    ]
+
+
+def test_infinite_iterable_validate_first():
+    class Model(BaseModel):
+        it: Iterable[int]
+        b: int
+
+        @validator('it')
+        def infinite_first_int(cls, it, field):
+            first_value = next(it)
+            if field.sub_fields:
+                sub_field = field.sub_fields[0]
+                v, error = sub_field.validate(first_value, {}, loc='first_value')
+                if error:
+                    raise ValidationError([error], cls)
+            return itertools.chain([first_value], it)
+
+    def int_iterable():
+        i = 0
+        while True:
+            i += 1
+            yield i
+
+    m = Model(it=int_iterable(), b=3)
+
+    assert m.b == 3
+    assert m.it
+
+    for i in m.it:
+        assert i
+        if i == 10:
+            break
+
+    def str_iterable():
+        while True:
+            for c in 'foobarbaz':
+                yield c
+
+    with pytest.raises(ValidationError) as exc_info:
+        Model(it=str_iterable(), b=3)
+    assert exc_info.value.errors() == [
+        {'loc': ('it', 'first_value'), 'msg': 'value is not a valid integer', 'type': 'type_error.integer'}
+    ]
+
+
 @pytest.mark.parametrize(
     'cls,value,errors',
     (