Add support for generics with __get_validators__ (#1159)

* ✨ Add support for generics with __get_validators__

* ✅ Add tests for Generics with __get_validators__

* 📝 Add change note

* ✨ Add support for Generic fields with validation of sub-types

* 📝 Add docs for arbitrary generic types

* ✅ Add tests for generic sub-type validation

* 📝 Update change note. Generic support is not so "basic" now

* 📝 Update docs with code review

* ♻️ Update fields module with code review changes

* ✅ Update tests from code review

* 📝 Update example for generics, try to simplify and explain better

* tweak docs example

Co-authored-by: Samuel Colvin <samcolvin@gmail.com>

changes/1159-tiangolo.md

@@ -0,0 +1 @@
+add support for generics that implement `__get_validators__` like a custom data type.

docs/examples/types_arbitrary_allowed.py

@@ -0,0 +1,34 @@
+from pydantic import BaseModel, ValidationError
+
+# This is not a pydantic model, it's an arbitrary class
+class Pet:
+    def __init__(self, name: str):
+        self.name = name
+
+class Model(BaseModel):
+    pet: Pet
+    owner: str
+
+    class Config:
+        arbitrary_types_allowed = True
+
+pet = Pet(name='Hedwig')
+# A simple check of instance type is used to validate the data
+model = Model(owner='Harry', pet=pet)
+print(model)
+print(model.pet)
+print(model.pet.name)
+print(type(model.pet))
+try:
+    # If the value is not an instance of the type, it's invalid
+    Model(owner='Harry', pet='Hedwig')
+except ValidationError as e:
+    print(e)
+# Nothing in the instance of the arbitrary type is checked
+# Here name probably should have been a str, but it's not validated
+pet2 = Pet(name=42)
+model2 = Model(owner='Harry', pet=pet2)
+print(model2)
+print(model2.pet)
+print(model2.pet.name)
+print(type(model2.pet))

docs/examples/types_generics.py

@@ -0,0 +1,80 @@
+from pydantic import BaseModel, ValidationError
+from pydantic.fields import ModelField
+from typing import TypeVar, Generic
+
+AgedType = TypeVar('AgedType')
+QualityType = TypeVar('QualityType')
+
+# This is not a pydantic model, it's an arbitrary generic class
+class TastingModel(Generic[AgedType, QualityType]):
+    def __init__(self, name: str, aged: AgedType, quality: QualityType):
+        self.name = name
+        self.aged = aged
+        self.quality = quality
+
+    @classmethod
+    def __get_validators__(cls):
+        yield cls.validate
+
+    @classmethod
+    # You don't need to add the "ModelField", but it will help your
+    # editor give you completion and catch errors
+    def validate(cls, v, field: ModelField):
+        if not isinstance(v, cls):
+            # The value is not even a TastingModel
+            raise TypeError('Invalid value')
+        if not field.sub_fields:
+            # Generic parameters were not provided so we don't try to validate
+            # them and just return the value as is
+            return v
+        aged_f = field.sub_fields[0]
+        quality_f = field.sub_fields[1]
+        errors = []
+        # Here we don't need the validated value, but we want the errors
+        valid_value, error = aged_f.validate(v.aged, {}, loc='aged')
+        if error:
+            errors.append(error)
+        # Here we don't need the validated value, but we want the errors
+        valid_value, error = quality_f.validate(v.quality, {}, loc='quality')
+        if error:
+            errors.append(error)
+        if errors:
+            raise ValidationError(errors, cls)
+        # Validation passed without errors, return the same instance received
+        return v
+
+class Model(BaseModel):
+    # for wine, "aged" is an int with years, "quality" is a float
+    wine: TastingModel[int, float]
+    # for cheese, "aged" is a bool, "quality" is a str
+    cheese: TastingModel[bool, str]
+    # for thing, "aged" is a Any, "quality" is Any
+    thing: TastingModel
+
+model = Model(
+    # This wine was aged for 20 years and has a quality of 85.6
+    wine=TastingModel(name='Cabernet Sauvignon', aged=20, quality=85.6),
+    # This cheese is aged (is mature) and has "Good" quality
+    cheese=TastingModel(name='Gouda', aged=True, quality='Good'),
+    # This Python thing has aged "Not much" and has a quality "Awesome"
+    thing=TastingModel(name='Python', aged='Not much', quality='Awesome')
+)
+print(model)
+print(model.wine.aged)
+print(model.wine.quality)
+print(model.cheese.aged)
+print(model.cheese.quality)
+print(model.thing.aged)
+try:
+    # If the values of the sub-types are invalid, we get an error
+    Model(
+        # For wine, aged should be an int with the years, and quality a float
+        wine=TastingModel(name='Merlot', aged=True, quality='Kinda good'),
+        # For cheese, aged should be a bool, and quality a str
+        cheese=TastingModel(name='Gouda', aged='yeah', quality=5),
+        # For thing, no type parameters are declared, and we skipped validation
+        # in those cases in the Assessment.validate() function
+        thing=TastingModel(name='Python', aged='Not much', quality='Awesome')
+    )
+except ValidationError as e:
+    print(e)

docs/usage/model_config.md

@@ -48,8 +48,10 @@ Options:
   Pass in a dictionary with keys matching the error messages you want to override (default: `{}`)
 
 **`arbitrary_types_allowed`**
-: whether to allow arbitrary user types for fields (they are validated simply by checking if the
-  value is an instance of the type). If `False`, `RuntimeError` will be raised on model declaration (default: `False`)
+: whether to allow arbitrary user types for fields (they are validated simply by
+  checking if the value is an instance of the type). If `False`, `RuntimeError` will be
+  raised on model declaration (default: `False`). See an example in
+  [Field Types](types.md#arbitrary-types-allowed).
 
 **`orm_mode`**
 : whether to allow usage of [ORM mode](models.md#orm-mode)

docs/usage/types.md

@@ -732,13 +732,19 @@ raw bytes and print out human readable versions of the bytes as well.
 ```
 _(This script is complete, it should run "as is")_
 
+## Custom Data Types
 
+You can also define your own custom data types. There are several ways to achieve it.
 
-## Custom Data Types
+### Classes with `__get_validators__`
 
-You can also define your own custom data types. The classmethod `__get_validators__` will be called
+You use a custom class with a classmethod `__get_validators__`. It will be called
 to get validators to parse and validate the input data.
 
+!!! tip
+    These validators have the same semantics as in [Validators](validators.md), you can
+    declare a parameter `config`, `field`, etc.
+
 ```py
 {!.tmp_examples/types_custom_type.py!}
 ```
@@ -748,3 +754,35 @@ Similar validation could be achieved using [`constr(regex=...)`](#constrained-ty
 formatted with a space, the schema would just include the full pattern and the returned value would be a vanilla string.
 
 See [Schema](schema.md) for more details on how the model's schema is generated.
+
+### Arbitrary Types Allowed
+
+You can allow arbitrary types using the `arbitrary_types_allowed` config in the
+[Model Config](model_config.md).
+
+```py
+{!.tmp_examples/types_arbitrary_allowed.py!}
+```
+_(This script is complete, it should run "as is")_
+
+### Generic Classes as Types
+
+!!! warning
+    This is an advanced technique that you might not need in the beginning. In most of
+    the cases you will probably be fine with standard *pydantic* models.
+
+You can use
+[Generic Classes](https://docs.python.org/3/library/typing.html#typing.Generic) as
+field types and perform custom validation based on the "type parameters" (or sub-types)
+with `__get_validators__`.
+
+If the Generic class that you are using as a sub-type has a classmethod
+`__get_validators__` you don't need to use `arbitrary_types_allowed` for it to work.
+
+Because you can declare validators that receive the current `field`, you can extract
+the `sub_fields` (from the generic class type parameters) and validate data with them.
+
+```py
+{!.tmp_examples/types_generics.py!}
+```
+_(This script is complete, it should run "as is")_

pydantic/fields.py

@@ -177,6 +177,7 @@ def Schema(default: Any, **kwargs: Any) -> Any:
 SHAPE_SEQUENCE = 7
 SHAPE_FROZENSET = 8
 SHAPE_ITERABLE = 9
+SHAPE_GENERIC = 10
 SHAPE_NAME_LOOKUP = {
     SHAPE_LIST: 'List[{}]',
     SHAPE_SET: 'Set[{}]',
@@ -428,6 +429,13 @@ def _type_analysis(self) -> None:  # noqa: C901 (ignore complexity)
             self.sub_fields = [self._create_sub_type(self.type_, f'{self.name}_type')]
         elif issubclass(origin, Type):  # type: ignore
             return
+        elif hasattr(origin, '__get_validators__') or self.model_config.arbitrary_types_allowed:
+            # Is a Pydantic-compatible generic that handles itself
+            # or we have arbitrary_types_allowed = True
+            self.shape = SHAPE_GENERIC
+            self.sub_fields = [self._create_sub_type(t, f'{self.name}_{i}') for i, t in enumerate(self.type_.__args__)]
+            self.type_ = origin
+            return
         else:
             raise TypeError(f'Fields of type "{origin}" are not supported.')
 
@@ -449,7 +457,7 @@ def populate_validators(self) -> None:
         without mis-configuring the field.
         """
         class_validators_ = self.class_validators.values()
-        if not self.sub_fields:
+        if not self.sub_fields or self.shape == SHAPE_GENERIC:
             get_validators = getattr(self.type_, '__get_validators__', None)
             v_funcs = (
                 *[v.func for v in class_validators_ if v.each_item and v.pre],
@@ -501,6 +509,8 @@ def validate(
             v, errors = self._validate_tuple(v, values, loc, cls)
         elif self.shape == SHAPE_ITERABLE:
             v, errors = self._validate_iterable(v, values, loc, cls)
+        elif self.shape == SHAPE_GENERIC:
+            v, errors = self._apply_validators(v, values, loc, cls, self.validators)
         else:
             #  sequence, list, set, generator, tuple with ellipsis, frozen set
             v, errors = self._validate_sequence_like(v, values, loc, cls)
@@ -685,6 +695,11 @@ def _type_display(self) -> PyObjectStr:
             t = f'Mapping[{display_as_type(self.key_field.type_)}, {t}]'  # type: ignore
         elif self.shape == SHAPE_TUPLE:
             t = 'Tuple[{}]'.format(', '.join(display_as_type(f.type_) for f in self.sub_fields))  # type: ignore
+        elif self.shape == SHAPE_GENERIC:
+            assert self.sub_fields
+            t = '{}[{}]'.format(
+                display_as_type(self.type_), ', '.join(display_as_type(f.type_) for f in self.sub_fields)
+            )
         elif self.shape != SHAPE_SINGLETON:
             t = SHAPE_NAME_LOOKUP[self.shape].format(t)