Allow setting type for values in __pydantic_extra__ (#6139)

docs/usage/models.md

@@ -34,7 +34,6 @@ In this example, `User` is a model with two fields:
 * `id`, which is an integer and is required
 * `name`, which is a string and is not required (it has a default value).
 
-
 ```py group="basic-model"
 user = User(id='123')
 ```
@@ -83,26 +82,26 @@ This model is mutable so field values can be changed.
 The example above only shows the tip of the iceberg of what models can do.
 Models possess the following methods and attributes:
 
-- `model_computed_fields`: a dictionary of the computed fields of this model instance.
-- `model_construct()`: a class method for creating models without running validation. See
+* `model_computed_fields`: a dictionary of the computed fields of this model instance.
+* `model_construct()`: a class method for creating models without running validation. See
     [Creating models without validation](#creating-models-without-validation).
-- `model_copy()`: returns a copy (by default, shallow copy) of the model. See
+* `model_copy()`: returns a copy (by default, shallow copy) of the model. See
     [Exporting models](exporting_models.md#modelcopy).
-- `model_dump()`: returns a dictionary of the model's fields and values. See
+* `model_dump()`: returns a dictionary of the model's fields and values. See
     [Exporting models](exporting_models.md#modeldump).
-- `model_dump_json()`: returns a JSON string representation of `model_dump()`. See
+* `model_dump_json()`: returns a JSON string representation of `model_dump()`. See
     [Exporting models](exporting_models.md#modeldumpjson).
-- `model_extra`: get extra fields set during validation.
-- `model_fields_set`: set of fields which were set when the model instance was initialised.
-- `model_json_schema()`: returns a dictionary representing the model as JSON Schema. See [JSON Schema](json_schema.md).
-- `model_modify_json_schema()`: a method for how the "generic" properties of the JSON schema are populated.
+* `model_extra`: get extra fields set during validation.
+* `model_fields_set`: set of fields which were set when the model instance was initialised.
+* `model_json_schema()`: returns a dictionary representing the model as JSON Schema. See [JSON Schema](json_schema.md).
+* `model_modify_json_schema()`: a method for how the "generic" properties of the JSON schema are populated.
     See [JSON Schema](json_schema.md).
-- `model_parameterized_name()`: compute the class name for parametrizations of generic classes.
-- `model_post_init()`: perform additional initialization after the model is initialised.
-- `model_rebuild()`: rebuild the model schema.
-- `model_validate()`: a utility for loading any object into a model with error handling if the object is not a
+* `model_parameterized_name()`: compute the class name for parametrizations of generic classes.
+* `model_post_init()`: perform additional initialization after the model is initialised.
+* `model_rebuild()`: rebuild the model schema.
+* `model_validate()`: a utility for loading any object into a model with error handling if the object is not a
     dictionary. See [Helper functions](#helper-functions).
-- `model_validate_json()`: a utility for validating the given JSON data against the Pydantic model. See
+* `model_validate_json()`: a utility for validating the given JSON data against the Pydantic model. See
     [Helper functions](#helper-functions).
 
 !!! note
@@ -463,6 +462,7 @@ Note that for subclasses of [`RootModel`](#rootmodel-and-custom-root-types), the
 positionally, instead of using a keyword argument.
 
 Here are some additional notes on the behavior of `model_construct`:
+
 * When we say "no validation is performed" — this includes converting dicts to model instances. So if you have a field
   with a `Model` type, you will need to convert the inner dict to a model yourself before passing it to
   `model_construct`.
@@ -475,7 +475,6 @@ Here are some additional notes on the behavior of `model_construct`:
 * When constructing an instance using `model_construct()`, no `__init__` method from the model or any of its parent
   classes will be called, even when a custom `__init__` method is defined.
 
-
 ## Generic models
 
 Pydantic supports the creation of generic models to make it easier to reuse a common model structure.
@@ -1247,7 +1246,7 @@ To be included in the signature, a field's alias or name must be a valid Python
 Pydantic prefers aliases over names, but may use field names if the alias is not a valid Python identifier.
 
 If a field's alias and name are both invalid identifiers, a `**data` argument will be added.
-In addition, the `**data` argument will always be present in the signature if `Config.extra` is `Extra.allow`.
+In addition, the `**data` argument will always be present in the signature if `Config.extra` is `'allow'`.
 
 ## Structural pattern matching
 
@@ -1312,3 +1311,94 @@ print('id(c1.arr) == id(c2.arr)  ', id(c1.arr) == id(c2.arr))
 
 !!! note
     There are some situations where Pydantic does not copy attributes, such as when passing models — we use the model as is. You can override this behaviour by setting [`config.revalidate_instances='always'`](../api/config.md#pydantic.config.ConfigDict) in your model.
+
+## Extra fields
+
+By default Pydantic models won't error when you provide fields that don't belong to the model, it will discard them instead:
+
+```py
+from pydantic import BaseModel
+
+
+class Model(BaseModel):
+    x: int
+
+
+m = Model(x=1, y='a')
+assert m.model_dump() == {'x': 1}
+```
+
+If you want to error instead you can set this via `model_config`:
+
+```py
+from pydantic import BaseModel, ConfigDict, ValidationError
+
+
+class Model(BaseModel):
+    x: int
+
+    model_config = ConfigDict(extra='forbid')
+
+
+try:
+    Model(x=1, y='a')
+except ValidationError as exc:
+    print(exc)
+    """
+    1 validation error for Model
+    y
+      Extra inputs are not permitted [type=extra_forbidden, input_value='a', input_type=str]
+    """
+```
+
+To preserve this data instead you can set `extra='allow'`.
+The extra fields will then be stored in `BaseModel.__pydantic_extra__`.
+
+```py
+from pydantic import BaseModel, ConfigDict
+
+
+class Model(BaseModel):
+    x: int
+
+    model_config = ConfigDict(extra='allow')
+
+
+m = Model(x=1, y='a')
+assert m.__pydantic_extra__ == {'y': 'a'}
+```
+
+By default no validation will be applied to these extra items, but you can set a type for the values by overriding the type annotation for `__pydantic_extra__`:
+
+```py
+from typing import Dict
+
+from pydantic import BaseModel, ConfigDict, ValidationError
+
+
+class Model(BaseModel):
+    __pydantic_extra__: Dict[str, int]
+
+    x: int
+
+    model_config = ConfigDict(extra='allow')
+
+
+try:
+    Model(x=1, y='a')
+except ValidationError as exc:
+    print(exc)
+    """
+    1 validation error for Model
+    y
+      Input should be a valid integer, unable to parse string as an integer [type=int_parsing, input_value='a', input_type=str]
+    """
+
+m = Model(x=1, y='2')
+assert m.x == 1
+assert m.y == 2
+assert m.model_dump() == {'x': 1, 'y': 2}
+assert m.__pydantic_extra__ == {'y': 2}
+```
+
+The same configurations apply to `TypedDict` and `dataclass`' except the config is set via a `__pydantic_config__`.

pydantic/_internal/_generate_schema.py

@@ -15,7 +15,7 @@
 from itertools import chain
 from operator import attrgetter
 from types import FunctionType, LambdaType, MethodType
-from typing import TYPE_CHECKING, Any, Callable, ForwardRef, Iterable, Iterator, Mapping, TypeVar, Union, cast
+from typing import TYPE_CHECKING, Any, Callable, Dict, ForwardRef, Iterable, Iterator, Mapping, TypeVar, Union, cast
 
 from pydantic_core import CoreSchema, core_schema
 from typing_extensions import Annotated, Final, Literal, TypeAliasType, TypedDict, get_args, get_origin, is_typeddict
@@ -320,6 +320,21 @@ def _model_schema(self, cls: type[BaseModel]) -> core_schema.CoreSchema:
 
             model_validators = decorators.model_validators.values()
 
+            extra_validator = None
+            if core_config.get('extra_fields_behavior') == 'allow':
+                for tp in (cls, *cls.__mro__):
+                    extras_annotation = cls.__annotations__.get('__pydantic_extra__', None)
+                    if extras_annotation is not None:
+                        tp = get_origin(extras_annotation)
+                        if tp not in (Dict, dict):
+                            raise PydanticSchemaGenerationError(
+                                'The type annotation for `__pydantic_extra__` must be `Dict[str, ...]`'
+                            )
+                        extra_items_type = get_args(cls.__annotations__['__pydantic_extra__'])[1]
+                        if extra_items_type is not Any:
+                            extra_validator = self.generate_schema(extra_items_type)
+                            break
+
             if cls.__pydantic_root_model__:
                 root_field = self._common_field_schema('root', fields['root'], decorators)
                 inner_schema = root_field['schema']
@@ -340,6 +355,7 @@ def _model_schema(self, cls: type[BaseModel]) -> core_schema.CoreSchema:
                     fields_schema: core_schema.CoreSchema = core_schema.model_fields_schema(
                         {k: self._generate_md_field_schema(k, v, decorators) for k, v in fields.items()},
                         computed_fields=[self._computed_field_schema(d) for d in decorators.computed_fields.values()],
+                        extra_validator=extra_validator,
                     )
                 finally:
                     self._config_wrapper_stack.pop()

pydantic/json_schema.py

@@ -1000,10 +1000,11 @@ def _update_class_schema(
             # referenced_schema['title'] = title
             schema_to_update.setdefault('title', title)
 
-        if extra == 'allow':
-            schema_to_update['additionalProperties'] = True
-        elif extra == 'forbid':
-            schema_to_update['additionalProperties'] = False
+        if 'additionalProperties' not in schema_to_update:
+            if extra == 'allow':
+                schema_to_update['additionalProperties'] = True
+            elif extra == 'forbid':
+                schema_to_update['additionalProperties'] = False
 
         if isinstance(json_schema_extra, (staticmethod, classmethod)):
             # In older versions of python, this is necessary to ensure staticmethod/classmethods are callable
@@ -1023,6 +1024,17 @@ def _update_class_schema(
 
         return json_schema
 
+    def resolve_schema_to_update(self, json_schema: JsonSchemaValue) -> JsonSchemaValue:
+        """Resolve a JsonSchemaValue to the non-ref schema if it is a $ref schema"""
+        if '$ref' in json_schema:
+            schema_to_update = self.get_schema_from_definitions(JsonRef(json_schema['$ref']))
+            if schema_to_update is None:
+                raise RuntimeError(f'Cannot update undefined schema for $ref={json_schema["$ref"]}')
+            return self.resolve_schema_to_update(schema_to_update)
+        else:
+            schema_to_update = json_schema
+        return schema_to_update
+
     def model_fields_schema(self, schema: core_schema.ModelFieldsSchema) -> JsonSchemaValue:
         named_required_fields: list[tuple[str, bool, CoreSchemaField]] = [
             (name, self.field_is_required(field), field)
@@ -1031,7 +1043,12 @@ def model_fields_schema(self, schema: core_schema.ModelFieldsSchema) -> JsonSche
         ]
         if self.mode == 'serialization':
             named_required_fields.extend(self._name_required_computed_fields(schema.get('computed_fields', [])))
-        return self._named_required_fields_schema(named_required_fields)
+        json_schema = self._named_required_fields_schema(named_required_fields)
+        extra_validator = schema.get('extra_validator', None)
+        if extra_validator is not None:
+            schema_to_update = self.resolve_schema_to_update(json_schema)
+            schema_to_update['additionalProperties'] = self.generate_inner(extra_validator)
+        return json_schema
 
     def field_is_present(self, field: CoreSchemaField) -> bool:
         """Whether the field should be included in the generated JSON schema."""

tests/test_main.py

@@ -2467,3 +2467,52 @@ class Outer(BaseModel):
         inner: Annotated[Inner, Marker()]
 
     assert Outer.model_validate({'inner': {'x': 1}}).inner.x == 2
+
+
+def test_extra_validator_scalar() -> None:
+    class Model(BaseModel):
+        model_config = ConfigDict(extra='allow')
+
+    class Child(Model):
+        __pydantic_extra__: Dict[str, int]
+
+    m = Child(a='1')
+    assert m.__pydantic_extra__ == {'a': 1}
+
+    # insert_assert(Child.model_json_schema())
+    assert Child.model_json_schema() == {
+        'additionalProperties': {'type': 'integer'},
+        'properties': {},
+        'title': 'Child',
+        'type': 'object',
+    }
+
+
+def test_extra_validator_named() -> None:
+    class Foo(BaseModel):
+        x: int
+
+    class Model(BaseModel):
+        model_config = ConfigDict(extra='allow')
+
+    class Child(Model):
+        __pydantic_extra__: Dict[str, Foo]
+
+    m = Child(a={'x': '1'})
+    assert m.__pydantic_extra__ == {'a': Foo(x=1)}
+
+    # insert_assert(Child.model_json_schema())
+    assert Child.model_json_schema() == {
+        '$defs': {
+            'Foo': {
+                'properties': {'x': {'title': 'X', 'type': 'integer'}},
+                'required': ['x'],
+                'title': 'Foo',
+                'type': 'object',
+            }
+        },
+        'additionalProperties': {'$ref': '#/$defs/Foo'},
+        'properties': {},
+        'title': 'Child',
+        'type': 'object',
+    }