Merge branch 'python-attrs:main' into class_methods_strategy

HISTORY.md

@@ -5,9 +5,13 @@
 - **Potentially breaking**: skip _attrs_ fields marked as `init=False` by default. This change is potentially breaking for unstructuring.
   See [here](https://catt.rs/en/latest/customizing.html#include_init_false) for instructions on how to restore the old behavior.
   ([#40](https://github.com/python-attrs/cattrs/issues/40) [#395](https://github.com/python-attrs/cattrs/pull/395))
-- The `omit` parameter of `cattrs.override()` is now of type `bool | None` (from `bool`). `None` is the new default and means to apply default `cattrs` handling to the attribute.
-- Fix `format_exception` parameter working for recursive calls to `transform_error`
-  ([#389](https://github.com/python-attrs/cattrs/issues/389)
+- **Potentially breaking**: {py:func}`cattrs.gen.make_dict_structure_fn` and {py:func}`cattrs.gen.typeddicts.make_dict_structure_fn` will use the values for the `detailed_validation` and `forbid_extra_keys` parameters from the given converter by default now.
+  If you're using these functions directly, the old behavior can be restored by passing in the desired values directly.
+  ([#410](https://github.com/python-attrs/cattrs/issues/410) [#411](https://github.com/python-attrs/cattrs/pull/411))
+- The `omit` parameter of {py:func}`cattrs.override` is now of type `bool | None` (from `bool`).
+  `None` is the new default and means to apply default _cattrs_ handling to the attribute, which is to omit the attribute if it's marked as `init=False`, and keep it otherwise.
+- Fix {py:func}`format_exception() <cattrs.v.format_exception>` parameter working for recursive calls to {py:func}`transform_error <cattrs.transform_error>`.
+  ([#389](https://github.com/python-attrs/cattrs/issues/389))
 - [_attrs_ aliases](https://www.attrs.org/en/stable/init.html#private-attributes-and-aliases) are now supported, although aliased fields still map to their attribute name instead of their alias by default when un/structuring.
   ([#322](https://github.com/python-attrs/cattrs/issues/322) [#391](https://github.com/python-attrs/cattrs/pull/391))
 - Use [PDM](https://pdm.fming.dev/latest/) instead of Poetry.
@@ -16,12 +20,23 @@
   ([#376](https://github.com/python-attrs/cattrs/issues/376) [#377](https://github.com/python-attrs/cattrs/pull/377))
 - Optimize and improve unstructuring of `Optional` (unions of one type and `None`).
   ([#380](https://github.com/python-attrs/cattrs/issues/380) [#381](https://github.com/python-attrs/cattrs/pull/381))
-- Fix `format_exception` and `transform_error` type annotations.
+- Fix {py:func}`format_exception <cattrs.v.format_exception>` and {py:func}`transform_error <cattrs.transform_error>` type annotations.
 - Improve the implementation of `cattrs._compat.is_typeddict`. The implementation is now simpler, and relies on fewer private implementation details from `typing` and typing_extensions.
   ([#384](https://github.com/python-attrs/cattrs/pull/384))
 - Improve handling of TypedDicts with forward references.
 - Speed up generated _attrs_ and TypedDict structuring functions by changing their signature slightly.
   ([#388](https://github.com/python-attrs/cattrs/pull/388))
+- Fix copying of converters with function hooks.
+  ([#398](https://github.com/python-attrs/cattrs/issues/398) [#399](https://github.com/python-attrs/cattrs/pull/399))
+- Broaden {py:func}`loads' <cattrs.preconf.orjson.OrjsonConverter.loads>` type definition for the preconf orjson converter.
+  ([#400](https://github.com/python-attrs/cattrs/pull/400))
+- {py:class}`AttributeValidationNote <cattrs.AttributeValidationNote>` and {py:class}`IterableValidationNote <cattrs.IterableValidationNote>` are now picklable.
+  ([#408](https://github.com/python-attrs/cattrs/pull/408))
+- Fix structuring `Final` lists.
+  ([#412](https://github.com/python-attrs/cattrs/issues/412))
+- Fix certain cases of structuring `Annotated` types.
+  ([#418](https://github.com/python-attrs/cattrs/issues/418))
+
 
 ## 23.1.2 (2023-06-02)
 

docs/customizing.md

@@ -102,6 +102,11 @@ TestClass(number=1)
 
 This behavior can only be applied to classes or to the default for the {class}`Converter <cattrs.Converter>`, and has no effect when generating unstructuring functions.
 
+```{versionchanged} 23.2.0
+The value for the `make_dict_structure_fn._cattrs_forbid_extra_keys` parameter is now taken from the given converter by default.
+```
+
+
 ### `rename`
 
 Using the rename override makes `cattrs` simply use the provided name instead

src/cattrs/converters.py

@@ -127,6 +127,8 @@ class BaseConverter:
         "_structure_func",
         "_prefer_attrib_converters",
         "detailed_validation",
+        "_struct_copy_skip",
+        "_unstruct_copy_skip",
     )
 
     def __init__(
@@ -228,6 +230,9 @@ def __init__(
         # Unions are instances now, not classes. We use different registries.
         self._union_struct_registry: Dict[Any, Callable[[Any, Type[T]], T]] = {}
 
+        self._unstruct_copy_skip = self._unstructure_func.get_num_fns()
+        self._struct_copy_skip = self._structure_func.get_num_fns()
+
     def unstructure(self, obj: Any, unstructure_as: Any = None) -> Any:
         return self._unstructure_func.dispatch(
             obj.__class__ if unstructure_as is None else unstructure_as
@@ -464,7 +469,7 @@ def _structure_final_factory(self, type):
         if res == self._structure_call:
             # It's not really `structure_call` for Finals (can't call Final())
             return lambda v, _: self._structure_call(v, base)
-        return res
+        return lambda v, _: res(v, base)
 
     # Attrs classes.
 
@@ -601,7 +606,7 @@ def _structure_set(
                         ix,
                         elem_type,
                     )
-                    exc.__notes__ = [*getattr(e, "__notes__", []), msg]
+                    exc.__notes__ = [*getattr(exc, "__notes__", []), msg]
                     errors.append(exc)
                 finally:
                     ix += 1
@@ -668,7 +673,7 @@ def _structure_tuple(self, obj: Any, tup: Type[T]) -> T:
                         msg = IterableValidationNote(
                             f"Structuring {tup} @ index {ix}", ix, tup_type
                         )
-                        exc.__notes__ = [*getattr(e, "__notes__", []), msg]
+                        exc.__notes__ = [*getattr(exc, "__notes__", []), msg]
                         errors.append(exc)
                     finally:
                         ix += 1
@@ -699,13 +704,13 @@ def _structure_tuple(self, obj: Any, tup: Type[T]) -> T:
                     msg = IterableValidationNote(
                         f"Structuring {tup} @ index {ix}", ix, t
                     )
-                    exc.__notes__ = [*getattr(e, "__notes__", []), msg]
+                    exc.__notes__ = [*getattr(exc, "__notes__", []), msg]
                     errors.append(exc)
             if len(res) < exp_len:
                 problem = "Not enough" if len(res) < len(tup_params) else "Too many"
                 exc = ValueError(f"{problem} values in {obj!r} to structure as {tup!r}")
                 msg = f"Structuring {tup}"
-                exc.__notes__ = [*getattr(e, "__notes__", []), msg]
+                exc.__notes__ = [*getattr(exc, "__notes__", []), msg]
                 errors.append(exc)
             if errors:
                 raise IterableValidationError(f"While structuring {tup!r}", errors, tup)
@@ -748,6 +753,7 @@ def copy(
         prefer_attrib_converters: Optional[bool] = None,
         detailed_validation: Optional[bool] = None,
     ) -> "BaseConverter":
+        """Create a copy of the converter, keeping all existing custom hooks."""
         res = self.__class__(
             dict_factory if dict_factory is not None else self._dict_factory,
             unstruct_strat
@@ -765,8 +771,8 @@ def copy(
             else self.detailed_validation,
         )
 
-        self._unstructure_func.copy_to(res._unstructure_func)
-        self._structure_func.copy_to(res._structure_func)
+        self._unstructure_func.copy_to(res._unstructure_func, self._unstruct_copy_skip)
+        self._structure_func.copy_to(res._structure_func, self._struct_copy_skip)
 
         return res
 
@@ -779,8 +785,6 @@ class Converter(BaseConverter):
         "forbid_extra_keys",
         "type_overrides",
         "_unstruct_collection_overrides",
-        "_struct_copy_skip",
-        "_unstruct_copy_skip",
     )
 
     def __init__(
@@ -913,9 +917,11 @@ def gen_unstructure_annotated(self, type):
         origin = type.__origin__
         return self._unstructure_func.dispatch(origin)
 
-    def gen_structure_annotated(self, type):
+    def gen_structure_annotated(self, type) -> Callable:
+        """A hook factory for annotated types."""
         origin = type.__origin__
-        return self._structure_func.dispatch(origin)
+        hook = self._structure_func.dispatch(origin)
+        return lambda v, _: hook(v, origin)
 
     def gen_unstructure_typeddict(self, cl: Any) -> Callable[[Dict], Dict]:
         """Generate a TypedDict unstructure function.
@@ -1047,6 +1053,7 @@ def copy(
         prefer_attrib_converters: Optional[bool] = None,
         detailed_validation: Optional[bool] = None,
     ) -> "Converter":
+        """Create a copy of the converter, keeping all existing custom hooks."""
         res = self.__class__(
             dict_factory if dict_factory is not None else self._dict_factory,
             unstruct_strat

src/cattrs/dispatch.py

@@ -1,10 +1,10 @@
 from functools import lru_cache, singledispatch
 from typing import Any, Callable, List, Optional, Tuple, Union
 
-import attr
+from attrs import Factory, define
 
 
-@attr.s
+@define
 class _DispatchNotFound:
     """A dummy object to help signify a dispatch not found."""
 
@@ -95,9 +95,10 @@ def copy_to(self, other: "MultiStrategyDispatch", skip: int = 0):
         self._function_dispatch.copy_to(other._function_dispatch, skip=skip)
         for cls, fn in self._single_dispatch.registry.items():
             other._single_dispatch.register(cls, fn)
+        other.clear_cache()
 
 
-@attr.s(slots=True)
+@define
 class FunctionDispatch:
     """
     FunctionDispatch is similar to functools.singledispatch, but
@@ -109,14 +110,16 @@ class FunctionDispatch:
 
     _handler_pairs: List[
         Tuple[Callable[[Any], bool], Callable[[Any, Any], Any], bool]
-    ] = attr.ib(factory=list)
+    ] = Factory(list)
 
-    def register(self, can_handle: Callable[[Any], bool], func, is_generator=False):
+    def register(
+        self, can_handle: Callable[[Any], bool], func, is_generator=False
+    ) -> None:
         self._handler_pairs.insert(0, (can_handle, func, is_generator))
 
     def dispatch(self, typ: Any) -> Optional[Callable[[Any, Any], Any]]:
         """
-        returns the appropriate handler, for the object passed.
+        Return the appropriate handler for the object passed.
         """
         for can_handle, handler, is_generator in self._handler_pairs:
             # can handle could raise an exception here
@@ -136,5 +139,5 @@ def dispatch(self, typ: Any) -> Optional[Callable[[Any, Any], Any]]:
     def get_num_fns(self) -> int:
         return len(self._handler_pairs)
 
-    def copy_to(self, other: "FunctionDispatch", skip: int = 0):
-        other._handler_pairs.extend(self._handler_pairs[skip:])
+    def copy_to(self, other: "FunctionDispatch", skip: int = 0) -> None:
+        other._handler_pairs = self._handler_pairs[:-skip] + other._handler_pairs

src/cattrs/errors.py

@@ -40,6 +40,9 @@ def __new__(
         instance.type = type
         return instance
 
+    def __getnewargs__(self) -> Tuple[str, Union[int, str], Any]:
+        return (str(self), self.index, self.type)
+
 
 class IterableValidationError(BaseValidationError):
     """Raised when structuring an iterable."""
@@ -76,6 +79,9 @@ def __new__(cls, string: str, name: str, type: Any) -> "AttributeValidationNote"
         instance.type = type
         return instance
 
+    def __getnewargs__(self) -> Tuple[str, str, Any]:
+        return (str(self), self.name, self.type)
+
 
 class ClassValidationError(BaseValidationError):
     """Raised when validating a class if any attributes are invalid."""

src/cattrs/gen/__init__.py

@@ -30,6 +30,8 @@
 from ._shared import find_structure_handler
 
 if TYPE_CHECKING:  # pragma: no cover
+    from typing_extensions import Literal
+
     from cattr.converters import BaseConverter
 
 
@@ -233,10 +235,10 @@ def make_dict_unstructure_fn(
 def make_dict_structure_fn(
     cl: type[T],
     converter: BaseConverter,
-    _cattrs_forbid_extra_keys: bool = False,
+    _cattrs_forbid_extra_keys: bool | Literal["from_converter"] = "from_converter",
     _cattrs_use_linecache: bool = True,
     _cattrs_prefer_attrib_converters: bool = False,
-    _cattrs_detailed_validation: bool = True,
+    _cattrs_detailed_validation: bool | Literal["from_converter"] = "from_converter",
     _cattrs_use_alias: bool = False,
     _cattrs_include_init_false: bool = False,
     **kwargs: AttributeOverride,
@@ -245,13 +247,20 @@ def make_dict_structure_fn(
     Generate a specialized dict structuring function for an attrs class or
     dataclass.
 
+    :param _cattrs_forbid_extra_keys: Whether the structuring function should raise a
+        `ForbiddenExtraKeysError` if unknown keys are encountered.
+    :param _cattrs_detailed_validation: Whether to use a slower mode that produces
+        more detailed errors.
     :param _cattrs_use_alias: If true, the attribute alias will be used as the
         dictionary key by default.
     :param _cattrs_include_init_false: If true, _attrs_ fields marked as `init=False`
         will be included.
 
     ..  versionadded:: 23.2.0 *_cattrs_use_alias*
     ..  versionadded:: 23.2.0 *_cattrs_include_init_false*
+    ..  versionchanged:: 23.2.0
+        The `_cattrs_forbid_extra_keys` and `_cattrs_detailed_validation` parameters
+        take their values from the given converter by default.
     """
 
     mapping = {}
@@ -305,6 +314,12 @@ def make_dict_structure_fn(
         resolve_types(cl)
 
     allowed_fields = set()
+    if _cattrs_forbid_extra_keys == "from_converter":
+        # BaseConverter doesn't have it so we're careful.
+        _cattrs_forbid_extra_keys = getattr(converter, "forbid_extra_keys", False)
+    if _cattrs_detailed_validation == "from_converter":
+        _cattrs_detailed_validation = converter.detailed_validation
+
     if _cattrs_forbid_extra_keys:
         globs["__c_a"] = allowed_fields
         globs["__c_feke"] = ForbiddenExtraKeysError

src/cattrs/gen/typeddicts.py

@@ -5,7 +5,7 @@
 import sys
 from typing import TYPE_CHECKING, Any, Callable, TypeVar
 
-from attr import NOTHING, Attribute
+from attrs import NOTHING, Attribute
 
 try:
     from inspect import get_annotations
@@ -51,6 +51,8 @@ def get_annots(cl) -> dict[str, Any]:
 from ._shared import find_structure_handler
 
 if TYPE_CHECKING:  # pragma: no cover
+    from typing_extensions import Literal
+
     from cattr.converters import BaseConverter
 
 __all__ = ["make_dict_unstructure_fn", "make_dict_structure_fn"]
@@ -242,9 +244,9 @@ def make_dict_unstructure_fn(
 def make_dict_structure_fn(
     cl: Any,
     converter: BaseConverter,
-    _cattrs_forbid_extra_keys: bool = False,
+    _cattrs_forbid_extra_keys: bool | Literal["from_converter"] = "from_converter",
     _cattrs_use_linecache: bool = True,
-    _cattrs_detailed_validation: bool = True,
+    _cattrs_detailed_validation: bool | Literal["from_converter"] = "from_converter",
     **kwargs: AttributeOverride,
 ) -> Callable[[dict, Any], Any]:
     """Generate a specialized dict structuring function for typed dicts.
@@ -259,6 +261,10 @@ def make_dict_structure_fn(
         `ForbiddenExtraKeysError` if unknown keys are encountered.
     :param _cattrs_detailed_validation: Whether to store the generated code in the
         _linecache_, for easier debugging and better stack traces.
+
+    ..  versionchanged:: 23.2.0
+        The `_cattrs_forbid_extra_keys` and `_cattrs_detailed_validation` parameters
+        take their values from the given converter by default.
     """
 
     mapping = {}
@@ -307,6 +313,12 @@ def make_dict_structure_fn(
     req_keys = _required_keys(cl)
 
     allowed_fields = set()
+    if _cattrs_forbid_extra_keys == "from_converter":
+        # BaseConverter doesn't have it so we're careful.
+        _cattrs_forbid_extra_keys = getattr(converter, "forbid_extra_keys", False)
+    if _cattrs_detailed_validation == "from_converter":
+        _cattrs_detailed_validation = converter.detailed_validation
+
     if _cattrs_forbid_extra_keys:
         globs["__c_a"] = allowed_fields
         globs["__c_feke"] = ForbiddenExtraKeysError

src/cattrs/preconf/orjson.py

@@ -2,7 +2,7 @@
 from base64 import b85decode, b85encode
 from datetime import datetime
 from enum import Enum
-from typing import Any, Type, TypeVar
+from typing import Any, Type, TypeVar, Union
 
 from orjson import dumps, loads
 
@@ -17,7 +17,7 @@ class OrjsonConverter(Converter):
     def dumps(self, obj: Any, unstructure_as: Any = None, **kwargs: Any) -> bytes:
         return dumps(self.unstructure(obj, unstructure_as=unstructure_as), **kwargs)
 
-    def loads(self, data: bytes, cl: Type[T]) -> T:
+    def loads(self, data: Union[bytes, bytearray, memoryview, str], cl: Type[T]) -> T:
         return self.structure(loads(data), cl)
 
 

tests/test_converter.py

@@ -671,21 +671,22 @@ class Inner:
     class Outer:
         i: Annotated[Inner, "test"]
         j: List[Annotated[Inner, "test"]]
+        k: Annotated[Union[Inner, None], "test"]
 
-    orig = Outer(Inner(1), [Inner(1)])
+    orig = Outer(Inner(1), [Inner(1)], Inner(1))
     raw = converter.unstructure(orig)
 
-    assert raw == {"i": {"a": 1}, "j": [{"a": 1}]}
+    assert raw == {"i": {"a": 1}, "j": [{"a": 1}], "k": {"a": 1}}
 
     structured = converter.structure(raw, Outer)
     assert structured == orig
 
     # Now register a hook and rerun the test.
-    converter.register_unstructure_hook(Inner, lambda v: {"a": 2})
+    converter.register_unstructure_hook(Inner, lambda _: {"a": 2})
 
-    raw = converter.unstructure(Outer(Inner(1), [Inner(1)]))
+    raw = converter.unstructure(Outer(Inner(1), [Inner(1)], Inner(1)))
 
-    assert raw == {"i": {"a": 2}, "j": [{"a": 2}]}
+    assert raw == {"i": {"a": 2}, "j": [{"a": 2}], "k": {"a": 2}}
 
     structured = converter.structure(raw, Outer)
-    assert structured == Outer(Inner(2), [Inner(2)])
+    assert structured == Outer(Inner(2), [Inner(2)], Inner(2))