Fix annotation processing and rebuilding, mark dataclass as complex

dlt/common/data_types/type_helpers.py

@@ -1,5 +1,6 @@
 import binascii
 import base64
+import dataclasses
 import datetime  # noqa: I251
 from collections.abc import Mapping as C_Mapping, Sequence as C_Sequence
 from typing import Any, Type, Literal, Union, cast
@@ -55,7 +56,7 @@ def py_type_to_sc_type(t: Type[Any]) -> TDataType:
         return "bigint"
     if issubclass(t, bytes):
         return "binary"
-    if issubclass(t, (C_Mapping, C_Sequence)):
+    if dataclasses.is_dataclass(t) or issubclass(t, (C_Mapping, C_Sequence)):
         return "complex"
     # Enum is coerced to str or int respectively
     if issubclass(t, Enum):

dlt/common/destination/reference.py

@@ -419,7 +419,7 @@ def should_truncate_table_before_load_on_staging_destination(self, table: TTable
         return True
 
 
-TDestinationReferenceArg = Union[str, "Destination", Callable[..., "Destination"], None]
+TDestinationReferenceArg = Union[str, "Destination", Callable[..., "Destination"], None]  # type: ignore[type-arg]
 
 
 class Destination(ABC, Generic[TDestinationConfig, TDestinationClient]):

dlt/common/libs/pydantic.py

@@ -100,15 +100,16 @@ def pydantic_to_table_schema_columns(
             # This applies to pydantic.Json fields, the inner type is the type after json parsing
             # (In pydantic 2 the outer annotation is the final type)
             annotation = inner_annotation
+
         nullable = is_optional_type(annotation)
 
-        if is_union_type(annotation):
-            inner_type = get_args(annotation)[0]
-        else:
-            inner_type = extract_inner_type(annotation)
+        inner_type = extract_inner_type(annotation)
+        if is_union_type(inner_type):
+            first_argument_type = get_args(inner_type)[0]
+            inner_type = extract_inner_type(first_argument_type)
 
         if inner_type is Json:  # Same as `field: Json[Any]`
-            inner_type = Any
+            inner_type = Any  # type: ignore[assignment]
 
         if inner_type is Any:  # Any fields will be inferred from data
             continue
@@ -229,7 +230,7 @@ def _process_annotation(t_: Type[Any]) -> Type[Any]:
         """Recursively recreates models with applied schema contract"""
         if is_annotated(t_):
             a_t, *a_m = get_args(t_)
-            return Annotated[_process_annotation(a_t), a_m]  # type: ignore
+            return Annotated[_process_annotation(a_t), tuple(a_m)]  # type: ignore[return-value]
         elif is_list_generic_type(t_):
             l_t: Type[Any] = get_args(t_)[0]
             try:

dlt/common/typing.py

@@ -87,7 +87,7 @@
 TVariantBase = TypeVar("TVariantBase", covariant=True)
 TVariantRV = Tuple[str, Any]
 VARIANT_FIELD_FORMAT = "v_%s"
-TFileOrPath = Union[str, os.PathLike, IO[Any]]
+TFileOrPath = Union[str, os.PathLike, IO[Any]]  # type: ignore[type-arg]
 
 
 @runtime_checkable

tests/common/test_typing.py

@@ -15,6 +15,7 @@
 )
 from typing_extensions import Annotated, get_args
 
+
 from dlt.common.configuration.specs.base_configuration import (
     BaseConfiguration,
     get_config_if_union_hint,

tests/libs/test_pydantic.py

@@ -1,14 +1,18 @@
 import sys
 from copy import copy
+from dataclasses import dataclass, field
+import uuid
 import pytest
 from typing import (
     ClassVar,
     Final,
+    Generic,
     Sequence,
     Mapping,
     Dict,
     MutableMapping,
     MutableSequence,
+    TypeVar,
     Union,
     Optional,
     List,
@@ -29,7 +33,7 @@
     validate_items,
     create_list_model,
 )
-from pydantic import BaseModel, Json, AnyHttpUrl, ConfigDict, ValidationError
+from pydantic import UUID4, BaseModel, Json, AnyHttpUrl, ConfigDict, ValidationError
 
 from dlt.common.schema.exceptions import DataValidationError
 
@@ -118,6 +122,18 @@ class ModelWithConfig(Model):
 )
 
 
+class BookGenre(str, Enum):
+    scifi = "scifi"
+    action = "action"
+    thriller = "thriller"
+
+
+@dataclass
+class BookInfo:
+    isbn: Optional[str] = field(default="ISBN")
+    author: Optional[str] = field(default="Charles Bukowski")
+
+
 class UserLabel(BaseModel):
     label: str
 
@@ -134,12 +150,16 @@ class UserAddress(BaseModel):
 
 class User(BaseModel):
     user_id: int
+    account_id: UUID4
+    optional_uuid: Optional[UUID4]
     name: Annotated[str, "PII", "name"]
+    favorite_book: Annotated[Union[Annotated[BookInfo, "meta"], BookGenre, None], "union metadata"]
     created_at: Optional[datetime]
     labels: List[str]
     user_label: UserLabel
     user_labels: List[UserLabel]
     address: Annotated[UserAddress, "PII", "address"]
+    uuid_or_str: Union[str, UUID4, None]
     unity: Union[UserAddress, UserLabel, Dict[str, UserAddress]]
     location: Annotated[Optional[Union[str, List[str]]], None]
     something_required: Annotated[Union[str, int], type(None)]
@@ -151,6 +171,9 @@ class User(BaseModel):
 
 USER_INSTANCE_DATA = dict(
     user_id=1,
+    account_id=uuid.uuid4(),
+    optional_uuid=None,
+    favorite_book=BookInfo(isbn="isbn-xyz", author="author"),
     name="random name",
     created_at=datetime.now(),
     labels=["str"],
@@ -180,6 +203,7 @@ class User(BaseModel):
         ],
     ),
     unity=dict(label="123"),
+    uuid_or_str=uuid.uuid4(),
     location="Florida keys",
     final_location="Ginnie Springs",
     something_required=123,
@@ -387,14 +411,9 @@ class UserPipe(BaseModel):
     # and can generate the same schema from the class and from the class instance.
 
     user = UserPipe(**USER_INSTANCE_DATA)  # type: ignore
-    user_using_optional = User(**USER_INSTANCE_DATA)  # type: ignore
     schema_from_user_class = pydantic_to_table_schema_columns(UserPipe)
     schema_from_user_instance = pydantic_to_table_schema_columns(user)
-    schema_from_user_class_using_optional = pydantic_to_table_schema_columns(user_using_optional)
-    schema_from_user_instance_using_optional = pydantic_to_table_schema_columns(user_using_optional)
     assert schema_from_user_class == schema_from_user_instance
-    assert schema_from_user_class_using_optional == schema_from_user_instance_using_optional
-    assert schema_from_user_class == schema_from_user_class_using_optional
     assert schema_from_user_class["location"]["nullable"] is True
     assert schema_from_user_class["final_location"]["nullable"] is False
     assert schema_from_user_class["something_required"]["nullable"] is False

tests/libs/test_typing_extended.py

@@ -0,0 +1,46 @@
+from dataclasses import dataclass
+from typing import (
+    Final,
+    Literal,
+    TypedDict,
+    Optional,
+    Union,
+)
+from typing_extensions import Annotated
+from uuid import UUID
+
+from pydantic import UUID4
+
+from dlt.common.typing import (
+    extract_inner_type,
+)
+
+
+class TTestTyDi(TypedDict):
+    field: str
+
+
+@dataclass
+class MyDataclass:
+    booba_tooba: str
+
+
+TTestLi = Literal["a", "b", "c"]
+TOptionalLi = Optional[TTestLi]
+TOptionalTyDi = Optional[TTestTyDi]
+
+TOptionalUnionLiTyDi = Optional[Union[TTestTyDi, TTestLi]]
+
+
+def test_extract_annotated_inner_type() -> None:
+    assert extract_inner_type(Annotated[TOptionalLi, Optional]) is str  # type: ignore[arg-type]
+    assert extract_inner_type(Annotated[TOptionalLi, "random metadata string"]) is str  # type: ignore[arg-type]
+    assert extract_inner_type(Optional[Annotated[str, "random metadata string"]]) is str  # type: ignore[arg-type]
+    assert extract_inner_type(Final[Annotated[Optional[str], "annotated metadata"]]) is str  # type: ignore[arg-type]
+    assert extract_inner_type(Final[Annotated[Optional[str], None]]) is str  # type: ignore[arg-type]
+    assert extract_inner_type(Final[Annotated[Union[str, int], None]]) is Union[str, int]  # type: ignore[arg-type]
+    assert extract_inner_type(Annotated[Union[str, int], type(None)]) is Union[str, int]  # type: ignore[arg-type]
+    assert extract_inner_type(Annotated[Optional[UUID4], "meta"]) is UUID  # type: ignore[arg-type]
+    assert extract_inner_type(Annotated[Optional[MyDataclass], "meta"]) is MyDataclass  # type: ignore[arg-type]
+    assert extract_inner_type(Annotated[MyDataclass, Optional]) is MyDataclass  # type: ignore[arg-type]
+    assert extract_inner_type(Annotated[MyDataclass, "random metadata string"]) is MyDataclass  # type: ignore[arg-type]