10x speedup over V1 - use TypedDict instead of BaseModel

.gitignore

@@ -158,4 +158,6 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
-.DS_Store
+.DS_Store
+
+/env*/

v2/schemas.py

@@ -1,46 +1,52 @@
-from pydantic import BaseModel, model_validator
+from pydantic import TypeAdapter, BeforeValidator, field_validator, constr
+from pydantic_core import PydanticOmit
+from typing_extensions import TypedDict, NotRequired, Annotated
 
 
-class Wine(BaseModel):
+def exclude_none(s: str | None) -> str:
+    if s is None:
+        # since we want `exclude_none=True` in the end,
+        # just omit it None during validation
+        raise PydanticOmit
+    else:
+        return s
+
+
+ExcludeNoneStr = Annotated[str, BeforeValidator(exclude_none)]
+
+
+class Wine(TypedDict):
     id: int
     points: int
     title: str
-    description: str | None
-    price: float | None
-    variety: str | None
-    winery: str | None
-    designation: str | None
-    country: str | None
-    province: str | None
-    region_1: str | None
-    region_2: str | None
-    taster_name: str | None
-    taster_twitter_handle: str | None
+    description: NotRequired[ExcludeNoneStr]
+    price: NotRequired[Annotated[float, BeforeValidator(exclude_none)]]
+    variety: NotRequired[ExcludeNoneStr]
+    winery: NotRequired[ExcludeNoneStr]
+    designation: NotRequired[constr(strip_whitespace=True)]
+    country: NotRequired[str]
+    province: NotRequired[str]
+    region_1: NotRequired[str]
+    region_2: NotRequired[str]
+    taster_name: NotRequired[ExcludeNoneStr]
+    taster_twitter_handle: NotRequired[ExcludeNoneStr]
+
+    @field_validator("designation", "province", "region_1", "region_2", mode="before")
+    def omit_null_none(cls, v):
+        if v is None or v == "null":
+            raise PydanticOmit
+        else:
+            return v
 
-    @model_validator(mode="before")
-    def _remove_unknowns(cls, values):
-        "Set other fields that have the value 'null' as None so that we can throw it away"
-        fields = ["designation", "province", "region_1", "region_2"]
-        for field in fields:
-            if not values.get(field) or values.get(field) == "null":
-                values[field] = None
-        return values
+    @field_validator("country", mode="before")
+    def country_unknown(cls, s: str | None) -> str:
+        if s is None or s == "null":
+            return "Unknown"
+        else:
+            return s
 
-    @model_validator(mode="before")
-    def _fill_country_unknowns(cls, values):
-        "Fill in missing country values with 'Unknown', as we always want this field to be queryable"
-        country = values.get("country")
-        if not country or country == "null":
-            values["country"] = "Unknown"
-        return values
 
-    @model_validator(mode="before")
-    def _get_vineyard(cls, values):
-        "Rename designation to vineyard"
-        vineyard = values.pop("designation", None)
-        if vineyard:
-            values["vineyard"] = vineyard.strip()
-        return values
+WinesTypeAdapter = TypeAdapter(list[Wine])
 
 
 if __name__ == "__main__":
@@ -58,7 +64,8 @@ def _get_vineyard(cls, values):
         "region_2": "null",
         "taster_name": "Michael Schachner",
         "taster_twitter_handle": "@wineschach",
+        "designation": "  The Vineyard  ",
     }
-    wine = Wine(**sample_data)
+    wines = WinesTypeAdapter.validate_python([sample_data])
     from pprint import pprint
-    pprint(wine.model_dump(exclude_none=True))
+    pprint(wines, sort_dicts=False)

v2/validator.py

@@ -4,16 +4,12 @@
 import srsly
 from codetiming import Timer
 
-from schemas import Wine
+from schemas import WinesTypeAdapter, Wine
 
 # Custom types
 JsonBlob = dict[str, Any]
 
 
-class FileNotFoundError(Exception):
-    pass
-
-
 def get_json_data(data_dir: Path, filename: str) -> list[JsonBlob]:
     """Get all line-delimited files from a directory with a given prefix"""
     file_path = data_dir / filename
@@ -30,17 +26,15 @@ def get_json_data(data_dir: Path, filename: str) -> list[JsonBlob]:
 
 def validate(
     data: list[JsonBlob],
-    exclude_none: bool = False,
 ) -> list[JsonBlob]:
     """Validate a list of JSON blobs against the Wine schema"""
-    validated_data = [Wine(**item).model_dump(exclude_none=exclude_none) for item in data]
-    return validated_data
+    return WinesTypeAdapter.validate_python(data)
 
 
 def run():
     """Wrapper function to time the validator over many runs"""
     with Timer(name="Single case", text="{name}: {seconds:.3f} sec"):
-        validated_data = validate(data, exclude_none=True)
+        validated_data = validate(data)
         print(f"Validated {len(validated_data)} records in cycle {i + 1} of {num}")
 
 
@@ -52,4 +46,4 @@ def run():
     num = 10
     with Timer(name="All cases", text="{name}: {seconds:.3f} sec"):
         for i in range(num):
-            run()
+            run()