Merge pull request #22 from lsst-sqre/tickets/DM-38801

DM-38801: New PydanticSchemaManager

.github/workflows/ci.yaml

@@ -25,7 +25,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
       - name: Run pre-commit
         uses: pre-commit/action@v3.0.0
@@ -66,7 +66,7 @@ jobs:
       - name: Run tox
         uses: lsst-sqre/run-tox@v1
         with:
-          python-version: "3.10"
+          python-version: "3.11"
           tox-envs: "docs,docs-linkcheck"
 
       # Only attempt documentation uploads for tagged releases and pull
@@ -98,5 +98,5 @@ jobs:
         uses: lsst-sqre/build-and-publish-to-pypi@v1
         with:
           pypi-token: ${{ secrets.PYPI_SQRE_ADMIN }}
-          python-version: "3.10"
+          python-version: "3.11"
           upload: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }}

.github/workflows/periodic-ci.yaml

@@ -39,7 +39,7 @@ jobs:
       - name: Build docs in tox
         uses: lsst-sqre/run-tox@v1
         with:
-          python-version: "3.10"
+          python-version: "3.11"
           tox-envs: "docs,docs-linkcheck"
           use-cache: false
 
@@ -55,5 +55,5 @@ jobs:
         uses: lsst-sqre/build-and-publish-to-pypi@v1
         with:
           pypi-token: ""
-          python-version: "3.10"
+          python-version: "3.11"
           upload: false

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Change log
 
+## Unreleased
+
+### New features
+
+- New `PydanticSchemaManager` enables you to use [Pydantic](https://pydantic-docs.helpmanual.io/) models as Avro schemas. Like the `RecordNameSchemaManager`, this new manager handles schema registration for you. To serialize a message, you simply supply a Pydantic object. The manager will also deserialize messages into Pydantic objects if the message's schema corresponds to a managed Pydantic model. Overall this feature provides end-to-end type checking of messages from development to production.
+
 ## 0.3.0 (2023-02-23)
 
 New features:

Makefile

@@ -5,7 +5,7 @@ help:
 
 .PHONY: init
 init:
-	pip install -e ".[aiohttp,httpx,dev]"
-	pip install tox pre-commit
+	pip install -e ".[aiohttp,httpx,pydantic,dev]"
+	pip install -U tox pre-commit
 	pre-commit install
 	rm -rf .tox

docs/dev/development.rst

@@ -43,9 +43,6 @@ Pre-commit hooks
 The pre-commit hooks, which are automatically installed by running the :command:`make init` command on :ref:`set up <dev-environment>`, ensure that files are valid and properly formatted.
 Some pre-commit hooks automatically reformat code:
 
-``seed-isort-config``
-    Adds configuration for isort to the :file:`pyproject.toml` file.
-
 ``isort``
     Automatically sorts imports in Python modules.
 
@@ -73,13 +70,13 @@ You can also run tox_, which tests the library the same way that the CI workflow
 
 .. code-block:: sh
 
-   tox
+   tox run
 
 To see a listing of test environments, run:
 
 .. code-block:: sh
 
-   tox -av
+   tox list
 
 .. _dev-build-docs:
 
@@ -92,7 +89,7 @@ Documentation is built with Sphinx_:
 
 .. code-block:: sh
 
-   tox -e docs
+   tox run -e docs
 
 The build documentation is located in the :file:`docs/_build/html` directory.
 
@@ -101,31 +98,28 @@ The build documentation is located in the :file:`docs/_build/html` directory.
 Updating the change log
 =======================
 
-Each pull request should update the change log (:file:`CHANGELOG.rst`).
+Each pull request should update the change log (:file:`CHANGELOG.md`).
 Add a description of new features and fixes as list items under a section at the top of the change log called "Unreleased:"
 
-.. code-block:: rst
+.. code-block:: md
 
-   Unreleased
-   ----------
+   ## Unreleased
 
    - Description of the feature or fix.
 
-If the next version is known (because Kafkit's master branch is being prepared for a new major or minor version), the section may contain that version information:
+If the next version is known (because Kafkit's main branch is being prepared for a new major or minor version), the section may contain that version information:
 
-.. code-block:: rst
+.. code-block:: md
 
-   X.Y.0 (unreleased)
-   ------------------
+   ## X.Y.0 (unreleased)
 
    - Description of the feature or fix.
 
 If the exact version and release date is known (:doc:`because a release is being prepared <release>`), the section header is formatted as:
 
 .. code-block:: rst
 
-   X.Y.0 (YYYY-MM-DD)
-   ------------------
+   ## X.Y.0 (YYYY-MM-DD)
 
    - Description of the feature or fix.
 

docs/dev/release.rst

@@ -10,15 +10,15 @@ When a semantic version tag is pushed to GitHub, `Kafkit is released to PyPI`_ w
 Similarly, documentation is built and pushed for each version (see https://kafkit.lsst.io/v).
 
 .. _`Kafkit is released to PyPI`: https://pypi.org/project/kafkit/
-.. _`ci.yaml`: https://github.com/lsst-sqre/kafkit/blob/master/.github/workflows/ci.yaml
+.. _`ci.yaml`: https://github.com/lsst-sqre/kafkit/blob/main/.github/workflows/ci.yaml
 
 .. _regular-release:
 
 Regular releases
 ================
 
-Regular releases happen from the ``master`` branch after changes have been merged.
-From the ``master`` branch you can release a new major version (``X.0.0``), a new minor version of the current major version (``X.Y.0``), or a new patch of the current major-minor version (``X.Y.Z``).
+Regular releases happen from the ``main`` branch after changes have been merged.
+From the ``main`` branch you can release a new major version (``X.0.0``), a new minor version of the current major version (``X.Y.0``), or a new patch of the current major-minor version (``X.Y.Z``).
 See :ref:`backport-release` to patch an earlier major-minor version.
 
 Release tags are semantic version identifiers following the :pep:`440` specification.
@@ -27,15 +27,15 @@ Release tags are semantic version identifiers following the :pep:`440` specifica
 -------------------------------
 
 Each PR should include updates to the change log.
-If the change log or documentation needs additional updates, now is the time to make those changes through the regular branch-and-PR development method against the ``master`` branch.
+If the change log or documentation needs additional updates, now is the time to make those changes through the regular branch-and-PR development method against the ``main`` branch.
 
 In particular, replace the "Unreleased" section headline with the semantic version and date.
 See :ref:`dev-change-log` in the *Developer guide* for details.
 
 2. Tag the release
 ------------------
 
-At the HEAD of the ``master`` branch, create and push a tag with the semantic version:
+At the HEAD of the ``main`` branch, create and push a tag with the semantic version:
 
 .. code-block:: sh
 
@@ -54,7 +54,7 @@ The `ci.yaml`_ GitHub Actions workflow uploads the new release to PyPI and docum
 Backport releases
 =================
 
-The regular release procedure works from the main line of development on the ``master`` Git branch.
+The regular release procedure works from the main line of development on the ``main`` Git branch.
 To create a release that patches an earlier major or minor version, you need to release from a **release branch.**
 
 Creating a release branch
@@ -72,12 +72,12 @@ If the release branch doesn't already exist, check out the latest patch for that
 Developing on a release branch
 ------------------------------
 
-Once a release branch exists, it becomes the "master" branch for patches of that major-minor version.
+Once a release branch exists, it becomes the "main" branch for patches of that major-minor version.
 Pull requests should be based on, and merged into, the release branch.
 
-If the development on the release branch is a backport of commits on the ``master`` branch, use ``git cherry-pick`` to copy those commits into a new pull request against the release branch.
+If the development on the release branch is a backport of commits on the ``main`` branch, use ``git cherry-pick`` to copy those commits into a new pull request against the release branch.
 
 Releasing from a release branch
 -------------------------------
 
-Releases from a release branch are equivalent to :ref:`regular releases <regular-release>`, except that the release branch takes the role of the ``master`` branch.
+Releases from a release branch are equivalent to :ref:`regular releases <regular-release>`, except that the release branch takes the role of the ``main`` branch.

docs/documenteer.toml

@@ -16,6 +16,14 @@ nitpick_ignore = [
         "py:class",
         "httpx.AsyncClient",
     ],
+    [
+        "py:class",
+        "AvroBaseModel",
+    ],
+    [
+        "py:class",
+        "dataclasses_avroschema.avrodantic.AvroBaseModel",
+    ],
 ]
 
 [sphinx.intersphinx.projects]

pyproject.toml

@@ -17,18 +17,20 @@ classifiers = [
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
     "Intended Audience :: Developers",
     "Natural Language :: English",
     "Operating System :: POSIX",
     "Typing :: Typed",
 ]
 requires-python = ">=3.8"
-dependencies = ["uritemplate", "fastavro"]
+dependencies = ["fastavro", "uritemplate"]
 dynamic = ["version"]
 
 [project.optional-dependencies]
 aiohttp = ["aiohttp"]
 httpx = ["httpx"]
+pydantic = ["dataclasses-avroschema[pydantic]"]
 dev = [
     # Testing
     "coverage[toml]",

src/kafkit/registry/__init__.py

@@ -8,16 +8,24 @@
     RegistryError,
     RegistryHttpError,
     RegistryRedirectionError,
+    UnmanagedSchemaError,
+)
+from kafkit.registry.serializer import (
+    Deserializer,
+    MessageInfo,
+    PolySerializer,
+    Serializer,
 )
-from kafkit.registry.serializer import Deserializer, PolySerializer, Serializer
 
 __all__ = [
     "Deserializer",
+    "MessageInfo",
     "Serializer",
     "PolySerializer",
     "RegistryBadRequestError",
     "RegistryBrokenError",
     "RegistryError",
     "RegistryHttpError",
     "RegistryRedirectionError",
+    "UnmanagedSchemaError",
 ]

src/kafkit/registry/errors.py

@@ -6,6 +6,7 @@
     "RegistryRedirectionError",
     "RegistryBadRequestError",
     "RegistryBrokenError",
+    "UnmanagedSchemaError",
 ]
 
 from typing import Any, Optional
@@ -61,3 +62,9 @@ class RegistryBadRequestError(RegistryHttpError):
 
 class RegistryBrokenError(RegistryHttpError):
     """An excpetion if the server is down (5XX errors)."""
+
+
+class UnmanagedSchemaError(Exception):
+    """An exception for when a schema is not managed by the Registry, and
+    therefore cannot be deserialized into a native Python object.
+    """

src/kafkit/registry/manager/__init__.py

@@ -0,0 +1,8 @@
+"""Schema managers register schemas with the registry and enable conventient
+serialization and deserialization of messages.
+"""
+
+from ._pydantic import PydanticSchemaManager
+from ._recordname import RecordNameSchemaManager
+
+__all__ = ["RecordNameSchemaManager", "PydanticSchemaManager"]