Add support for Pydantic as configuration loader (#1432)

* Add support for Pydantic as configuration loader

Signed-off-by: Willem Pienaar <git@willem.co>

* Add sphinx sources back

Signed-off-by: Willem Pienaar <git@willem.co>

sdk/python/docs/source/feast.rst

@@ -63,6 +63,14 @@ feast.data\_source module
    :undoc-members:
    :show-inheritance:
 
+feast.driver\_test\_data module
+-------------------------------
+
+.. automodule:: feast.driver_test_data
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 feast.entity module
 -------------------
 
@@ -71,6 +79,14 @@ feast.entity module
    :undoc-members:
    :show-inheritance:
 
+feast.example\_repo module
+--------------------------
+
+.. automodule:: feast.example_repo
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 feast.feature module
 --------------------
 

sdk/python/feast/feature_store.py

@@ -68,7 +68,7 @@ def __init__(
                 project="default",
                 provider="local",
                 online_store=OnlineStoreConfig(
-                    local=LocalOnlineStoreConfig("online_store.db")
+                    local=LocalOnlineStoreConfig(path="online_store.db")
                 ),
             )
 

sdk/python/feast/repo_config.py

@@ -1,44 +1,55 @@
 from pathlib import Path
-from typing import NamedTuple, Optional
+from typing import Optional
 
 import yaml
-from bindr import bind
-from jsonschema import ValidationError, validate
+from pydantic import BaseModel, StrictStr, ValidationError
 
 
-class LocalOnlineStoreConfig(NamedTuple):
+class FeastBaseModel(BaseModel):
+    """ Feast Pydantic Configuration Class """
+
+    class Config:
+        arbitrary_types_allowed = True
+        extra = "forbid"
+
+
+class LocalOnlineStoreConfig(FeastBaseModel):
     """ Online store config for local (SQLite-based) online store """
 
-    path: str
+    path: StrictStr
     """ str: Path to sqlite db """
 
 
-class DatastoreOnlineStoreConfig(NamedTuple):
+class DatastoreOnlineStoreConfig(FeastBaseModel):
     """ Online store config for GCP Datastore """
 
-    project_id: str
+    project_id: StrictStr
     """ str: GCP Project Id """
 
 
-class OnlineStoreConfig(NamedTuple):
+class OnlineStoreConfig(FeastBaseModel):
     datastore: Optional[DatastoreOnlineStoreConfig] = None
     """ DatastoreOnlineStoreConfig: Optional DatastoreConfig """
+
     local: Optional[LocalOnlineStoreConfig] = None
     """ LocalOnlineStoreConfig: Optional local online store config """
 
 
-class RepoConfig(NamedTuple):
+class RepoConfig(FeastBaseModel):
     """ Repo config. Typically loaded from `feature_store.yaml` """
 
-    metadata_store: str
+    metadata_store: StrictStr
     """ str: Path to metadata store. Can be a local path, or remote object storage path, e.g. gcs://foo/bar """
-    project: str
+
+    project: StrictStr
     """ str: Feast project id. This can be any alphanumeric string up to 16 characters.
         You can have multiple independent feature repositories deployed to the same cloud
         provider account, as long as they have different project ids.
     """
-    provider: str
+
+    provider: StrictStr
     """ str: local or gcp """
+
     online_store: Optional[OnlineStoreConfig] = None
     """ OnlineStoreConfig: Online store configuration (optional depending on provider) """
 
@@ -79,20 +90,18 @@ class RepoConfig(NamedTuple):
 
 
 class FeastConfigError(Exception):
-    def __init__(self, error_message, error_path, config_path):
+    def __init__(self, error_message, config_path):
         self._error_message = error_message
-        self._error_path = error_path
         self._config_path = config_path
         super().__init__(self._error_message)
 
     def __str__(self) -> str:
-        if self._error_path:
-            return f'{self._error_message} under {"->".join(self._error_path)} in {self._config_path}'
-        else:
-            return f"{self._error_message} in {self._config_path}"
+        return f"{self._error_message}\nat {self._config_path}"
 
     def __repr__(self) -> str:
-        return f"FeastConfigError({repr(self._error_message)}, {repr(self._error_path)}, {repr(self._config_path)})"
+        return (
+            f"FeastConfigError({repr(self._error_message)}, {repr(self._config_path)})"
+        )
 
 
 def load_repo_config(repo_path: Path) -> RepoConfig:
@@ -101,7 +110,6 @@ def load_repo_config(repo_path: Path) -> RepoConfig:
     with open(config_path) as f:
         raw_config = yaml.safe_load(f)
         try:
-            validate(raw_config, config_schema)
-            return bind(RepoConfig, raw_config)
+            return RepoConfig(**raw_config)
         except ValidationError as e:
-            raise FeastConfigError(e.message, e.absolute_path, config_path)
+            raise FeastConfigError(e, config_path)

sdk/python/setup.py

@@ -39,7 +39,6 @@
 
 REQUIRED = [
     "Click==7.*",
-    "bindr",
     "fastavro>=0.22.11,<0.23",
     "google-api-core>=1.23.0",
     "google-cloud-bigquery>=2.0.*",
@@ -56,6 +55,7 @@
     "pandavro==1.5.*",
     "protobuf>=3.10",
     "pyarrow==2.0.0",
+    "pydantic>=1.0.0",
     "PyYAML==5.3.*",
     "tabulate==0.8.*",
     "toml==0.10.*",

sdk/python/tests/test_historical_retrieval.py

@@ -234,7 +234,7 @@ def test_historical_features_from_parquet_sources():
                 provider="local",
                 online_store=OnlineStoreConfig(
                     local=LocalOnlineStoreConfig(
-                        os.path.join(temp_dir, "online_store.db"),
+                        path=os.path.join(temp_dir, "online_store.db")
                     )
                 ),
             )

sdk/python/tests/test_repo_config.py

@@ -1,4 +1,3 @@
-import re
 import tempfile
 from pathlib import Path
 from textwrap import dedent
@@ -26,7 +25,7 @@ def _test_config(self, config_text, expect_error: Optional[str]):
                 error = e
 
             if expect_error is not None:
-                assert re.search(expect_error, str(error)) is not None
+                assert expect_error in str(error)
             else:
                 assert error is None
 
@@ -69,7 +68,8 @@ def test_errors(self) -> None:
                     path: "online_store.db"
             """
             ),
-            expect_error=r"'that_field_should_not_be_here' was unexpected.*online_store->local",
+            expect_error="online_store -> local -> that_field_should_not_be_here\n"
+            "  extra fields not permitted (type=value_error.extra)",
         )
 
         self._test_config(
@@ -83,7 +83,9 @@ def test_errors(self) -> None:
                     path: 100500
             """
             ),
-            expect_error=r"100500 is not of type 'string'",
+            expect_error="1 validation error for RepoConfig\n"
+            "online_store -> local -> path\n"
+            "  str type expected (type=type_error.str)",
         )
 
         self._test_config(
@@ -96,5 +98,7 @@ def test_errors(self) -> None:
                     path: foo
             """
             ),
-            expect_error=r"'project' is a required property",
+            expect_error="1 validation error for RepoConfig\n"
+            "project\n"
+            "  field required (type=value_error.missing)",
         )