[FEATURE] ValidationConfigStore (#9523)

Co-authored-by: Bill Dirks <bill@greatexpectations.io>

docs/docusaurus/docs/oss/guides/connecting_to_your_data/fluent/database/gcp_deployment_patterns_file_bigquery.py

@@ -40,6 +40,7 @@
     "evaluation_parameter_store",
     "validations_store",
     "profiler_store",
+    "validation_config_store",
 ]
 for store in pop_stores:
     stores.pop(store)
@@ -124,6 +125,7 @@
     "expectations_store",
     "expectations_GCS_store",
     "profiler_store",
+    "validation_config_store",
 ]
 for store in pop_stores:
     stores.pop(store)

docs/docusaurus/docs/oss/guides/connecting_to_your_data/fluent/database/gcp_deployment_patterns_file_gcs.py

@@ -41,6 +41,7 @@
     "evaluation_parameter_store",
     "validations_store",
     "profiler_store",
+    "validation_config_store",
 ]
 for store in pop_stores:
     stores.pop(store)
@@ -126,6 +127,7 @@
     "expectations_store",
     "expectations_GCS_store",
     "profiler_store",
+    "validation_config_store",
 ]
 for store in pop_stores:
     stores.pop(store)

docs/docusaurus/docs/snippets/aws_cloud_storage_pandas.py

@@ -31,6 +31,7 @@
     "evaluation_parameter_store",
     "validations_store",
     "profiler_store",
+    "validation_config_store",
 ]
 for store in pop_stores:
     stores.pop(store)
@@ -110,6 +111,7 @@
     "expectations_store",
     "expectations_S3_store",
     "profiler_store",
+    "validation_config_store",
 ]
 for store in pop_stores:
     stores.pop(store)

docs/docusaurus/docs/snippets/aws_redshift_deployment_patterns.py

@@ -32,6 +32,7 @@
     "evaluation_parameter_store",
     "validations_store",
     "profiler_store",
+    "validation_config_store",
 ]
 for store in pop_stores:
     stores.pop(store)
@@ -111,6 +112,7 @@
     "expectations_store",
     "expectations_S3_store",
     "profiler_store",
+    "validation_config_store",
 ]
 for store in pop_stores:
     stores.pop(store)

great_expectations/core/expectation_suite.py

@@ -1207,4 +1207,4 @@ def _convert_uuids_to_str(self, data, **kwargs):
         return data
 
 
-expectationSuiteSchema = ExpectationSuiteSchema()
+expectationSuiteSchema: ExpectationSuiteSchema = ExpectationSuiteSchema()

great_expectations/core/factory/validation_factory.py

@@ -1,15 +1,21 @@
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
+
 from great_expectations._docs_decorators import public_api
 from great_expectations.compatibility.typing_extensions import override
 from great_expectations.core.factory.factory import Factory
 from great_expectations.core.validation_config import ValidationConfig
 
+if TYPE_CHECKING:
+    from great_expectations.data_context.store.validation_config_store import (
+        ValidationConfigStore,
+    )
+
 
 # TODO: Add analytics as needed
 class ValidationFactory(Factory[ValidationConfig]):
-    def __init__(self, store) -> None:
-        # TODO: Update type hints when new ValidationConfigStore is implemented
+    def __init__(self, store: ValidationConfigStore) -> None:
         self._store = store
 
     @public_api

great_expectations/core/validation_config.py

@@ -1,15 +1,14 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import Union
 
 from great_expectations._docs_decorators import public_api
-from great_expectations.compatibility.pydantic import BaseModel
-
-if TYPE_CHECKING:
-    from great_expectations.core.batch_config import BatchConfig
-    from great_expectations.core.expectation_suite import ExpectationSuite
-
-    # from great_expectations.datasource.fluent.interfaces import DataAsset
+from great_expectations.compatibility.pydantic import BaseModel, validator
+from great_expectations.core.batch_config import BatchConfig  # noqa: TCH001
+from great_expectations.core.expectation_suite import (
+    ExpectationSuite,
+    expectationSuiteSchema,
+)
 
 
 class ValidationConfig(BaseModel):
@@ -23,9 +22,26 @@ class ValidationConfig(BaseModel):
 
     """
 
+    class Config:
+        arbitrary_types_allowed = True
+        json_encoders = {
+            ExpectationSuite: lambda v: expectationSuiteSchema.dump(v),
+        }
+
     name: str
     data: BatchConfig  # TODO: Should support a union of Asset | BatchConfig
     suite: ExpectationSuite
+    id: Union[str, None] = None
+
+    @validator("suite", pre=True)
+    def _validate_suite(cls, v):
+        if isinstance(v, dict):
+            return ExpectationSuite(**expectationSuiteSchema.load(v))
+        elif isinstance(v, ExpectationSuite):
+            return v
+        raise ValueError(
+            "Suite must be a dictionary (if being deserialized) or an ExpectationSuite object."
+        )
 
     @public_api
     def run(self):

great_expectations/data_context/cloud_constants.py

@@ -31,3 +31,4 @@ class GXCloudRESTResource(str, Enum):
     PROFILER = "profiler"
     RENDERED_DATA_DOC = "rendered_data_doc"
     VALIDATION_RESULT = "validation_result"
+    VALIDATION_CONFIG = "validation_config"

great_expectations/data_context/data_context/abstract_data_context.py

@@ -148,6 +148,9 @@
         DataDocsSiteConfigTypedDict,
         StoreConfigTypedDict,
     )
+    from great_expectations.data_context.store.validation_config_store import (
+        ValidationConfigStore,
+    )
     from great_expectations.data_context.store.validations_store import ValidationsStore
     from great_expectations.data_context.types.resource_identifiers import (
         GXCloudIdentifier,
@@ -331,8 +334,9 @@ def _init_factories(self) -> None:
                 context=self,
             )
 
-        # TODO: Update to follow existing pattern once new ValidationConfigStore is implemented
-        self._validations: ValidationFactory | None = None
+        self._validations: ValidationFactory = ValidationFactory(
+            store=self.validation_config_store
+        )
 
     def _init_analytics(self) -> None:
         init_analytics(
@@ -617,6 +621,13 @@ def validations_store_name(self, value: str) -> None:
     def validations_store(self) -> ValidationsStore:
         return self.stores[self.validations_store_name]
 
+    @property
+    def validation_config_store(self) -> ValidationConfigStore:
+        # Purposely not exposing validation_config_store_name as a user-configurable property
+        return self.stores[
+            DataContextConfigDefaults.DEFAULT_VALIDATION_CONFIG_STORE_NAME.value
+        ]
+
     @property
     def checkpoint_store_name(self) -> Optional[str]:
         from great_expectations.data_context.store.checkpoint_store import (

great_expectations/data_context/data_context/serializable_data_context.py

@@ -48,6 +48,7 @@ class SerializableDataContext(AbstractDataContext):
         DataContextConfigDefaults.EXPECTATIONS_BASE_DIRECTORY.value,
         DataContextConfigDefaults.PLUGINS_BASE_DIRECTORY.value,
         DataContextConfigDefaults.PROFILERS_BASE_DIRECTORY.value,
+        DataContextConfigDefaults.VALIDATION_CONFIGS_BASE_DIRECTORY.value,
         GX_UNCOMMITTED_DIR,
     ]
     GX_DIR: ClassVar[str] = "gx"

great_expectations/data_context/store/__init__.py

@@ -30,3 +30,4 @@
 from .profiler_store import ProfilerStore  # isort:skip
 from .data_context_store import DataContextStore  # isort:skip
 from .data_asset_store import DataAssetStore  # isort:skip
+from .validation_config_store import ValidationConfigStore  # isort:skip

great_expectations/data_context/store/gx_cloud_store_backend.py

@@ -133,6 +133,7 @@ class GXCloudStoreBackend(StoreBackend, metaclass=ABCMeta):
         GXCloudRESTResource.PROFILER: "profiler",
         GXCloudRESTResource.RENDERED_DATA_DOC: "rendered_data_doc",
         GXCloudRESTResource.VALIDATION_RESULT: "result",
+        GXCloudRESTResource.VALIDATION_CONFIG: "validation_config",
     }
 
     ALLOWED_SET_KWARGS_BY_RESOURCE_TYPE: Dict[GXCloudRESTResource, Set[str]] = {
@@ -157,6 +158,7 @@ class GXCloudStoreBackend(StoreBackend, metaclass=ABCMeta):
             GXCloudRESTResource.PROFILER: "profilers",
             GXCloudRESTResource.RENDERED_DATA_DOC: "rendered_data_docs",
             GXCloudRESTResource.VALIDATION_RESULT: "validation_results",
+            GXCloudRESTResource.VALIDATION_CONFIG: "validation_configs",
         }
     )
 

great_expectations/data_context/store/validation_config_store.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from great_expectations.compatibility.typing_extensions import override
+from great_expectations.core.data_context_key import StringKey
+from great_expectations.core.validation_config import ValidationConfig
+from great_expectations.data_context.cloud_constants import GXCloudRESTResource
+from great_expectations.data_context.store.store import Store
+from great_expectations.data_context.types.resource_identifiers import (
+    GXCloudIdentifier,
+)
+
+
+class ValidationConfigStore(Store):
+    _key_class = StringKey
+
+    def get_key(
+        self, name: str, id: str | None = None
+    ) -> GXCloudIdentifier | StringKey:
+        """Given a name and optional ID, build the correct key for use in the ValidationConfigStore."""
+        if self.cloud_mode:
+            return GXCloudIdentifier(
+                resource_type=GXCloudRESTResource.VALIDATION_CONFIG,
+                id=id,
+                resource_name=name,
+            )
+        return StringKey(key=name)
+
+    @override
+    def serialize(self, value):
+        if self.cloud_mode:
+            data = value.dict()
+            data["suite"] = data["suite"].to_json_dict()
+            return data
+
+        return value.json()
+
+    @override
+    def deserialize(self, value):
+        return ValidationConfig.parse_raw(value)

great_expectations/data_context/templates.py

@@ -104,6 +104,13 @@ def dump(self, data, stream=None, **kw):
 PROFILER_STORE_STRING = yaml.dump(
     {"profiler_store": DataContextConfigDefaults.DEFAULT_STORES.value["profiler_store"]}
 ).replace("\n", "\n  ")[:-2]
+VALIDATION_CONFIG_STORE_STRING = yaml.dump(
+    {
+        "validation_config_store": DataContextConfigDefaults.DEFAULT_STORES.value[
+            "validation_config_store"
+        ]
+    }
+).replace("\n", "\n  ")[:-2]
 
 PROJECT_OPTIONAL_CONFIG_COMMENT = (
     CONFIG_VARIABLES_INTRO
@@ -130,6 +137,7 @@ def dump(self, data, stream=None, **kw):
     {EVALUATION_PARAMETER_STORE_STRING}
   {CHECKPOINT_STORE_STRING}
   {PROFILER_STORE_STRING}
+  {VALIDATION_CONFIG_STORE_STRING}
 expectations_store_name: expectations_store
 validations_store_name: validations_store
 evaluation_parameter_store_name: evaluation_parameter_store