Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[FEATURE] ValidationConfig.save() #9579

Merged
merged 6 commits into from
Mar 7, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions great_expectations/core/validation_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from great_expectations._docs_decorators import public_api
from great_expectations.compatibility.pydantic import (
BaseModel,
PrivateAttr,
ValidationError,
validator,
)
Expand All @@ -14,6 +15,9 @@
ExpectationSuite,
expectationSuiteSchema,
)
from great_expectations.data_context.store.validation_config_store import (
ValidationConfigStore, # noqa: TCH001
)
from great_expectations.validator.v1_validator import ResultFormat, Validator

if TYPE_CHECKING:
Expand Down Expand Up @@ -121,6 +125,13 @@ class Config:
suite: ExpectationSuite
id: Union[str, None] = None

# private attributes that must be set immediately after instantiation
_store: ValidationConfigStore = PrivateAttr()

def __init__(self, **data: Any):
super().__init__(**data)
self._store = project_manager.get_validation_config_store()

@validator("suite", pre=True)
def _validate_suite(cls, v: dict | ExpectationSuite):
# Input will be a dict of identifiers if being deserialized or a suite object if being constructed by a user.
Expand Down Expand Up @@ -222,3 +233,12 @@ def run(
result_format=result_format,
)
return validator.validate_expectation_suite(self.suite, evaluation_parameters)

@public_api
def save(self) -> None:
key = self._store.get_key(name=self.name, id=self.id)

try:
self._store.update(key=key, value=self)
except gx_exceptions.StoreBackendError:
raise ValueError("ValidationConfig must be added to a store before saving.")
11 changes: 11 additions & 0 deletions great_expectations/data_context/data_context/context_factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@
ExpectationsStore,
ValidationsStore,
)
from great_expectations.data_context.store.validation_config_store import (
ValidationConfigStore,
)
from great_expectations.data_context.types.base import DataContextConfig
from great_expectations.datasource.datasource_dict import DatasourceDict
from great_expectations.datasource.fluent.batch_request import BatchRequest
Expand Down Expand Up @@ -105,6 +108,14 @@ def get_validations_store(self) -> ValidationsStore:
)
return self._project.validations_store

def get_validation_config_store(self) -> ValidationConfigStore:
if not self._project:
raise RuntimeError(
"This action requires an active DataContext. "
+ "Please call `great_expectations.get_context()` first, then try your action again."
)
return self._project.validation_config_store

def get_evaluation_parameters_store(self) -> EvaluationParameterStore:
if not self._project:
raise RuntimeError(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,16 +1,19 @@
from __future__ import annotations

import uuid
from typing import TYPE_CHECKING

from great_expectations.compatibility.typing_extensions import override
from great_expectations.core.data_context_key import DataContextKey, StringKey
from great_expectations.core.validation_config import ValidationConfig
from great_expectations.data_context.cloud_constants import GXCloudRESTResource
from great_expectations.data_context.store.store import Store
from great_expectations.data_context.types.resource_identifiers import (
GXCloudIdentifier,
)

if TYPE_CHECKING:
from great_expectations.core.validation_config import ValidationConfig


class ValidationConfigStore(Store):
_key_class = StringKey
Expand Down Expand Up @@ -65,6 +68,8 @@ def serialize(self, value):

@override
def deserialize(self, value):
from great_expectations.core.validation_config import ValidationConfig

return ValidationConfig.parse_raw(value)

@override
Expand Down
59 changes: 45 additions & 14 deletions tests/core/test_validation_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,21 +34,27 @@
)


class TestValidationRun:
@pytest.fixture
def validation_config(self) -> ValidationConfig:
context = gx.get_context(mode="ephemeral")
batch_config = (
context.sources.add_pandas("my_datasource")
.add_csv_asset("csv_asset", "taxi.csv") # type: ignore
.add_batch_config("my_batch_config")
)
return ValidationConfig(
name="my_validation",
data=batch_config,
suite=ExpectationSuite(name="my_suite"),
)
@pytest.fixture
def ephemeral_context():
return gx.get_context(mode="ephemeral")


@pytest.fixture
def validation_config(ephemeral_context: EphemeralDataContext) -> ValidationConfig:
context = ephemeral_context
batch_config = (
context.sources.add_pandas("my_datasource")
.add_csv_asset("csv_asset", "taxi.csv") # type: ignore
.add_batch_config("my_batch_config")
)
return ValidationConfig(
name="my_validation",
data=batch_config,
suite=ExpectationSuite(name="my_suite"),
)


class TestValidationRun:
@pytest.fixture
def mock_validator(self):
"""Set up our ProjectManager to return a mock Validator"""
Expand Down Expand Up @@ -491,3 +497,28 @@ def test_validation_config_deserialization_non_existant_resource(
):
with pytest.raises(ValueError, match=f"{error_substring}*."):
ValidationConfig.parse_obj(serialized_config)


@pytest.mark.unit
def test_validation_config_save_success(
ephemeral_context: EphemeralDataContext, validation_config: ValidationConfig
):
context = ephemeral_context
vc = validation_config

vc = context.validations.add(vc)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: to make the intention of the test here more clear, having the initial context.validations.add(vc) being done in a fixture might make sense, e.g. like a saved_validation_config fixture.


other_suite = ExpectationSuite(name="my_other_suite")
vc.suite = other_suite
vc.save()

assert vc.suite == other_suite
assert context.validations.get(vc.name).suite == other_suite


@pytest.mark.unit
def test_validation_config_save_failure(validation_config: ValidationConfig):
with pytest.raises(
ValueError, match="ValidationConfig must be added to a store before saving."
):
validation_config.save()
Loading