Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[MAINTENANCE] Add checkpoint store to store backend defaults #2378

Merged
merged 21 commits into from
Feb 4, 2021
Merged

[MAINTENANCE] Add checkpoint store to store backend defaults #2378

Show file tree
Hide file tree
Changes from 17 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
212c246
add checkpoint store root_directory to FilesystemStoreBackendDefaults
anthonyburdi Feb 2, 2021
2ad32c5
add checkpoint store to S3StoreBackendDefaults
anthonyburdi Feb 2, 2021
066ca1e
add checkpoint store to GCSStoreBackendDefaults
anthonyburdi Feb 2, 2021
8a3d546
add checkpoint store to DatabaseStoreBackendDefaults
anthonyburdi Feb 2, 2021
443d56b
add checkpoint_store_name default to DataContextConfig
anthonyburdi Feb 2, 2021
8e4ce90
fix default checkpoint store names
anthonyburdi Feb 2, 2021
884e028
fix default checkpoint store prefixes
anthonyburdi Feb 2, 2021
af35926
fix test_DataContextConfig_with_S3StoreBackendDefaults
anthonyburdi Feb 2, 2021
9b66f0a
fix test_DataContextConfig_with_S3StoreBackendDefaults_using_all_para…
anthonyburdi Feb 2, 2021
b197e59
fix test_DataContextConfig_with_FilesystemStoreBackendDefaults_and_si…
anthonyburdi Feb 2, 2021
f701cd1
fix test_DataContextConfig_with_GCSStoreBackendDefaults
anthonyburdi Feb 2, 2021
1d025a5
fix test_DataContextConfig_with_GCSStoreBackendDefaults_using_all_par…
anthonyburdi Feb 2, 2021
dfbc617
fix test_DataContextConfig_with_DatabaseStoreBackendDefaults
anthonyburdi Feb 2, 2021
6a10c6e
fix test_DataContextConfig_with_DatabaseStoreBackendDefaults_using_al…
anthonyburdi Feb 2, 2021
628c72a
fix test_DataContextConfig_with_S3StoreBackendDefaults_and_simple_def…
anthonyburdi Feb 2, 2021
c0f895c
Merge branch 'develop' into anthony/maintenance/add-checkpoint-store-…
anthonyburdi Feb 2, 2021
43c5f82
update changelog
anthonyburdi Feb 2, 2021
8092d6d
Merge branch 'develop' into anthony/maintenance/add-checkpoint-store-…
anthonyburdi Feb 3, 2021
25aae27
Merge branch 'develop' into anthony/maintenance/add-checkpoint-store-…
anthonyburdi Feb 3, 2021
30a45d0
Merge branch 'develop' into anthony/maintenance/add-checkpoint-store-…
anthonyburdi Feb 3, 2021
1e86da4
Merge branch 'develop' into anthony/maintenance/add-checkpoint-store-…
anthonyburdi Feb 3, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/changelog.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ Develop
-----------------
* [DOCS] How to load a Pandas DataFrame as a Batch #2327
* [ENHANCEMENT] Add possibility to pass boto3 configuration to TupleS3StoreBackend (Thanks for #1691 to @mgorsk1!) #2371

* [MAINTENANCE] Add checkpoint store to store backend defaults #2378

0.13.8
-----------------
Expand Down
58 changes: 58 additions & 0 deletions great_expectations/data_context/types/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1110,12 +1110,15 @@ class S3StoreBackendDefaults(BaseStoreBackendDefaults):
expectations_store_bucket_name: Overrides default_bucket_name if supplied
validations_store_bucket_name: Overrides default_bucket_name if supplied
data_docs_bucket_name: Overrides default_bucket_name if supplied
checkpoint_store_bucket_name: Overrides default_bucket_name if supplied
expectations_store_prefix: Overrides default if supplied
validations_store_prefix: Overrides default if supplied
data_docs_prefix: Overrides default if supplied
checkpoint_store_prefix: Overrides default if supplied
expectations_store_name: Overrides default if supplied
validations_store_name: Overrides default if supplied
evaluation_parameter_store_name: Overrides default if supplied
checkpoint_store_name: Overrides default if supplied
"""

def __init__(
Expand All @@ -1124,12 +1127,15 @@ def __init__(
expectations_store_bucket_name: Optional[str] = None,
validations_store_bucket_name: Optional[str] = None,
data_docs_bucket_name: Optional[str] = None,
checkpoint_store_bucket_name: Optional[str] = None,
expectations_store_prefix: str = "expectations",
validations_store_prefix: str = "validations",
data_docs_prefix: str = "data_docs",
checkpoint_store_prefix: str = "checkpoints",
expectations_store_name: str = "expectations_S3_store",
validations_store_name: str = "validations_S3_store",
evaluation_parameter_store_name: str = "evaluation_parameter_store",
checkpoint_store_name: str = "checkpoint_S3_store",
):
# Initialize base defaults
super().__init__()
Expand All @@ -1141,11 +1147,14 @@ def __init__(
validations_store_bucket_name = default_bucket_name
if data_docs_bucket_name is None:
data_docs_bucket_name = default_bucket_name
if checkpoint_store_bucket_name is None:
checkpoint_store_bucket_name = default_bucket_name

# Overwrite defaults
self.expectations_store_name = expectations_store_name
self.validations_store_name = validations_store_name
self.evaluation_parameter_store_name = evaluation_parameter_store_name
self.checkpoint_store_name = checkpoint_store_name
self.stores = {
expectations_store_name: {
"class_name": "ExpectationsStore",
Expand All @@ -1164,6 +1173,14 @@ def __init__(
},
},
evaluation_parameter_store_name: {"class_name": "EvaluationParameterStore"},
checkpoint_store_name: {
"class_name": "CheckpointStore",
"store_backend": {
"class_name": "TupleS3StoreBackend",
"bucket": checkpoint_store_bucket_name,
"prefix": checkpoint_store_prefix,
},
},
}
self.data_docs_sites = {
"s3_site": {
Expand Down Expand Up @@ -1209,6 +1226,9 @@ def __init__(
self.stores[self.validations_store_name]["store_backend"][
"root_directory"
] = root_directory
self.stores[self.checkpoint_store_name]["store_backend"][
"root_directory"
] = root_directory
self.data_docs_sites[self.data_docs_site_name]["store_backend"][
"root_directory"
] = root_directory
Expand All @@ -1223,15 +1243,19 @@ class GCSStoreBackendDefaults(BaseStoreBackendDefaults):
expectations_store_bucket_name: Overrides default_bucket_name if supplied
validations_store_bucket_name: Overrides default_bucket_name if supplied
data_docs_bucket_name: Overrides default_bucket_name if supplied
checkpoint_store_bucket_name: Overrides default_bucket_name if supplied
expectations_store_project_name: Overrides default_project_name if supplied
validations_store_project_name: Overrides default_project_name if supplied
data_docs_project_name: Overrides default_project_name if supplied
checkpoint_store_project_name: Overrides default_project_name if supplied
expectations_store_prefix: Overrides default if supplied
validations_store_prefix: Overrides default if supplied
data_docs_prefix: Overrides default if supplied
checkpoint_store_prefix: Overrides default if supplied
expectations_store_name: Overrides default if supplied
validations_store_name: Overrides default if supplied
evaluation_parameter_store_name: Overrides default if supplied
checkpoint_store_name: Overrides default if supplied
"""

def __init__(
Expand All @@ -1241,15 +1265,19 @@ def __init__(
expectations_store_bucket_name: Optional[str] = None,
validations_store_bucket_name: Optional[str] = None,
data_docs_bucket_name: Optional[str] = None,
checkpoint_store_bucket_name: Optional[str] = None,
expectations_store_project_name: Optional[str] = None,
validations_store_project_name: Optional[str] = None,
data_docs_project_name: Optional[str] = None,
checkpoint_store_project_name: Optional[str] = None,
expectations_store_prefix: str = "expectations",
validations_store_prefix: str = "validations",
data_docs_prefix: str = "data_docs",
checkpoint_store_prefix: str = "checkpoints",
expectations_store_name: str = "expectations_GCS_store",
validations_store_name: str = "validations_GCS_store",
evaluation_parameter_store_name: str = "evaluation_parameter_store",
checkpoint_store_name: str = "checkpoint_GCS_store",
):
# Initialize base defaults
super().__init__()
Expand All @@ -1261,6 +1289,8 @@ def __init__(
validations_store_bucket_name = default_bucket_name
if data_docs_bucket_name is None:
data_docs_bucket_name = default_bucket_name
if checkpoint_store_bucket_name is None:
checkpoint_store_bucket_name = default_bucket_name

# Use default_project_name if separate store projects are not provided
if expectations_store_project_name is None:
Expand All @@ -1269,11 +1299,14 @@ def __init__(
validations_store_project_name = default_project_name
if data_docs_project_name is None:
data_docs_project_name = default_project_name
if checkpoint_store_project_name is None:
checkpoint_store_project_name = default_project_name

# Overwrite defaults
self.expectations_store_name = expectations_store_name
self.validations_store_name = validations_store_name
self.evaluation_parameter_store_name = evaluation_parameter_store_name
self.checkpoint_store_name = checkpoint_store_name
self.stores = {
expectations_store_name: {
"class_name": "ExpectationsStore",
Expand All @@ -1294,6 +1327,15 @@ def __init__(
},
},
evaluation_parameter_store_name: {"class_name": "EvaluationParameterStore"},
checkpoint_store_name: {
"class_name": "CheckpointStore",
"store_backend": {
"class_name": "TupleGCSStoreBackend",
"project": checkpoint_store_project_name,
"bucket": checkpoint_store_bucket_name,
"prefix": checkpoint_store_prefix,
},
},
}
self.data_docs_sites = {
"gcs_site": {
Expand All @@ -1319,19 +1361,23 @@ class DatabaseStoreBackendDefaults(BaseStoreBackendDefaults):
default_credentials: Use these credentials for all stores that do not have credentials provided
expectations_store_credentials: Overrides default_credentials if supplied
validations_store_credentials: Overrides default_credentials if supplied
checkpoint_store_credentials: Overrides default_credentials if supplied
expectations_store_name: Overrides default if supplied
validations_store_name: Overrides default if supplied
evaluation_parameter_store_name: Overrides default if supplied
checkpoint_store_name: Overrides default if supplied
"""

def __init__(
self,
default_credentials: Optional[Dict] = None,
expectations_store_credentials: Optional[Dict] = None,
validations_store_credentials: Optional[Dict] = None,
checkpoint_store_credentials: Optional[Dict] = None,
expectations_store_name: str = "expectations_database_store",
validations_store_name: str = "validations_database_store",
evaluation_parameter_store_name: str = "evaluation_parameter_store",
checkpoint_store_name: str = "checkpoint_database_store",
):
# Initialize base defaults
super().__init__()
Expand All @@ -1341,11 +1387,14 @@ def __init__(
expectations_store_credentials = default_credentials
if validations_store_credentials is None:
validations_store_credentials = default_credentials
if checkpoint_store_credentials is None:
checkpoint_store_credentials = default_credentials

# Overwrite defaults
self.expectations_store_name = expectations_store_name
self.validations_store_name = validations_store_name
self.evaluation_parameter_store_name = evaluation_parameter_store_name
self.checkpoint_store_name = checkpoint_store_name

self.stores = {
expectations_store_name: {
Expand All @@ -1363,6 +1412,13 @@ def __init__(
},
},
evaluation_parameter_store_name: {"class_name": "EvaluationParameterStore"},
checkpoint_store_name: {
"class_name": "CheckpointStore",
"store_backend": {
"class_name": "DatabaseStoreBackend",
"credentials": checkpoint_store_credentials,
},
},
}


Expand Down Expand Up @@ -1412,6 +1468,8 @@ def __init__(
)
if data_docs_sites is None:
data_docs_sites = store_backend_defaults.data_docs_sites
if checkpoint_store_name is None:
checkpoint_store_name = store_backend_defaults.checkpoint_store_name

self._config_version = config_version
if datasources is None:
Expand Down