From 212c246ace0fcb228121f9f9c136d74049634106 Mon Sep 17 00:00:00 2001 From: Anthony Burdi Date: Tue, 2 Feb 2021 14:23:01 -0500 Subject: [PATCH 01/16] add checkpoint store root_directory to FilesystemStoreBackendDefaults --- great_expectations/data_context/types/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/great_expectations/data_context/types/base.py b/great_expectations/data_context/types/base.py index 62a05a574144..bb0378aca7d1 100644 --- a/great_expectations/data_context/types/base.py +++ b/great_expectations/data_context/types/base.py @@ -1209,6 +1209,9 @@ def __init__( self.stores[self.validations_store_name]["store_backend"][ "root_directory" ] = root_directory + self.stores[self.checkpoint_store_name]["store_backend"][ + "root_directory" + ] = root_directory self.data_docs_sites[self.data_docs_site_name]["store_backend"][ "root_directory" ] = root_directory From 2ad32c58f07171c2db80ecb12194f72171e9371f Mon Sep 17 00:00:00 2001 From: Anthony Burdi Date: Tue, 2 Feb 2021 16:24:04 -0500 Subject: [PATCH 02/16] add checkpoint store to S3StoreBackendDefaults --- great_expectations/data_context/types/base.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/great_expectations/data_context/types/base.py b/great_expectations/data_context/types/base.py index bb0378aca7d1..db47d3401ddf 100644 --- a/great_expectations/data_context/types/base.py +++ b/great_expectations/data_context/types/base.py @@ -1110,12 +1110,15 @@ class S3StoreBackendDefaults(BaseStoreBackendDefaults): expectations_store_bucket_name: Overrides default_bucket_name if supplied validations_store_bucket_name: Overrides default_bucket_name if supplied data_docs_bucket_name: Overrides default_bucket_name if supplied + checkpoint_store_bucket_name: Overrides default_bucket_name if supplied expectations_store_prefix: Overrides default if supplied validations_store_prefix: Overrides default if supplied data_docs_prefix: Overrides default if supplied + checkpoint_store_prefix: Overrides default if supplied expectations_store_name: Overrides default if supplied validations_store_name: Overrides default if supplied evaluation_parameter_store_name: Overrides default if supplied + checkpoint_store_name: Overrides default if supplied """ def __init__( @@ -1124,12 +1127,15 @@ def __init__( expectations_store_bucket_name: Optional[str] = None, validations_store_bucket_name: Optional[str] = None, data_docs_bucket_name: Optional[str] = None, + checkpoint_store_bucket_name: Optional[str] = None, expectations_store_prefix: str = "expectations", validations_store_prefix: str = "validations", data_docs_prefix: str = "data_docs", + checkpoint_store_prefix: str = "data_docs", expectations_store_name: str = "expectations_S3_store", validations_store_name: str = "validations_S3_store", evaluation_parameter_store_name: str = "evaluation_parameter_store", + checkpoint_store_name: str = "checkpoint_store", ): # Initialize base defaults super().__init__() @@ -1141,11 +1147,14 @@ def __init__( validations_store_bucket_name = default_bucket_name if data_docs_bucket_name is None: data_docs_bucket_name = default_bucket_name + if checkpoint_store_bucket_name is None: + checkpoint_store_bucket_name = default_bucket_name # Overwrite defaults self.expectations_store_name = expectations_store_name self.validations_store_name = validations_store_name self.evaluation_parameter_store_name = evaluation_parameter_store_name + self.checkpoint_store_name = checkpoint_store_name self.stores = { expectations_store_name: { "class_name": "ExpectationsStore", @@ -1164,6 +1173,14 @@ def __init__( }, }, evaluation_parameter_store_name: {"class_name": "EvaluationParameterStore"}, + checkpoint_store_name: { + "class_name": "CheckpointStore", + "store_backend": { + "class_name": "TupleS3StoreBackend", + "bucket": checkpoint_store_bucket_name, + "prefix": checkpoint_store_prefix, + }, + }, } self.data_docs_sites = { "s3_site": { From 066ca1e0b8035cfdd1bc2844250253b01b37654d Mon Sep 17 00:00:00 2001 From: Anthony Burdi Date: Tue, 2 Feb 2021 16:42:46 -0500 Subject: [PATCH 03/16] add checkpoint store to GCSStoreBackendDefaults --- great_expectations/data_context/types/base.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/great_expectations/data_context/types/base.py b/great_expectations/data_context/types/base.py index db47d3401ddf..d6053f7d8150 100644 --- a/great_expectations/data_context/types/base.py +++ b/great_expectations/data_context/types/base.py @@ -1243,15 +1243,19 @@ class GCSStoreBackendDefaults(BaseStoreBackendDefaults): expectations_store_bucket_name: Overrides default_bucket_name if supplied validations_store_bucket_name: Overrides default_bucket_name if supplied data_docs_bucket_name: Overrides default_bucket_name if supplied + checkpoint_store_bucket_name: Overrides default_bucket_name if supplied expectations_store_project_name: Overrides default_project_name if supplied validations_store_project_name: Overrides default_project_name if supplied data_docs_project_name: Overrides default_project_name if supplied + checkpoint_store_project_name: Overrides default_project_name if supplied expectations_store_prefix: Overrides default if supplied validations_store_prefix: Overrides default if supplied data_docs_prefix: Overrides default if supplied + checkpoint_store_prefix: Overrides default if supplied expectations_store_name: Overrides default if supplied validations_store_name: Overrides default if supplied evaluation_parameter_store_name: Overrides default if supplied + checkpoint_store_name: Overrides default if supplied """ def __init__( @@ -1261,15 +1265,19 @@ def __init__( expectations_store_bucket_name: Optional[str] = None, validations_store_bucket_name: Optional[str] = None, data_docs_bucket_name: Optional[str] = None, + checkpoint_store_bucket_name: Optional[str] = None, expectations_store_project_name: Optional[str] = None, validations_store_project_name: Optional[str] = None, data_docs_project_name: Optional[str] = None, + checkpoint_store_project_name: Optional[str] = None, expectations_store_prefix: str = "expectations", validations_store_prefix: str = "validations", data_docs_prefix: str = "data_docs", + checkpoint_store_prefix: str = "data_docs", expectations_store_name: str = "expectations_GCS_store", validations_store_name: str = "validations_GCS_store", evaluation_parameter_store_name: str = "evaluation_parameter_store", + checkpoint_store_name: str = "evaluation_parameter_store", ): # Initialize base defaults super().__init__() @@ -1281,6 +1289,8 @@ def __init__( validations_store_bucket_name = default_bucket_name if data_docs_bucket_name is None: data_docs_bucket_name = default_bucket_name + if checkpoint_store_bucket_name is None: + checkpoint_store_bucket_name = default_bucket_name # Use default_project_name if separate store projects are not provided if expectations_store_project_name is None: @@ -1289,11 +1299,14 @@ def __init__( validations_store_project_name = default_project_name if data_docs_project_name is None: data_docs_project_name = default_project_name + if checkpoint_store_project_name is None: + checkpoint_store_project_name = default_project_name # Overwrite defaults self.expectations_store_name = expectations_store_name self.validations_store_name = validations_store_name self.evaluation_parameter_store_name = evaluation_parameter_store_name + self.checkpoint_store_name = checkpoint_store_name self.stores = { expectations_store_name: { "class_name": "ExpectationsStore", @@ -1314,6 +1327,14 @@ def __init__( }, }, evaluation_parameter_store_name: {"class_name": "EvaluationParameterStore"}, + checkpoint_store_name: { + "class_name": "CheckpointStore", + "store_backend": { + "class_name": "TupleS3StoreBackend", + "bucket": checkpoint_store_bucket_name, + "prefix": checkpoint_store_prefix, + }, + }, } self.data_docs_sites = { "gcs_site": { From 8a3d5469858f0b2879799ab2a81386152ebb3d56 Mon Sep 17 00:00:00 2001 From: Anthony Burdi Date: Tue, 2 Feb 2021 16:54:27 -0500 Subject: [PATCH 04/16] add checkpoint store to DatabaseStoreBackendDefaults --- great_expectations/data_context/types/base.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/great_expectations/data_context/types/base.py b/great_expectations/data_context/types/base.py index d6053f7d8150..e2d6e7a733d3 100644 --- a/great_expectations/data_context/types/base.py +++ b/great_expectations/data_context/types/base.py @@ -1360,9 +1360,11 @@ class DatabaseStoreBackendDefaults(BaseStoreBackendDefaults): default_credentials: Use these credentials for all stores that do not have credentials provided expectations_store_credentials: Overrides default_credentials if supplied validations_store_credentials: Overrides default_credentials if supplied + checkpoint_store_credentials: Overrides default_credentials if supplied expectations_store_name: Overrides default if supplied validations_store_name: Overrides default if supplied evaluation_parameter_store_name: Overrides default if supplied + checkpoint_store_name: Overrides default if supplied """ def __init__( @@ -1370,9 +1372,11 @@ def __init__( default_credentials: Optional[Dict] = None, expectations_store_credentials: Optional[Dict] = None, validations_store_credentials: Optional[Dict] = None, + checkpoint_store_credentials: Optional[Dict] = None, expectations_store_name: str = "expectations_database_store", validations_store_name: str = "validations_database_store", evaluation_parameter_store_name: str = "evaluation_parameter_store", + checkpoint_store_name: str = "checkpoint_store", ): # Initialize base defaults super().__init__() @@ -1382,11 +1386,14 @@ def __init__( expectations_store_credentials = default_credentials if validations_store_credentials is None: validations_store_credentials = default_credentials + if checkpoint_store_credentials is None: + checkpoint_store_credentials = default_credentials # Overwrite defaults self.expectations_store_name = expectations_store_name self.validations_store_name = validations_store_name self.evaluation_parameter_store_name = evaluation_parameter_store_name + self.checkpoint_store_name = checkpoint_store_name self.stores = { expectations_store_name: { @@ -1404,6 +1411,13 @@ def __init__( }, }, evaluation_parameter_store_name: {"class_name": "EvaluationParameterStore"}, + checkpoint_store_name: { + "class_name": "CheckpointStore", + "store_backend": { + "class_name": "DatabaseStoreBackend", + "credentials": checkpoint_store_credentials, + }, + }, } From 443d56b98dd806c2e4af7ec341bcafe692ef9b8f Mon Sep 17 00:00:00 2001 From: Anthony Burdi Date: Tue, 2 Feb 2021 17:01:54 -0500 Subject: [PATCH 05/16] add checkpoint_store_name default to DataContextConfig --- great_expectations/data_context/types/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/great_expectations/data_context/types/base.py b/great_expectations/data_context/types/base.py index e2d6e7a733d3..9e53c5d17b21 100644 --- a/great_expectations/data_context/types/base.py +++ b/great_expectations/data_context/types/base.py @@ -1467,6 +1467,8 @@ def __init__( ) if data_docs_sites is None: data_docs_sites = store_backend_defaults.data_docs_sites + if checkpoint_store_name is None: + checkpoint_store_name = store_backend_defaults.checkpoint_store_name self._config_version = config_version if datasources is None: From 8e4ce906a12b63ca45df80ea2e61f07097de3361 Mon Sep 17 00:00:00 2001 From: Anthony Burdi Date: Tue, 2 Feb 2021 17:17:07 -0500 Subject: [PATCH 06/16] fix default checkpoint store names --- great_expectations/data_context/types/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/great_expectations/data_context/types/base.py b/great_expectations/data_context/types/base.py index 9e53c5d17b21..2eed841573e9 100644 --- a/great_expectations/data_context/types/base.py +++ b/great_expectations/data_context/types/base.py @@ -1135,7 +1135,7 @@ def __init__( expectations_store_name: str = "expectations_S3_store", validations_store_name: str = "validations_S3_store", evaluation_parameter_store_name: str = "evaluation_parameter_store", - checkpoint_store_name: str = "checkpoint_store", + checkpoint_store_name: str = "checkpoint_S3_store", ): # Initialize base defaults super().__init__() @@ -1277,7 +1277,7 @@ def __init__( expectations_store_name: str = "expectations_GCS_store", validations_store_name: str = "validations_GCS_store", evaluation_parameter_store_name: str = "evaluation_parameter_store", - checkpoint_store_name: str = "evaluation_parameter_store", + checkpoint_store_name: str = "checkpoint_GCS_store", ): # Initialize base defaults super().__init__() @@ -1376,7 +1376,7 @@ def __init__( expectations_store_name: str = "expectations_database_store", validations_store_name: str = "validations_database_store", evaluation_parameter_store_name: str = "evaluation_parameter_store", - checkpoint_store_name: str = "checkpoint_store", + checkpoint_store_name: str = "checkpoint_database_store", ): # Initialize base defaults super().__init__() From 884e0288d1f980474a833506bcc45a5dcde0f347 Mon Sep 17 00:00:00 2001 From: Anthony Burdi Date: Tue, 2 Feb 2021 17:19:46 -0500 Subject: [PATCH 07/16] fix default checkpoint store prefixes --- great_expectations/data_context/types/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/great_expectations/data_context/types/base.py b/great_expectations/data_context/types/base.py index 2eed841573e9..2f8275293a52 100644 --- a/great_expectations/data_context/types/base.py +++ b/great_expectations/data_context/types/base.py @@ -1131,7 +1131,7 @@ def __init__( expectations_store_prefix: str = "expectations", validations_store_prefix: str = "validations", data_docs_prefix: str = "data_docs", - checkpoint_store_prefix: str = "data_docs", + checkpoint_store_prefix: str = "checkpoints", expectations_store_name: str = "expectations_S3_store", validations_store_name: str = "validations_S3_store", evaluation_parameter_store_name: str = "evaluation_parameter_store", @@ -1273,7 +1273,7 @@ def __init__( expectations_store_prefix: str = "expectations", validations_store_prefix: str = "validations", data_docs_prefix: str = "data_docs", - checkpoint_store_prefix: str = "data_docs", + checkpoint_store_prefix: str = "checkpoints", expectations_store_name: str = "expectations_GCS_store", validations_store_name: str = "validations_GCS_store", evaluation_parameter_store_name: str = "evaluation_parameter_store", From af359263cc80b7254914d7c1b1f4b1547c0b998a Mon Sep 17 00:00:00 2001 From: Anthony Burdi Date: Tue, 2 Feb 2021 17:26:47 -0500 Subject: [PATCH 08/16] fix test_DataContextConfig_with_S3StoreBackendDefaults --- tests/data_context/test_data_context_config_ui.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/data_context/test_data_context_config_ui.py b/tests/data_context/test_data_context_config_ui.py index 19f820e45acb..82e650f2ef6c 100644 --- a/tests/data_context/test_data_context_config_ui.py +++ b/tests/data_context/test_data_context_config_ui.py @@ -175,7 +175,6 @@ def test_DataContextConfig_with_S3StoreBackendDefaults( ) }, store_backend_defaults=store_backend_defaults, - checkpoint_store_name=store_backend_defaults.checkpoint_store_name, ) # Create desired config @@ -197,6 +196,14 @@ def test_DataContextConfig_with_S3StoreBackendDefaults( "prefix": "validations", }, }, + "checkpoint_S3_store": { + "class_name": "CheckpointStore", + "store_backend": { + "bucket": "my_default_bucket", + "class_name": "TupleS3StoreBackend", + "prefix": "checkpoints", + }, + }, } desired_data_docs_sites_config = { "s3_site": { @@ -219,6 +226,7 @@ def test_DataContextConfig_with_S3StoreBackendDefaults( expectations_store_name="expectations_S3_store", validations_store_name="validations_S3_store", evaluation_parameter_store_name=DataContextConfigDefaults.DEFAULT_EVALUATION_PARAMETER_STORE_NAME.value, + checkpoint_store_name="checkpoint_S3_store", stores=desired_stores_config, data_docs_sites=desired_data_docs_sites_config, ) From 9b66f0a4f42782f702607f316a1e700bcee5617d Mon Sep 17 00:00:00 2001 From: Anthony Burdi Date: Tue, 2 Feb 2021 17:32:37 -0500 Subject: [PATCH 09/16] fix test_DataContextConfig_with_S3StoreBackendDefaults_using_all_parameters --- tests/data_context/test_data_context_config_ui.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/data_context/test_data_context_config_ui.py b/tests/data_context/test_data_context_config_ui.py index 82e650f2ef6c..f19002b5f881 100644 --- a/tests/data_context/test_data_context_config_ui.py +++ b/tests/data_context/test_data_context_config_ui.py @@ -252,12 +252,15 @@ def test_DataContextConfig_with_S3StoreBackendDefaults_using_all_parameters( expectations_store_bucket_name="custom_expectations_store_bucket_name", validations_store_bucket_name="custom_validations_store_bucket_name", data_docs_bucket_name="custom_data_docs_store_bucket_name", + checkpoint_store_bucket_name="custom_checkpoint_store_bucket_name", expectations_store_prefix="custom_expectations_store_prefix", validations_store_prefix="custom_validations_store_prefix", data_docs_prefix="custom_data_docs_prefix", + checkpoint_store_prefix="custom_checkpoint_store_prefix", expectations_store_name="custom_expectations_S3_store_name", validations_store_name="custom_validations_S3_store_name", evaluation_parameter_store_name="custom_evaluation_parameter_store_name", + checkpoint_store_name="custom_checkpoint_S3_store_name", ) data_context_config = DataContextConfig( datasources={ @@ -277,7 +280,6 @@ def test_DataContextConfig_with_S3StoreBackendDefaults_using_all_parameters( ) }, store_backend_defaults=store_backend_defaults, - checkpoint_store_name=store_backend_defaults.checkpoint_store_name, ) # Create desired config @@ -301,6 +303,14 @@ def test_DataContextConfig_with_S3StoreBackendDefaults_using_all_parameters( "prefix": "custom_validations_store_prefix", }, }, + "custom_checkpoint_S3_store_name": { + "class_name": "CheckpointStore", + "store_backend": { + "bucket": "custom_checkpoint_store_bucket_name", + "class_name": "TupleS3StoreBackend", + "prefix": "custom_checkpoint_store_prefix", + }, + }, } desired_data_docs_sites_config = { "s3_site": { @@ -323,6 +333,7 @@ def test_DataContextConfig_with_S3StoreBackendDefaults_using_all_parameters( expectations_store_name="custom_expectations_S3_store_name", validations_store_name="custom_validations_S3_store_name", evaluation_parameter_store_name="custom_evaluation_parameter_store_name", + checkpoint_store_name="custom_checkpoint_S3_store_name", stores=desired_stores_config, data_docs_sites=desired_data_docs_sites_config, ) From b197e59d271615489b5b56bf8f31fbae746f1446 Mon Sep 17 00:00:00 2001 From: Anthony Burdi Date: Tue, 2 Feb 2021 17:33:59 -0500 Subject: [PATCH 10/16] fix test_DataContextConfig_with_FilesystemStoreBackendDefaults_and_simple_defaults --- tests/data_context/test_data_context_config_ui.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/data_context/test_data_context_config_ui.py b/tests/data_context/test_data_context_config_ui.py index f19002b5f881..302e0000c55a 100644 --- a/tests/data_context/test_data_context_config_ui.py +++ b/tests/data_context/test_data_context_config_ui.py @@ -372,7 +372,6 @@ def test_DataContextConfig_with_FilesystemStoreBackendDefaults_and_simple_defaul ) }, store_backend_defaults=store_backend_defaults, - checkpoint_store_name=store_backend_defaults.checkpoint_store_name, ) # Create desired config @@ -387,6 +386,9 @@ def test_DataContextConfig_with_FilesystemStoreBackendDefaults_and_simple_defaul desired_config["stores"][desired_config["validations_store_name"]]["store_backend"][ "root_directory" ] = test_root_directory + desired_config["stores"][desired_config["checkpoint_store_name"]]["store_backend"][ + "root_directory" + ] = test_root_directory desired_config["data_docs_sites"]["local_site"]["store_backend"][ "root_directory" ] = test_root_directory From f701cd1ff43399f2a115e12d186f999a8e565325 Mon Sep 17 00:00:00 2001 From: Anthony Burdi Date: Tue, 2 Feb 2021 17:46:31 -0500 Subject: [PATCH 11/16] fix test_DataContextConfig_with_GCSStoreBackendDefaults --- great_expectations/data_context/types/base.py | 3 ++- tests/data_context/test_data_context_config_ui.py | 11 ++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/great_expectations/data_context/types/base.py b/great_expectations/data_context/types/base.py index 2f8275293a52..2557e784cb0c 100644 --- a/great_expectations/data_context/types/base.py +++ b/great_expectations/data_context/types/base.py @@ -1330,7 +1330,8 @@ def __init__( checkpoint_store_name: { "class_name": "CheckpointStore", "store_backend": { - "class_name": "TupleS3StoreBackend", + "class_name": "TupleGCSStoreBackend", + "project": checkpoint_store_project_name, "bucket": checkpoint_store_bucket_name, "prefix": checkpoint_store_prefix, }, diff --git a/tests/data_context/test_data_context_config_ui.py b/tests/data_context/test_data_context_config_ui.py index 302e0000c55a..3852cbd579cd 100644 --- a/tests/data_context/test_data_context_config_ui.py +++ b/tests/data_context/test_data_context_config_ui.py @@ -471,7 +471,6 @@ def test_DataContextConfig_with_GCSStoreBackendDefaults( ) }, store_backend_defaults=store_backend_defaults, - checkpoint_store_name=store_backend_defaults.checkpoint_store_name, ) # Create desired config @@ -496,6 +495,15 @@ def test_DataContextConfig_with_GCSStoreBackendDefaults( "prefix": "validations", }, }, + "checkpoint_GCS_store": { + "class_name": "CheckpointStore", + "store_backend": { + "bucket": "my_default_bucket", + "project": "my_default_project", + "class_name": "TupleGCSStoreBackend", + "prefix": "checkpoints", + }, + }, } desired_data_docs_sites_config = { "gcs_site": { @@ -518,6 +526,7 @@ def test_DataContextConfig_with_GCSStoreBackendDefaults( datasources=default_pandas_datasource_config, expectations_store_name="expectations_GCS_store", validations_store_name="validations_GCS_store", + checkpoint_store_name="checkpoint_GCS_store", evaluation_parameter_store_name=DataContextConfigDefaults.DEFAULT_EVALUATION_PARAMETER_STORE_NAME.value, stores=desired_stores_config, data_docs_sites=desired_data_docs_sites_config, From 1d025a5ec72e91ab87a3d1bf2acff37bf1b3154a Mon Sep 17 00:00:00 2001 From: Anthony Burdi Date: Tue, 2 Feb 2021 17:50:46 -0500 Subject: [PATCH 12/16] fix test_DataContextConfig_with_GCSStoreBackendDefaults_using_all_parameters --- tests/data_context/test_data_context_config_ui.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/data_context/test_data_context_config_ui.py b/tests/data_context/test_data_context_config_ui.py index 3852cbd579cd..af8f2e989603 100644 --- a/tests/data_context/test_data_context_config_ui.py +++ b/tests/data_context/test_data_context_config_ui.py @@ -554,15 +554,19 @@ def test_DataContextConfig_with_GCSStoreBackendDefaults_using_all_parameters( expectations_store_bucket_name="custom_expectations_store_bucket_name", validations_store_bucket_name="custom_validations_store_bucket_name", data_docs_bucket_name="custom_data_docs_store_bucket_name", + checkpoint_store_bucket_name="custom_checkpoint_store_bucket_name", expectations_store_project_name="custom_expectations_store_project_name", validations_store_project_name="custom_validations_store_project_name", data_docs_project_name="custom_data_docs_store_project_name", + checkpoint_store_project_name="custom_checkpoint_store_project_name", expectations_store_prefix="custom_expectations_store_prefix", validations_store_prefix="custom_validations_store_prefix", data_docs_prefix="custom_data_docs_prefix", + checkpoint_store_prefix="custom_checkpoint_store_prefix", expectations_store_name="custom_expectations_GCS_store_name", validations_store_name="custom_validations_GCS_store_name", evaluation_parameter_store_name="custom_evaluation_parameter_store_name", + checkpoint_store_name="custom_checkpoint_GCS_store_name", ) data_context_config = DataContextConfig( datasources={ @@ -582,7 +586,6 @@ def test_DataContextConfig_with_GCSStoreBackendDefaults_using_all_parameters( ) }, store_backend_defaults=store_backend_defaults, - checkpoint_store_name=store_backend_defaults.checkpoint_store_name, ) # Create desired config @@ -608,6 +611,15 @@ def test_DataContextConfig_with_GCSStoreBackendDefaults_using_all_parameters( "prefix": "custom_validations_store_prefix", }, }, + "custom_checkpoint_GCS_store_name": { + "class_name": "CheckpointStore", + "store_backend": { + "bucket": "custom_checkpoint_store_bucket_name", + "project": "custom_checkpoint_store_project_name", + "class_name": "TupleGCSStoreBackend", + "prefix": "custom_checkpoint_store_prefix", + }, + }, } desired_data_docs_sites_config = { "gcs_site": { @@ -630,6 +642,7 @@ def test_DataContextConfig_with_GCSStoreBackendDefaults_using_all_parameters( expectations_store_name="custom_expectations_GCS_store_name", validations_store_name="custom_validations_GCS_store_name", evaluation_parameter_store_name="custom_evaluation_parameter_store_name", + checkpoint_store_name="custom_checkpoint_GCS_store_name", stores=desired_stores_config, data_docs_sites=desired_data_docs_sites_config, ) From dfbc6173effccaacdf7391abef27047bc847f1d1 Mon Sep 17 00:00:00 2001 From: Anthony Burdi Date: Tue, 2 Feb 2021 17:52:15 -0500 Subject: [PATCH 13/16] fix test_DataContextConfig_with_DatabaseStoreBackendDefaults --- .../data_context/test_data_context_config_ui.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/data_context/test_data_context_config_ui.py b/tests/data_context/test_data_context_config_ui.py index af8f2e989603..37a84068dfc8 100644 --- a/tests/data_context/test_data_context_config_ui.py +++ b/tests/data_context/test_data_context_config_ui.py @@ -691,7 +691,6 @@ def test_DataContextConfig_with_DatabaseStoreBackendDefaults( ) }, store_backend_defaults=store_backend_defaults, - checkpoint_store_name=store_backend_defaults.checkpoint_store_name, ) # Create desired config @@ -725,6 +724,20 @@ def test_DataContextConfig_with_DatabaseStoreBackendDefaults( }, }, }, + "checkpoint_database_store": { + "class_name": "CheckpointStore", + "store_backend": { + "class_name": "DatabaseStoreBackend", + "credentials": { + "drivername": "postgresql", + "host": "localhost", + "port": "65432", + "username": "ge_tutorials", + "password": "ge_tutorials", + "database": "ge_tutorials", + }, + }, + }, } desired_data_docs_sites_config = { "local_site": { @@ -745,6 +758,7 @@ def test_DataContextConfig_with_DatabaseStoreBackendDefaults( datasources=default_pandas_datasource_config, expectations_store_name="expectations_database_store", validations_store_name="validations_database_store", + checkpoint_store_name="checkpoint_database_store", evaluation_parameter_store_name=DataContextConfigDefaults.DEFAULT_EVALUATION_PARAMETER_STORE_NAME.value, stores=desired_stores_config, data_docs_sites=desired_data_docs_sites_config, From 6a10c6ecde03c79cd5853d6c9ab78cfd94f2abcb Mon Sep 17 00:00:00 2001 From: Anthony Burdi Date: Tue, 2 Feb 2021 18:04:45 -0500 Subject: [PATCH 14/16] fix test_DataContextConfig_with_DatabaseStoreBackendDefaults_using_all_parameters --- .../test_data_context_config_ui.py | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/data_context/test_data_context_config_ui.py b/tests/data_context/test_data_context_config_ui.py index 37a84068dfc8..419d01995ae1 100644 --- a/tests/data_context/test_data_context_config_ui.py +++ b/tests/data_context/test_data_context_config_ui.py @@ -805,9 +805,18 @@ def test_DataContextConfig_with_DatabaseStoreBackendDefaults_using_all_parameter "password": "custom_validations_store_password", "database": "custom_validations_store_database", }, + checkpoint_store_credentials={ + "drivername": "custom_checkpoint_store_drivername", + "host": "custom_checkpoint_store_host", + "port": "custom_checkpoint_store_port", + "username": "custom_checkpoint_store_username", + "password": "custom_checkpoint_store_password", + "database": "custom_checkpoint_store_database", + }, expectations_store_name="custom_expectations_database_store_name", validations_store_name="custom_validations_database_store_name", evaluation_parameter_store_name="custom_evaluation_parameter_store_name", + checkpoint_store_name="custom_checkpoint_database_store_name", ) data_context_config = DataContextConfig( datasources={ @@ -827,7 +836,6 @@ def test_DataContextConfig_with_DatabaseStoreBackendDefaults_using_all_parameter ) }, store_backend_defaults=store_backend_defaults, - checkpoint_store_name=store_backend_defaults.checkpoint_store_name, ) # Create desired config @@ -863,6 +871,20 @@ def test_DataContextConfig_with_DatabaseStoreBackendDefaults_using_all_parameter }, }, }, + "custom_checkpoint_database_store_name": { + "class_name": "CheckpointStore", + "store_backend": { + "class_name": "DatabaseStoreBackend", + "credentials": { + "database": "custom_checkpoint_store_database", + "drivername": "custom_checkpoint_store_drivername", + "host": "custom_checkpoint_store_host", + "password": "custom_checkpoint_store_password", + "port": "custom_checkpoint_store_port", + "username": "custom_checkpoint_store_username", + }, + }, + }, } desired_data_docs_sites_config = { "local_site": { @@ -884,6 +906,7 @@ def test_DataContextConfig_with_DatabaseStoreBackendDefaults_using_all_parameter expectations_store_name="custom_expectations_database_store_name", validations_store_name="custom_validations_database_store_name", evaluation_parameter_store_name="custom_evaluation_parameter_store_name", + checkpoint_store_name="custom_checkpoint_database_store_name", stores=desired_stores_config, data_docs_sites=desired_data_docs_sites_config, ) From 628c72a255b7edcbf5d3d87815b826e9e79d03db Mon Sep 17 00:00:00 2001 From: Anthony Burdi Date: Tue, 2 Feb 2021 18:06:42 -0500 Subject: [PATCH 15/16] fix test_DataContextConfig_with_S3StoreBackendDefaults_and_simple_defaults_with_variable_sub --- tests/data_context/test_data_context_config_ui.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/data_context/test_data_context_config_ui.py b/tests/data_context/test_data_context_config_ui.py index 419d01995ae1..5521ef22258e 100644 --- a/tests/data_context/test_data_context_config_ui.py +++ b/tests/data_context/test_data_context_config_ui.py @@ -1186,7 +1186,6 @@ def test_DataContextConfig_with_S3StoreBackendDefaults_and_simple_defaults_with_ ) }, store_backend_defaults=store_backend_defaults, - checkpoint_store_name=store_backend_defaults.checkpoint_store_name, ) # Create desired config @@ -1208,6 +1207,14 @@ def test_DataContextConfig_with_S3StoreBackendDefaults_and_simple_defaults_with_ "prefix": "validations", }, }, + "checkpoint_S3_store": { + "class_name": "CheckpointStore", + "store_backend": { + "bucket": "my_default_bucket", + "class_name": "TupleS3StoreBackend", + "prefix": "checkpoints", + }, + }, } desired_data_docs_sites_config = { "s3_site": { @@ -1229,6 +1236,7 @@ def test_DataContextConfig_with_S3StoreBackendDefaults_and_simple_defaults_with_ datasources=default_pandas_datasource_config, expectations_store_name="expectations_S3_store", validations_store_name="validations_S3_store", + checkpoint_store_name="checkpoint_S3_store", evaluation_parameter_store_name=DataContextConfigDefaults.DEFAULT_EVALUATION_PARAMETER_STORE_NAME.value, stores=desired_stores_config, data_docs_sites=desired_data_docs_sites_config, From 43c5f820234452c3b19882db8109fc1b2774a47f Mon Sep 17 00:00:00 2001 From: Anthony Burdi Date: Tue, 2 Feb 2021 18:22:51 -0500 Subject: [PATCH 16/16] update changelog --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 00235ba10c13..99900cbfcbbd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,7 +9,7 @@ Develop ----------------- * [DOCS] How to load a Pandas DataFrame as a Batch #2327 * [ENHANCEMENT] Add possibility to pass boto3 configuration to TupleS3StoreBackend (Thanks for #1691 to @mgorsk1!) #2371 - +* [MAINTENANCE] Add checkpoint store to store backend defaults #2378 0.13.8 -----------------