diff --git a/docs/changelog.rst b/docs/changelog.rst index 15c6c1154b86..8657c70c5a34 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,7 +10,7 @@ Develop * [FEATURE] Add TupleAzureBlobStoreBackend * [DOCS] How to load a Pandas DataFrame as a Batch #2327 * [ENHANCEMENT] Add possibility to pass boto3 configuration to TupleS3StoreBackend (Thanks for #1691 to @mgorsk1!) #2371 - +* [MAINTENANCE] Add checkpoint store to store backend defaults #2378 0.13.8 ----------------- diff --git a/great_expectations/data_context/types/base.py b/great_expectations/data_context/types/base.py index 62a05a574144..2557e784cb0c 100644 --- a/great_expectations/data_context/types/base.py +++ b/great_expectations/data_context/types/base.py @@ -1110,12 +1110,15 @@ class S3StoreBackendDefaults(BaseStoreBackendDefaults): expectations_store_bucket_name: Overrides default_bucket_name if supplied validations_store_bucket_name: Overrides default_bucket_name if supplied data_docs_bucket_name: Overrides default_bucket_name if supplied + checkpoint_store_bucket_name: Overrides default_bucket_name if supplied expectations_store_prefix: Overrides default if supplied validations_store_prefix: Overrides default if supplied data_docs_prefix: Overrides default if supplied + checkpoint_store_prefix: Overrides default if supplied expectations_store_name: Overrides default if supplied validations_store_name: Overrides default if supplied evaluation_parameter_store_name: Overrides default if supplied + checkpoint_store_name: Overrides default if supplied """ def __init__( @@ -1124,12 +1127,15 @@ def __init__( expectations_store_bucket_name: Optional[str] = None, validations_store_bucket_name: Optional[str] = None, data_docs_bucket_name: Optional[str] = None, + checkpoint_store_bucket_name: Optional[str] = None, expectations_store_prefix: str = "expectations", validations_store_prefix: str = "validations", data_docs_prefix: str = "data_docs", + checkpoint_store_prefix: str = "checkpoints", expectations_store_name: str = "expectations_S3_store", validations_store_name: str = "validations_S3_store", evaluation_parameter_store_name: str = "evaluation_parameter_store", + checkpoint_store_name: str = "checkpoint_S3_store", ): # Initialize base defaults super().__init__() @@ -1141,11 +1147,14 @@ def __init__( validations_store_bucket_name = default_bucket_name if data_docs_bucket_name is None: data_docs_bucket_name = default_bucket_name + if checkpoint_store_bucket_name is None: + checkpoint_store_bucket_name = default_bucket_name # Overwrite defaults self.expectations_store_name = expectations_store_name self.validations_store_name = validations_store_name self.evaluation_parameter_store_name = evaluation_parameter_store_name + self.checkpoint_store_name = checkpoint_store_name self.stores = { expectations_store_name: { "class_name": "ExpectationsStore", @@ -1164,6 +1173,14 @@ def __init__( }, }, evaluation_parameter_store_name: {"class_name": "EvaluationParameterStore"}, + checkpoint_store_name: { + "class_name": "CheckpointStore", + "store_backend": { + "class_name": "TupleS3StoreBackend", + "bucket": checkpoint_store_bucket_name, + "prefix": checkpoint_store_prefix, + }, + }, } self.data_docs_sites = { "s3_site": { @@ -1209,6 +1226,9 @@ def __init__( self.stores[self.validations_store_name]["store_backend"][ "root_directory" ] = root_directory + self.stores[self.checkpoint_store_name]["store_backend"][ + "root_directory" + ] = root_directory self.data_docs_sites[self.data_docs_site_name]["store_backend"][ "root_directory" ] = root_directory @@ -1223,15 +1243,19 @@ class GCSStoreBackendDefaults(BaseStoreBackendDefaults): expectations_store_bucket_name: Overrides default_bucket_name if supplied validations_store_bucket_name: Overrides default_bucket_name if supplied data_docs_bucket_name: Overrides default_bucket_name if supplied + checkpoint_store_bucket_name: Overrides default_bucket_name if supplied expectations_store_project_name: Overrides default_project_name if supplied validations_store_project_name: Overrides default_project_name if supplied data_docs_project_name: Overrides default_project_name if supplied + checkpoint_store_project_name: Overrides default_project_name if supplied expectations_store_prefix: Overrides default if supplied validations_store_prefix: Overrides default if supplied data_docs_prefix: Overrides default if supplied + checkpoint_store_prefix: Overrides default if supplied expectations_store_name: Overrides default if supplied validations_store_name: Overrides default if supplied evaluation_parameter_store_name: Overrides default if supplied + checkpoint_store_name: Overrides default if supplied """ def __init__( @@ -1241,15 +1265,19 @@ def __init__( expectations_store_bucket_name: Optional[str] = None, validations_store_bucket_name: Optional[str] = None, data_docs_bucket_name: Optional[str] = None, + checkpoint_store_bucket_name: Optional[str] = None, expectations_store_project_name: Optional[str] = None, validations_store_project_name: Optional[str] = None, data_docs_project_name: Optional[str] = None, + checkpoint_store_project_name: Optional[str] = None, expectations_store_prefix: str = "expectations", validations_store_prefix: str = "validations", data_docs_prefix: str = "data_docs", + checkpoint_store_prefix: str = "checkpoints", expectations_store_name: str = "expectations_GCS_store", validations_store_name: str = "validations_GCS_store", evaluation_parameter_store_name: str = "evaluation_parameter_store", + checkpoint_store_name: str = "checkpoint_GCS_store", ): # Initialize base defaults super().__init__() @@ -1261,6 +1289,8 @@ def __init__( validations_store_bucket_name = default_bucket_name if data_docs_bucket_name is None: data_docs_bucket_name = default_bucket_name + if checkpoint_store_bucket_name is None: + checkpoint_store_bucket_name = default_bucket_name # Use default_project_name if separate store projects are not provided if expectations_store_project_name is None: @@ -1269,11 +1299,14 @@ def __init__( validations_store_project_name = default_project_name if data_docs_project_name is None: data_docs_project_name = default_project_name + if checkpoint_store_project_name is None: + checkpoint_store_project_name = default_project_name # Overwrite defaults self.expectations_store_name = expectations_store_name self.validations_store_name = validations_store_name self.evaluation_parameter_store_name = evaluation_parameter_store_name + self.checkpoint_store_name = checkpoint_store_name self.stores = { expectations_store_name: { "class_name": "ExpectationsStore", @@ -1294,6 +1327,15 @@ def __init__( }, }, evaluation_parameter_store_name: {"class_name": "EvaluationParameterStore"}, + checkpoint_store_name: { + "class_name": "CheckpointStore", + "store_backend": { + "class_name": "TupleGCSStoreBackend", + "project": checkpoint_store_project_name, + "bucket": checkpoint_store_bucket_name, + "prefix": checkpoint_store_prefix, + }, + }, } self.data_docs_sites = { "gcs_site": { @@ -1319,9 +1361,11 @@ class DatabaseStoreBackendDefaults(BaseStoreBackendDefaults): default_credentials: Use these credentials for all stores that do not have credentials provided expectations_store_credentials: Overrides default_credentials if supplied validations_store_credentials: Overrides default_credentials if supplied + checkpoint_store_credentials: Overrides default_credentials if supplied expectations_store_name: Overrides default if supplied validations_store_name: Overrides default if supplied evaluation_parameter_store_name: Overrides default if supplied + checkpoint_store_name: Overrides default if supplied """ def __init__( @@ -1329,9 +1373,11 @@ def __init__( default_credentials: Optional[Dict] = None, expectations_store_credentials: Optional[Dict] = None, validations_store_credentials: Optional[Dict] = None, + checkpoint_store_credentials: Optional[Dict] = None, expectations_store_name: str = "expectations_database_store", validations_store_name: str = "validations_database_store", evaluation_parameter_store_name: str = "evaluation_parameter_store", + checkpoint_store_name: str = "checkpoint_database_store", ): # Initialize base defaults super().__init__() @@ -1341,11 +1387,14 @@ def __init__( expectations_store_credentials = default_credentials if validations_store_credentials is None: validations_store_credentials = default_credentials + if checkpoint_store_credentials is None: + checkpoint_store_credentials = default_credentials # Overwrite defaults self.expectations_store_name = expectations_store_name self.validations_store_name = validations_store_name self.evaluation_parameter_store_name = evaluation_parameter_store_name + self.checkpoint_store_name = checkpoint_store_name self.stores = { expectations_store_name: { @@ -1363,6 +1412,13 @@ def __init__( }, }, evaluation_parameter_store_name: {"class_name": "EvaluationParameterStore"}, + checkpoint_store_name: { + "class_name": "CheckpointStore", + "store_backend": { + "class_name": "DatabaseStoreBackend", + "credentials": checkpoint_store_credentials, + }, + }, } @@ -1412,6 +1468,8 @@ def __init__( ) if data_docs_sites is None: data_docs_sites = store_backend_defaults.data_docs_sites + if checkpoint_store_name is None: + checkpoint_store_name = store_backend_defaults.checkpoint_store_name self._config_version = config_version if datasources is None: diff --git a/tests/data_context/test_data_context_config_ui.py b/tests/data_context/test_data_context_config_ui.py index 19f820e45acb..5521ef22258e 100644 --- a/tests/data_context/test_data_context_config_ui.py +++ b/tests/data_context/test_data_context_config_ui.py @@ -175,7 +175,6 @@ def test_DataContextConfig_with_S3StoreBackendDefaults( ) }, store_backend_defaults=store_backend_defaults, - checkpoint_store_name=store_backend_defaults.checkpoint_store_name, ) # Create desired config @@ -197,6 +196,14 @@ def test_DataContextConfig_with_S3StoreBackendDefaults( "prefix": "validations", }, }, + "checkpoint_S3_store": { + "class_name": "CheckpointStore", + "store_backend": { + "bucket": "my_default_bucket", + "class_name": "TupleS3StoreBackend", + "prefix": "checkpoints", + }, + }, } desired_data_docs_sites_config = { "s3_site": { @@ -219,6 +226,7 @@ def test_DataContextConfig_with_S3StoreBackendDefaults( expectations_store_name="expectations_S3_store", validations_store_name="validations_S3_store", evaluation_parameter_store_name=DataContextConfigDefaults.DEFAULT_EVALUATION_PARAMETER_STORE_NAME.value, + checkpoint_store_name="checkpoint_S3_store", stores=desired_stores_config, data_docs_sites=desired_data_docs_sites_config, ) @@ -244,12 +252,15 @@ def test_DataContextConfig_with_S3StoreBackendDefaults_using_all_parameters( expectations_store_bucket_name="custom_expectations_store_bucket_name", validations_store_bucket_name="custom_validations_store_bucket_name", data_docs_bucket_name="custom_data_docs_store_bucket_name", + checkpoint_store_bucket_name="custom_checkpoint_store_bucket_name", expectations_store_prefix="custom_expectations_store_prefix", validations_store_prefix="custom_validations_store_prefix", data_docs_prefix="custom_data_docs_prefix", + checkpoint_store_prefix="custom_checkpoint_store_prefix", expectations_store_name="custom_expectations_S3_store_name", validations_store_name="custom_validations_S3_store_name", evaluation_parameter_store_name="custom_evaluation_parameter_store_name", + checkpoint_store_name="custom_checkpoint_S3_store_name", ) data_context_config = DataContextConfig( datasources={ @@ -269,7 +280,6 @@ def test_DataContextConfig_with_S3StoreBackendDefaults_using_all_parameters( ) }, store_backend_defaults=store_backend_defaults, - checkpoint_store_name=store_backend_defaults.checkpoint_store_name, ) # Create desired config @@ -293,6 +303,14 @@ def test_DataContextConfig_with_S3StoreBackendDefaults_using_all_parameters( "prefix": "custom_validations_store_prefix", }, }, + "custom_checkpoint_S3_store_name": { + "class_name": "CheckpointStore", + "store_backend": { + "bucket": "custom_checkpoint_store_bucket_name", + "class_name": "TupleS3StoreBackend", + "prefix": "custom_checkpoint_store_prefix", + }, + }, } desired_data_docs_sites_config = { "s3_site": { @@ -315,6 +333,7 @@ def test_DataContextConfig_with_S3StoreBackendDefaults_using_all_parameters( expectations_store_name="custom_expectations_S3_store_name", validations_store_name="custom_validations_S3_store_name", evaluation_parameter_store_name="custom_evaluation_parameter_store_name", + checkpoint_store_name="custom_checkpoint_S3_store_name", stores=desired_stores_config, data_docs_sites=desired_data_docs_sites_config, ) @@ -353,7 +372,6 @@ def test_DataContextConfig_with_FilesystemStoreBackendDefaults_and_simple_defaul ) }, store_backend_defaults=store_backend_defaults, - checkpoint_store_name=store_backend_defaults.checkpoint_store_name, ) # Create desired config @@ -368,6 +386,9 @@ def test_DataContextConfig_with_FilesystemStoreBackendDefaults_and_simple_defaul desired_config["stores"][desired_config["validations_store_name"]]["store_backend"][ "root_directory" ] = test_root_directory + desired_config["stores"][desired_config["checkpoint_store_name"]]["store_backend"][ + "root_directory" + ] = test_root_directory desired_config["data_docs_sites"]["local_site"]["store_backend"][ "root_directory" ] = test_root_directory @@ -450,7 +471,6 @@ def test_DataContextConfig_with_GCSStoreBackendDefaults( ) }, store_backend_defaults=store_backend_defaults, - checkpoint_store_name=store_backend_defaults.checkpoint_store_name, ) # Create desired config @@ -475,6 +495,15 @@ def test_DataContextConfig_with_GCSStoreBackendDefaults( "prefix": "validations", }, }, + "checkpoint_GCS_store": { + "class_name": "CheckpointStore", + "store_backend": { + "bucket": "my_default_bucket", + "project": "my_default_project", + "class_name": "TupleGCSStoreBackend", + "prefix": "checkpoints", + }, + }, } desired_data_docs_sites_config = { "gcs_site": { @@ -497,6 +526,7 @@ def test_DataContextConfig_with_GCSStoreBackendDefaults( datasources=default_pandas_datasource_config, expectations_store_name="expectations_GCS_store", validations_store_name="validations_GCS_store", + checkpoint_store_name="checkpoint_GCS_store", evaluation_parameter_store_name=DataContextConfigDefaults.DEFAULT_EVALUATION_PARAMETER_STORE_NAME.value, stores=desired_stores_config, data_docs_sites=desired_data_docs_sites_config, @@ -524,15 +554,19 @@ def test_DataContextConfig_with_GCSStoreBackendDefaults_using_all_parameters( expectations_store_bucket_name="custom_expectations_store_bucket_name", validations_store_bucket_name="custom_validations_store_bucket_name", data_docs_bucket_name="custom_data_docs_store_bucket_name", + checkpoint_store_bucket_name="custom_checkpoint_store_bucket_name", expectations_store_project_name="custom_expectations_store_project_name", validations_store_project_name="custom_validations_store_project_name", data_docs_project_name="custom_data_docs_store_project_name", + checkpoint_store_project_name="custom_checkpoint_store_project_name", expectations_store_prefix="custom_expectations_store_prefix", validations_store_prefix="custom_validations_store_prefix", data_docs_prefix="custom_data_docs_prefix", + checkpoint_store_prefix="custom_checkpoint_store_prefix", expectations_store_name="custom_expectations_GCS_store_name", validations_store_name="custom_validations_GCS_store_name", evaluation_parameter_store_name="custom_evaluation_parameter_store_name", + checkpoint_store_name="custom_checkpoint_GCS_store_name", ) data_context_config = DataContextConfig( datasources={ @@ -552,7 +586,6 @@ def test_DataContextConfig_with_GCSStoreBackendDefaults_using_all_parameters( ) }, store_backend_defaults=store_backend_defaults, - checkpoint_store_name=store_backend_defaults.checkpoint_store_name, ) # Create desired config @@ -578,6 +611,15 @@ def test_DataContextConfig_with_GCSStoreBackendDefaults_using_all_parameters( "prefix": "custom_validations_store_prefix", }, }, + "custom_checkpoint_GCS_store_name": { + "class_name": "CheckpointStore", + "store_backend": { + "bucket": "custom_checkpoint_store_bucket_name", + "project": "custom_checkpoint_store_project_name", + "class_name": "TupleGCSStoreBackend", + "prefix": "custom_checkpoint_store_prefix", + }, + }, } desired_data_docs_sites_config = { "gcs_site": { @@ -600,6 +642,7 @@ def test_DataContextConfig_with_GCSStoreBackendDefaults_using_all_parameters( expectations_store_name="custom_expectations_GCS_store_name", validations_store_name="custom_validations_GCS_store_name", evaluation_parameter_store_name="custom_evaluation_parameter_store_name", + checkpoint_store_name="custom_checkpoint_GCS_store_name", stores=desired_stores_config, data_docs_sites=desired_data_docs_sites_config, ) @@ -648,7 +691,6 @@ def test_DataContextConfig_with_DatabaseStoreBackendDefaults( ) }, store_backend_defaults=store_backend_defaults, - checkpoint_store_name=store_backend_defaults.checkpoint_store_name, ) # Create desired config @@ -682,6 +724,20 @@ def test_DataContextConfig_with_DatabaseStoreBackendDefaults( }, }, }, + "checkpoint_database_store": { + "class_name": "CheckpointStore", + "store_backend": { + "class_name": "DatabaseStoreBackend", + "credentials": { + "drivername": "postgresql", + "host": "localhost", + "port": "65432", + "username": "ge_tutorials", + "password": "ge_tutorials", + "database": "ge_tutorials", + }, + }, + }, } desired_data_docs_sites_config = { "local_site": { @@ -702,6 +758,7 @@ def test_DataContextConfig_with_DatabaseStoreBackendDefaults( datasources=default_pandas_datasource_config, expectations_store_name="expectations_database_store", validations_store_name="validations_database_store", + checkpoint_store_name="checkpoint_database_store", evaluation_parameter_store_name=DataContextConfigDefaults.DEFAULT_EVALUATION_PARAMETER_STORE_NAME.value, stores=desired_stores_config, data_docs_sites=desired_data_docs_sites_config, @@ -748,9 +805,18 @@ def test_DataContextConfig_with_DatabaseStoreBackendDefaults_using_all_parameter "password": "custom_validations_store_password", "database": "custom_validations_store_database", }, + checkpoint_store_credentials={ + "drivername": "custom_checkpoint_store_drivername", + "host": "custom_checkpoint_store_host", + "port": "custom_checkpoint_store_port", + "username": "custom_checkpoint_store_username", + "password": "custom_checkpoint_store_password", + "database": "custom_checkpoint_store_database", + }, expectations_store_name="custom_expectations_database_store_name", validations_store_name="custom_validations_database_store_name", evaluation_parameter_store_name="custom_evaluation_parameter_store_name", + checkpoint_store_name="custom_checkpoint_database_store_name", ) data_context_config = DataContextConfig( datasources={ @@ -770,7 +836,6 @@ def test_DataContextConfig_with_DatabaseStoreBackendDefaults_using_all_parameter ) }, store_backend_defaults=store_backend_defaults, - checkpoint_store_name=store_backend_defaults.checkpoint_store_name, ) # Create desired config @@ -806,6 +871,20 @@ def test_DataContextConfig_with_DatabaseStoreBackendDefaults_using_all_parameter }, }, }, + "custom_checkpoint_database_store_name": { + "class_name": "CheckpointStore", + "store_backend": { + "class_name": "DatabaseStoreBackend", + "credentials": { + "database": "custom_checkpoint_store_database", + "drivername": "custom_checkpoint_store_drivername", + "host": "custom_checkpoint_store_host", + "password": "custom_checkpoint_store_password", + "port": "custom_checkpoint_store_port", + "username": "custom_checkpoint_store_username", + }, + }, + }, } desired_data_docs_sites_config = { "local_site": { @@ -827,6 +906,7 @@ def test_DataContextConfig_with_DatabaseStoreBackendDefaults_using_all_parameter expectations_store_name="custom_expectations_database_store_name", validations_store_name="custom_validations_database_store_name", evaluation_parameter_store_name="custom_evaluation_parameter_store_name", + checkpoint_store_name="custom_checkpoint_database_store_name", stores=desired_stores_config, data_docs_sites=desired_data_docs_sites_config, ) @@ -1106,7 +1186,6 @@ def test_DataContextConfig_with_S3StoreBackendDefaults_and_simple_defaults_with_ ) }, store_backend_defaults=store_backend_defaults, - checkpoint_store_name=store_backend_defaults.checkpoint_store_name, ) # Create desired config @@ -1128,6 +1207,14 @@ def test_DataContextConfig_with_S3StoreBackendDefaults_and_simple_defaults_with_ "prefix": "validations", }, }, + "checkpoint_S3_store": { + "class_name": "CheckpointStore", + "store_backend": { + "bucket": "my_default_bucket", + "class_name": "TupleS3StoreBackend", + "prefix": "checkpoints", + }, + }, } desired_data_docs_sites_config = { "s3_site": { @@ -1149,6 +1236,7 @@ def test_DataContextConfig_with_S3StoreBackendDefaults_and_simple_defaults_with_ datasources=default_pandas_datasource_config, expectations_store_name="expectations_S3_store", validations_store_name="validations_S3_store", + checkpoint_store_name="checkpoint_S3_store", evaluation_parameter_store_name=DataContextConfigDefaults.DEFAULT_EVALUATION_PARAMETER_STORE_NAME.value, stores=desired_stores_config, data_docs_sites=desired_data_docs_sites_config,