Merge branch 'feature/GREAT-1009/remaining-migration-for-init' into f…

…eature/GREAT-954/data-assistant-logic-migration

* feature/GREAT-1009/remaining-migration-for-init:
  save_project_config_to_file
  docstrings and type hints
  refactor: remove datasource_name as arg to update_datasource (#5440)
  [MAINTENANCE] Test serialization of text, table, and bulleted list `rendered_content` in `ExpectationValidationResult` (#5438)
  DOC-322 (#5326)
  [FEATURE] Refactor Splitter Testing Modules so as to Make them More General and Add Unit and Integration Tests for "split_on_whole_table" and "split_on_column_value" on SQLite and All Supported Major SQL Backends (#5430)

docs/reference/expectations/standard_arguments.md

@@ -44,7 +44,7 @@ helpful markdown annotations to Expectations (shown below). These Expectation "n
 Expectation Suite pages in Data Docs.
 
 ```python
-my_df.expect_column_values_to_be_in_set(
+validator.expect_column_values_to_be_in_set(
     "my_column",
     ["a", "b", "c"],
     meta={

great_expectations/cli/upgrade_helpers/upgrade_helper_v13.py

@@ -360,7 +360,7 @@ def _upgrade_configuration_automatically(self) -> None:
                 DataContextConfig.from_commented_map(commented_map=config_commented_map)
             )
             self.data_context.set_config(project_config=data_context_config)
-            self.data_context._save_project_config()
+            self.data_context._save_project_config_to_disk()
 
             checkpoint_log_entry = {
                 "stores": {

great_expectations/cli/v012/upgrade_helpers/upgrade_helper_v13.py

@@ -104,7 +104,7 @@ def _upgrade_configuration(self):
         )
         self.data_context.set_config(project_config=data_context_config)
         # noinspection PyProtectedMember
-        self.data_context._save_project_config()
+        self.data_context._save_project_config_to_disk()
         self._update_upgrade_log()
 
     def _update_upgrade_log(self) -> None:

great_expectations/core/expectation_suite.py

@@ -210,7 +210,7 @@ def to_json_dict(self):
         myself["meta"] = convert_to_json_serializable(myself["meta"])
         return myself
 
-    def get_evaluation_parameter_dependencies(self):
+    def get_evaluation_parameter_dependencies(self) -> dict:
         dependencies = {}
         for expectation in self.expectations:
             t = expectation.get_evaluation_parameter_dependencies()

great_expectations/core/usage_statistics/anonymizers/anonymizer.py

@@ -149,7 +149,7 @@ def _anonymize_stores_init_payload(
         return anonymized_values
 
     def _anonymize_validation_operator_init_payload(
-        self, payload: Optional[Dict[str, "ValidationOperator"]]  # noqa: F821
+        self, payload: Optional[Dict[str, "ValidationOperator"]] = None  # noqa: F821
     ) -> List[dict]:
 
         from great_expectations.core.usage_statistics.anonymizers.validation_operator_anonymizer import (

great_expectations/data_context/data_context/abstract_data_context.py

@@ -115,15 +115,16 @@ class AbstractDataContext(ABC):
         + TEST_YAML_CONFIG_SUPPORTED_PROFILER_TYPES
     )
 
-    def __init__(self, runtime_environment: dict):
+    def __init__(self, runtime_environment: Optional[dict] = None) -> None:
         """
         Constructor for AbstractDataContext. Will handle instantiation logic that is common to all DataContext objects
 
         Args:
             runtime_environment (dict): a dictionary of config variables that
                 override both those set in config_variables.yml and the environment
         """
-        # TODO:
+        if runtime_environment is None:
+            runtime_environment = {}
         self.runtime_environment = runtime_environment
         # these attributes that are set downstream.
         self._variables = None
@@ -167,7 +168,13 @@ def _init_variables(self) -> DataContextVariables:
         raise NotImplementedError
 
     @abstractmethod
-    def _save_project_config(self) -> None:
+    def _save_project_config_to_disk(self) -> None:
+        """
+        Each DataContext will define how its project_config will be saved.
+            - FileDataContext : Filesystem.
+            - CloudDataContext : Cloud endpoint
+            - Ephemeral : not saved, and logging message outputted
+        """
         raise NotImplementedError
 
     @abstractmethod
@@ -179,6 +186,9 @@ def save_expectation_suite(
         ge_cloud_id: Optional[str] = None,
         **kwargs,
     ) -> None:
+        """
+        Each DataContext will define how ExpectationSuite will be saved.
+        """
         raise NotImplementedError
 
     # Properties
@@ -224,7 +234,7 @@ def project_config_with_variables_substituted(self) -> DataContextConfig:
         return self.get_config_with_variables_substituted()
 
     @property
-    def plugins_directory(self):
+    def plugins_directory(self) -> Optional[str]:
         """The directory in which custom plugin modules should be placed.
 
         Why does this exist in AbstractDataContext? CloudDataContext and FileDataContext both use it
@@ -234,7 +244,7 @@ def plugins_directory(self):
         )
 
     @property
-    def stores(self):
+    def stores(self) -> dict:
         """A single holder for all Stores in this context"""
         return self._stores
 
@@ -613,7 +623,7 @@ def _get_global_config_value(
             return os.environ.get(environment_variable)
         if conf_file_section and conf_file_option:
             for config_path in AbstractDataContext.GLOBAL_CONFIG_PATHS:
-                config = configparser.ConfigParser()
+                config: configparser.ConfigParser = configparser.ConfigParser()
                 config.read(config_path)
                 config_value: Optional[str] = config.get(
                     conf_file_section, conf_file_option, fallback=None
@@ -1125,7 +1135,9 @@ def _compile_evaluation_parameter_dependencies(self) -> None:
                 **expectation_suite_dict, data_context=self
             )
 
-            dependencies = expectation_suite.get_evaluation_parameter_dependencies()
+            dependencies: dict = (
+                expectation_suite.get_evaluation_parameter_dependencies()
+            )
             if len(dependencies) > 0:
                 nested_update(self._evaluation_parameter_dependencies, dependencies)
 

great_expectations/data_context/data_context/base_data_context.py

@@ -363,7 +363,7 @@ def _apply_temporary_overrides(self) -> None:
             self._data_context._evaluation_parameter_dependencies
         )
 
-    def _save_project_config(self) -> None:
+    def _save_project_config_to_disk(self) -> None:
         """Save the current project to disk."""
         logger.debug("Starting DataContext._save_project_config")
 
@@ -1458,20 +1458,19 @@ def add_datasource(
 
     def update_datasource(
         self,
-        datasource_name: str,
         datasource: Union[LegacyDatasource, BaseDatasource],
         save_changes: bool = False,
     ) -> None:
         """
         Updates a DatasourceConfig that already exists in the store.
 
         Args:
-            datasource_name: The name of the Datasource to update.
             datasource_config: The config object to persist using the DatasourceStore.
             save_changes: do I save changes to disk?
         """
         datasource_config_dict: dict = datasourceConfigSchema.dump(datasource.config)
         datasource_config: DatasourceConfig = DatasourceConfig(**datasource_config_dict)
+        datasource_name: str = datasource.name
 
         if save_changes:
             self._datasource_store.update_by_name(

great_expectations/data_context/data_context/cloud_data_context.py

@@ -84,7 +84,7 @@ def _construct_data_context_id(self) -> str:
         # if in ge_cloud_mode, use ge_cloud_organization_id
         return self.ge_cloud_config.organization_id
 
-    def _save_project_config(self) -> None:
+    def _save_project_config_to_disk(self) -> None:
         """Save the current project to disk."""
         if self.ge_cloud_mode:
             logger.debug(

great_expectations/data_context/data_context/data_context.py

@@ -368,7 +368,7 @@ def __init__(
             project_config.anonymous_usage_statistics.explicit_id is False
             or project_config_dict != dataContextConfigSchema.dump(self.config)
         ):
-            self._save_project_config()
+            self._save_project_config_to_disk()
 
     def _retrieve_data_context_config_from_ge_cloud(self) -> DataContextConfig:
         """
@@ -443,7 +443,7 @@ def add_store(self, store_name, store_config):
         logger.debug(f"Starting DataContext.add_store for store {store_name}")
 
         new_store = super().add_store(store_name, store_config)
-        self._save_project_config()
+        self._save_project_config_to_disk()
         return new_store
 
     def add_datasource(
@@ -454,32 +454,30 @@ def add_datasource(
         new_datasource: Optional[
             Union[LegacyDatasource, BaseDatasource]
         ] = super().add_datasource(name=name, **kwargs)
-        self._save_project_config()
+        self._save_project_config_to_disk()
         return new_datasource
 
     def update_datasource(
         self,
-        datasource_name: str,
         datasource: Union[LegacyDatasource, BaseDatasource],
     ) -> None:
         """
         See parent `BaseDataContext.update_datasource` for more details.
         Note that this method persists changes using an underlying Store.
         """
         logger.debug(
-            f"Starting DataContext.update_datasource for datasource {datasource_name}"
+            f"Starting DataContext.update_datasource for datasource {datasource.name}"
         )
 
         super().update_datasource(
-            datasource_name=datasource_name,
             datasource=datasource,
             save_changes=True,
         )
 
     def delete_datasource(self, name: str) -> None:
         logger.debug(f"Starting DataContext.delete_datasource for datasource {name}")
         super().delete_datasource(datasource_name=name)
-        self._save_project_config()
+        self._save_project_config_to_disk()
 
     @classmethod
     def find_context_root_dir(cls):

great_expectations/data_context/data_context/ephemeral_data_context.py

@@ -47,7 +47,7 @@ def __init__(
     def _init_variables(self) -> EphemeralDataContextVariables:
         raise NotImplementedError
 
-    def _save_project_config(self) -> None:
+    def _save_project_config_to_disk(self) -> None:
         """Since EphemeralDataContext does not have config as a file, display logging message instead"""
         logger.debug("EphemeralDataContext does not save project_config to file")
         return

great_expectations/data_context/data_context/file_data_context.py

@@ -52,7 +52,7 @@ def __init__(
     def _init_variables(self) -> FileDataContextVariables:
         raise NotImplementedError
 
-    def _save_project_config(self) -> None:
+    def _save_project_config_to_disk(self) -> None:
         """Save the current project to disk."""
         logger.debug("Starting DataContext._save_project_config")
 

great_expectations/data_context/store/inline_store_backend.py

@@ -78,7 +78,7 @@ def _get(self, key: Tuple[str, ...]) -> Any:
             return project_config
 
         try:
-            self._data_context._save_project_config()
+            self._data_context._save_project_config_to_disk()
         except AttributeError as e:
             logger.warning(
                 f"DataContext of type {type(self._data_context)} used with {self.__class__.__name__} is not file-system enabled: {e}"
@@ -201,7 +201,7 @@ def _validate_key(self, key: Tuple[str, ...]) -> None:
             )
 
     def _save_changes(self) -> None:
-        self._data_context._save_project_config()
+        self._data_context._save_project_config_to_disk()
 
     @staticmethod
     def _determine_resource_type_and_name_from_key(

tests/checkpoint/conftest.py

@@ -133,5 +133,5 @@ def titanic_spark_data_context_with_v013_datasource_with_checkpoints_v1_with_emp
         name="my_datasource", yaml_config=datasource_config, pretty_print=False
     )
     # noinspection PyProtectedMember
-    context._save_project_config()
+    context._save_project_config_to_disk()
     return context

tests/cli/conftest.py

@@ -30,7 +30,7 @@ def empty_context_with_checkpoint_v1_stats_enabled(
     checkpoints_file = os.path.join(root_dir, "checkpoints", fixture_name)
     shutil.copy(fixture_path, checkpoints_file)
     # # noinspection PyProtectedMember
-    context._save_project_config()
+    context._save_project_config_to_disk()
     return context
 
 

tests/cli/test_checkpoint.py

@@ -184,7 +184,7 @@ def titanic_data_context_with_spark_datasource(
         pretty_print=False,
     )
     # noinspection PyProtectedMember
-    context._save_project_config()
+    context._save_project_config_to_disk()
 
     csv_path: str
 

tests/cli/test_docs.py

@@ -221,7 +221,7 @@ def context_with_two_sites(titanic_data_context_stats_enabled_config_version_3):
     temp_context = BaseDataContext(config, context_root_dir=context.root_directory)
     new_context = DataContext(context.root_directory)
     new_context.set_config(temp_context.get_config_with_variables_substituted())
-    new_context._save_project_config()
+    new_context._save_project_config_to_disk()
     assert new_context.get_site_names() == ["local_site", "team_site"]
     return new_context
 

tests/cli/v012/test_store.py

@@ -9,7 +9,7 @@ def test_store_list_with_zero_stores(caplog, empty_data_context):
     project_dir = empty_data_context.root_directory
     context = DataContext(project_dir)
     context._project_config.stores = {}
-    context._save_project_config()
+    context._save_project_config_to_disk()
     runner = CliRunner(mix_stderr=False)
 
     result = runner.invoke(
@@ -35,7 +35,7 @@ def test_store_list_with_two_stores(caplog, empty_data_context):
     context._project_config.validations_store_name = "expectations_store"
     context._project_config.evaluation_parameter_store_name = "expectations_store"
     context._project_config.profiler_store_name = "profiler_store"
-    context._save_project_config()
+    context._save_project_config_to_disk()
 
     runner = CliRunner(mix_stderr=False)
 

tests/cli/v012/test_validation_operator.py

@@ -281,7 +281,7 @@ def test_validation_operator_list_with_zero_validation_operators(
     project_dir = empty_data_context.root_directory
     context = DataContext(project_dir)
     context._project_config.validation_operators = {}
-    context._save_project_config()
+    context._save_project_config_to_disk()
     runner = CliRunner(mix_stderr=False)
 
     result = runner.invoke(
@@ -356,7 +356,7 @@ def test_validation_operator_list_with_multiple_validation_operators(
             "slack_webhook": "https://hooks.slack.com/services/dummy",
         },
     )
-    context._save_project_config()
+    context._save_project_config_to_disk()
     expected_result = """Heads up! This feature is Experimental. It may change. Please give us your feedback!
 2 Validation Operators found:
 

tests/conftest.py

@@ -759,7 +759,7 @@ def titanic_pandas_data_context_with_v013_datasource_with_checkpoints_v1_with_em
         name="my_datasource", yaml_config=datasource_config, pretty_print=False
     )
     # noinspection PyProtectedMember
-    context._save_project_config()
+    context._save_project_config_to_disk()
 
     return context
 
@@ -922,7 +922,7 @@ def deterministic_asset_dataconnector_context(
         name="my_datasource", yaml_config=datasource_config, pretty_print=False
     )
     # noinspection PyProtectedMember
-    context._save_project_config()
+    context._save_project_config_to_disk()
     return context
 
 
@@ -1206,7 +1206,7 @@ def titanic_pandas_data_context_with_v013_datasource_stats_enabled_with_checkpoi
     )
 
     # noinspection PyProtectedMember
-    context._save_project_config()
+    context._save_project_config_to_disk()
     return context
 
 

tests/data_context/test_data_context.py

@@ -448,13 +448,9 @@ def test_update_datasource_persists_changes_with_store(
 ) -> None:
     context: DataContext = data_context_parameterized_expectation_suite
 
-    datasource_name: str
-    datasource_to_update: Datasource
-    datasource_name, datasource_to_update = tuple(context.datasources.items())[0]
+    datasource_to_update: Datasource = tuple(context.datasources.values())[0]
 
-    context.update_datasource(
-        datasource_name=datasource_name, datasource=datasource_to_update
-    )
+    context.update_datasource(datasource=datasource_to_update)
 
     assert mock_update_by_name.call_count == 1
 

tests/data_context/test_data_context_config_variables.py

@@ -244,7 +244,7 @@ def test_substituted_config_variables_not_written_to_file(tmp_path_factory):
 
     # instantiate data_context twice to go through cycle of loading config from file then saving
     context = ge.data_context.DataContext(context_path)
-    context._save_project_config()
+    context._save_project_config_to_disk()
     context_config_commented_map = dataContextConfigSchema.dump(
         ge.data_context.DataContext(context_path)._project_config
     )

tests/data_context/test_data_context_config_variables_pre_v013.py

@@ -47,7 +47,7 @@ def test_substituted_config_variables_not_written_to_file(tmp_path_factory):
 
     # instantiate data_context twice to go through cycle of loading config from file then saving
     context = ge.data_context.DataContext(context_path)
-    context._save_project_config()
+    context._save_project_config_to_disk()
     context_config_commented_map = dataContextConfigSchema.dump(
         ge.data_context.DataContext(context_path)._project_config
     )