[MAINTENANCE] Make BatchData Serialization More Robust (#3791)

great_expectations/core/batch.py

@@ -838,16 +838,19 @@ def get_runtime_parameters_batch_data_references_from_config(
     if (
         ("batch_request" in config or hasattr(config, "batch_request"))
         and config["batch_request"] is not None
-        and config["batch_request"].get("runtime_parameters") is not None
-        and config["batch_request"]["runtime_parameters"].get("batch_data") is not None
+        and "runtime_parameters" in config["batch_request"]
+        and config["batch_request"]["runtime_parameters"] is not None
+        and "batch_data" in config["batch_request"]["runtime_parameters"]
+        and config["batch_request"]["runtime_parameters"]["batch_data"] is not None
         and default_batch_data is None
     ):
         default_batch_data = config["batch_request"]["runtime_parameters"]["batch_data"]
     else:
         if (
             ("runtime_parameters" in config or hasattr(config, "runtime_parameters"))
             and config["runtime_parameters"] is not None
-            and config["runtime_parameters"].get("batch_data") is not None
+            and "batch_data" in config["runtime_parameters"]
+            and config["runtime_parameters"]["batch_data"] is not None
             and default_batch_data is None
         ):
             default_batch_data = config["runtime_parameters"]["batch_data"]
@@ -858,10 +861,12 @@ def get_runtime_parameters_batch_data_references_from_config(
         validations_batch_data_list = []
         for val in config["validations"]:
             if (
-                val.get("batch_request") is not None
-                and val["batch_request"].get("runtime_parameters") is not None
-                and val["batch_request"]["runtime_parameters"].get("batch_data")
-                is not None
+                "batch_request" in val
+                and val["batch_request"] is not None
+                and "runtime_parameters" in val["batch_request"]
+                and val["batch_request"]["runtime_parameters"] is not None
+                and "batch_data" in val["batch_request"]["runtime_parameters"]
+                and val["batch_request"]["runtime_parameters"]["batch_data"] is not None
             ):
                 validations_batch_data_list.append(
                     val["batch_request"]["runtime_parameters"]["batch_data"]
@@ -885,30 +890,41 @@ def delete_runtime_parameters_batch_data_references_from_config(
     if (
         ("batch_request" in config or hasattr(config, "batch_request"))
         and config["batch_request"] is not None
-        and config["batch_request"].get("runtime_parameters") is not None
+        and "runtime_parameters" in config["batch_request"]
+        and config["batch_request"]["runtime_parameters"] is not None
         and "batch_data" in config["batch_request"]["runtime_parameters"]
     ):
-        config["batch_request"]["runtime_parameters"].pop("batch_data")
+        if hasattr(config["batch_request"]["runtime_parameters"], "pop"):
+            config["batch_request"]["runtime_parameters"].pop("batch_data")
+        else:
+            delattr(config["batch_request"]["runtime_parameters"], "batch_data")
     else:
         if (
             ("runtime_parameters" in config or hasattr(config, "runtime_parameters"))
             and config["runtime_parameters"] is not None
             and "batch_data" in config["runtime_parameters"]
         ):
-            config["runtime_parameters"].pop("batch_data")
+            if hasattr(config["runtime_parameters"], "pop"):
+                config["runtime_parameters"].pop("batch_data")
+            else:
+                delattr(config["runtime_parameters"], "batch_data")
 
     if ("validations" in config or hasattr(config, "validations")) and config[
         "validations"
     ]:
         for val in config["validations"]:
             if (
-                val.get("batch_request") is not None
-                and val["batch_request"].get("runtime_parameters") is not None
-                and "batch_data"
-                in val["batch_request"]["runtime_parameters"]
-                is not None
+                "batch_request" in val
+                and val["batch_request"] is not None
+                and "runtime_parameters" in val["batch_request"]
+                and val["batch_request"]["runtime_parameters"] is not None
+                and "batch_data" in val["batch_request"]["runtime_parameters"]
+                and val["batch_request"]["runtime_parameters"]["batch_data"] is not None
             ):
-                val["batch_request"]["runtime_parameters"].pop("batch_data")
+                if hasattr(val["batch_request"]["runtime_parameters"], "pop"):
+                    val["batch_request"]["runtime_parameters"].pop("batch_data")
+                else:
+                    delattr(val["batch_request"]["runtime_parameters"], "batch_data")
 
 
 def restore_runtime_parameters_batch_data_references_into_config(
@@ -929,7 +945,8 @@ def restore_runtime_parameters_batch_data_references_into_config(
     if (
         ("batch_request" in config or hasattr(config, "batch_request"))
         and config["batch_request"] is not None
-        and config["batch_request"].get("runtime_parameters") is not None
+        and "runtime_parameters" in config["batch_request"]
+        and config["batch_request"]["runtime_parameters"] is not None
         and default_batch_data is not None
     ):
         if replace_value_with_type_string:
@@ -960,9 +977,14 @@ def restore_runtime_parameters_batch_data_references_into_config(
     ):
         for idx, val in enumerate(config["validations"]):
             if (
-                val.get("batch_request") is not None
-                and val["batch_request"].get("runtime_parameters") is not None
-                and val["batch_request"]["runtime_parameters"].get("batch_data") is None
+                "batch_request" in val
+                and val["batch_request"] is not None
+                and "runtime_parameters" in val["batch_request"]
+                and val["batch_request"]["runtime_parameters"] is not None
+                and (
+                    "batch_data" not in val["batch_request"]["runtime_parameters"]
+                    or val["batch_request"]["runtime_parameters"]["batch_data"] is None
+                )
                 and validations_batch_data_list[idx] is not None
             ):
                 if replace_value_with_type_string:

great_expectations/types/__init__.py

@@ -13,6 +13,12 @@ def __setitem__(self, key, value):
     def __delitem__(self, key):
         delattr(self, key)
 
+    def __contains__(self, key):
+        return key in self.__dict__
+
+    def __len__(self):
+        return len(self.__dict__)
+
 
 class SerializableDictDot(DictDot):
     def to_json_dict(self) -> dict:

tests/datasource/data_connector/test_inferred_asset_azure_data_connector.py

@@ -1288,7 +1288,6 @@ def test_return_all_batch_definitions_too_many_sorters(
 @mock.patch(
     "great_expectations.core.usage_statistics.usage_statistics.UsageStatisticsHandler.emit"
 )
-# TODO: <Alex>ALEX -- Remove this test once dependency of "DataConnector._get_full_file_path()" on ExecutionEngine type is eliminated  dependency of "DataConnector._get_full_file_path()" on ExecutionEngine type is eliminated.</Alex>
 def test_get_full_file_path_pandas(
     mock_azure_conn, mock_list_keys, mock_emit, empty_data_context_stats_enabled
 ):
@@ -1363,7 +1362,6 @@ def test_get_full_file_path_pandas(
 @mock.patch(
     "great_expectations.core.usage_statistics.usage_statistics.UsageStatisticsHandler.emit"
 )
-# TODO: <Alex>ALEX -- Remove this test once dependency of "DataConnector._get_full_file_path()" on ExecutionEngine type is eliminated  dependency of "DataConnector._get_full_file_path()" on ExecutionEngine type is eliminated.</Alex>
 def test_get_full_file_path_spark(
     mock_azure_conn,
     mock_list_keys,
@@ -1442,7 +1440,6 @@ def test_get_full_file_path_spark(
 @mock.patch(
     "great_expectations.core.usage_statistics.usage_statistics.UsageStatisticsHandler.emit"
 )
-# TODO: <Alex>ALEX -- Remove this test once dependency of "DataConnector._get_full_file_path()" on ExecutionEngine type is eliminated  dependency of "DataConnector._get_full_file_path()" on ExecutionEngine type is eliminated.</Alex>
 def test_get_full_file_path_bad_execution_engine(
     mock_azure_conn, mock_list_keys, mock_emit, empty_data_context_stats_enabled
 ):