[BUGFIX] Enables successful parsing of test cases for multi-table exp…

…ectations (#4906)

* Remove pointless _generate_expectation_tests wrapper method and update docstring on generate_expectation_tests

* Remove accepting 'return_only_gallery_example's arg from run_diagnostics method

* Update build_gallery.py script to receive --no-core --no-contrib and arbitrary Expectation list

* Use phrase 'Has a valid library_metadata object'

* Update ExpectationTestDiagnostics to have include_in_gallery

* Update _get_metric_list to accept expectation_config instead of executed_test_cases

* Update ExpectationTestDiagnostics to include validation_result and error_diagnostics

* Delete _execute_test_examples, _choose_example, _instantiate_example_validation_results, and ExecutedExpectationTestCase

* Reformat with black

* Update run_diagnostics to determine maturity level based on checks passed

* Update evaluate_json_test_cfe to accept raise_exception and return a tuple

* Update _get_test_results to include more in ExpectationErrorDiagnostics via evaluate_json_test_cfe

* Add backend_test_result_counts to ExpectationDiagnostics and use in helpers

* Reformat with black

* Remove unused imports (flake8)

* Fix fix tests

* Update asserts at end of creating_custom_expectations/expect_xxx.py

* Add some print statements to generate_expectation_tests when get_test_validator_with_data has a problem

* test setup

* fixes diagnostics for multi-table expectations

* wrap tmp_dir -> abspath in func

* apply  to test_expectations/test_expectations_cfe

* docstring

Co-authored-by: Ken Wade <ken@superconductive.com>
Co-authored-by: kenwade4 <95714847+kenwade4@users.noreply.github.com>

great_expectations/self_check/util.py

@@ -273,6 +273,7 @@
     mssqlDialect = None
     MSSQL_TYPES = {}
 
+import tempfile
 
 SQL_DIALECT_NAMES = (
     "sqlite",
@@ -1580,10 +1581,26 @@ def generate_expectation_tests(
 
         for c in backends:
 
+            datasets = []
+
             try:
-                validator_with_data = get_test_validator_with_data(
-                    c, d["data"], d["schemas"]
-                )
+                if isinstance(d["data"], list):
+                    sqlite_db_path = generate_sqlite_db_path()
+                    for dataset in d["data"]:
+                        datasets.append(
+                            get_test_validator_with_data(
+                                c,
+                                dataset["data"],
+                                dataset.get("schemas"),
+                                table_name=dataset.get("dataset_name"),
+                                sqlite_db_path=sqlite_db_path,
+                            )
+                        )
+                    validator_with_data = datasets[0]
+                else:
+                    validator_with_data = get_test_validator_with_data(
+                        c, d["data"], d["schemas"]
+                    )
             except Exception as e:
                 # Adding these print statements for build_gallery.py's console output
                 print("\n\n[[ Problem calling get_test_validator_with_data ]]")
@@ -2052,3 +2069,25 @@ def _bigquery_dataset() -> str:
             "Environment Variable GE_TEST_BIGQUERY_DATASET is required to run BigQuery expectation tests"
         )
     return dataset
+
+
+def generate_sqlite_db_path():
+    """Creates a temporary directory and absolute path to an ephemeral sqlite_db within that temp directory.
+
+    Used to support testing of multi-table expectations without creating temp directories at import.
+
+    Returns:
+        str: An absolute path to the ephemeral db within the created temporary directory.
+    """
+    tmp_dir = str(tempfile.mkdtemp())
+    abspath = os.path.abspath(
+        os.path.join(
+            tmp_dir,
+            "sqlite_db"
+            + "".join(
+                [random.choice(string.ascii_letters + string.digits) for _ in range(8)]
+            )
+            + ".db",
+        )
+    )
+    return abspath

tests/test_definitions/test_expectations.py

@@ -4,7 +4,6 @@
 import os
 import random
 import string
-import tempfile
 from collections import OrderedDict
 
 import pandas as pd
@@ -15,6 +14,7 @@
     BigQueryDialect,
     candidate_test_is_on_temporary_notimplemented_list,
     evaluate_json_test,
+    generate_sqlite_db_path,
     get_dataset,
     mssqlDialect,
     mysqlDialect,
@@ -24,7 +24,6 @@
 from tests.conftest import build_test_backends_list
 
 logger = logging.getLogger(__name__)
-tmp_dir = str(tempfile.mkdtemp())
 
 
 def pytest_generate_tests(metafunc):
@@ -74,21 +73,7 @@ def pytest_generate_tests(metafunc):
                     else:
                         skip_expectation = False
                         if isinstance(d["data"], list):
-                            sqlite_db_path = os.path.abspath(
-                                os.path.join(
-                                    tmp_dir,
-                                    "sqlite_db"
-                                    + "".join(
-                                        [
-                                            random.choice(
-                                                string.ascii_letters + string.digits
-                                            )
-                                            for _ in range(8)
-                                        ]
-                                    )
-                                    + ".db",
-                                )
-                            )
+                            sqlite_db_path = generate_sqlite_db_path()
                             for dataset in d["data"]:
                                 datasets.append(
                                     get_dataset(

tests/test_definitions/test_expectations_cfe.py

@@ -16,14 +16,14 @@
     BigQueryDialect,
     candidate_test_is_on_temporary_notimplemented_list_cfe,
     evaluate_json_test_cfe,
+    generate_sqlite_db_path,
     get_test_validator_with_data,
     mssqlDialect,
     mysqlDialect,
     postgresqlDialect,
     sqliteDialect,
 )
 from tests.conftest import build_test_backends_list_cfe
-from tests.test_definitions.test_expectations import tmp_dir
 
 
 def pytest_generate_tests(metafunc):
@@ -71,21 +71,7 @@ def pytest_generate_tests(metafunc):
                     else:
                         skip_expectation = False
                         if isinstance(d["data"], list):
-                            sqlite_db_path = os.path.abspath(
-                                os.path.join(
-                                    tmp_dir,
-                                    "sqlite_db"
-                                    + "".join(
-                                        [
-                                            random.choice(
-                                                string.ascii_letters + string.digits
-                                            )
-                                            for _ in range(8)
-                                        ]
-                                    )
-                                    + ".db",
-                                )
-                            )
+                            sqlite_db_path = generate_sqlite_db_path()
                             for dataset in d["data"]:
                                 datasets.append(
                                     get_test_validator_with_data(