[BUGFIX] Remove spark from bic Expectations since it never worked for…

… them (#7619)
great-expectations · Apr 13, 2023 · 1e43593 · 1e43593
1 parent 4aec193
commit 1e43593
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 16 deletions.
diff --git a/...ns_semantic_types_expectations/expectations/expect_column_values_bic_belong_to_country.py b/...ns_semantic_types_expectations/expectations/expect_column_values_bic_belong_to_country.py
@@ -7,14 +7,12 @@
 from great_expectations.core.expectation_configuration import ExpectationConfiguration
 from great_expectations.execution_engine import (
     PandasExecutionEngine,
-    SparkDFExecutionEngine,
 )
 from great_expectations.expectations.expectation import ColumnMapExpectation
 from great_expectations.expectations.metrics import (
     ColumnMapMetricProvider,
     column_condition_partial,
 )
-from great_expectations.expectations.metrics.import_manager import F, sparktypes
 
 
 def bic_belong_to_country(bic: str, country_code: str) -> bool:
@@ -36,13 +34,13 @@ class ColumnValuesBicBelongToCountry(ColumnMapMetricProvider):
     def _pandas(cls, column, country_code, **kwargs):
         return column.apply(partial(bic_belong_to_country, country_code=country_code))
 
-    @column_condition_partial(engine=SparkDFExecutionEngine)
-    def _spark(cls, column, country_code, **kwargs):
-        @F.udf(sparktypes.BooleanType())
-        def bic_belong_to_country_udf(bic: str) -> bool:
-            return bic_belong_to_country(bic, country_code)
+    # @column_condition_partial(engine=SparkDFExecutionEngine)
+    # def _spark(cls, column, country_code, **kwargs):
+    #     @F.udf(sparktypes.BooleanType())
+    #     def bic_belong_to_country_udf(bic: str) -> bool:
+    #         return bic_belong_to_country(bic, country_code)
 
-        return bic_belong_to_country_udf(column)
+    #     return bic_belong_to_country_udf(column)
 
     # This method defines the business logic for evaluating your metric when using a SqlAlchemyExecutionEngine
     # @column_condition_partial(engine=SqlAlchemyExecutionEngine)

diff --git a/...ctations_semantic_types_expectations/expectations/expect_column_values_to_be_valid_bic.py b/...ctations_semantic_types_expectations/expectations/expect_column_values_to_be_valid_bic.py
@@ -6,14 +6,14 @@
 from great_expectations.core.expectation_configuration import ExpectationConfiguration
 from great_expectations.execution_engine import (
     PandasExecutionEngine,
-    SparkDFExecutionEngine,
 )
 from great_expectations.expectations.expectation import ColumnMapExpectation
 from great_expectations.expectations.metrics import (
     ColumnMapMetricProvider,
     column_condition_partial,
 )
-from great_expectations.expectations.metrics.import_manager import F, sparktypes
+
+# from great_expectations.expectations.metrics.import_manager import F, sparktypes
 
 
 def is_valid_bic(bic_code: str) -> bool:
@@ -24,9 +24,9 @@ def is_valid_bic(bic_code: str) -> bool:
         return False
 
 
-@F.udf(sparktypes.BooleanType())
-def is_valid_bic_udf(bic: str) -> bool:
-    return is_valid_bic(bic)
+# @F.udf(sparktypes.BooleanType())
+# def is_valid_bic_udf(bic: str) -> bool:
+#     return is_valid_bic(bic)
 
 
 class ColumnValuesToBeValidBic(ColumnMapMetricProvider):
@@ -36,9 +36,9 @@ class ColumnValuesToBeValidBic(ColumnMapMetricProvider):
     def _pandas(cls, column, **kwargs):
         return column.apply(is_valid_bic)
 
-    @column_condition_partial(engine=SparkDFExecutionEngine)
-    def _spark(cls, column, **kwargs):
-        return is_valid_bic_udf(column)
+    # @column_condition_partial(engine=SparkDFExecutionEngine)
+    # def _spark(cls, column, **kwargs):
+    #     return is_valid_bic_udf(column)
 
     # This method defines the business logic for evaluating your metric when using a SqlAlchemyExecutionEngine
     # @column_condition_partial(engine=SqlAlchemyExecutionEngine)