Merge branch 'develop' into test-removal

great-expectations · Apr 30, 2024 · 5ba5f30 · 5ba5f30
2 parents 6050474 + c716838
commit 5ba5f30
Show file tree

Hide file tree

Showing 593 changed files with 7,277 additions and 4,769 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -18,7 +18,7 @@ repos:
       - id: detect-private-key
         exclude: tests/test_fixtures/database_key_test*
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.3.7"
+    rev: "v0.4.2"
     hooks:
       - id: ruff
         args: ["--fix"]

diff --git a/assets/scripts/gx_cloud/experimental/onboarding_script.py b/assets/scripts/gx_cloud/experimental/onboarding_script.py
@@ -26,7 +26,7 @@
 assert datasource_name, "Please set datasource_name."
 
 # Get or add Datasource
-datasource: Datasource = context.sources.add_or_update_pandas(datasource_name)
+datasource: Datasource = context.data_sources.add_or_update_pandas(datasource_name)
 
 # Provide an Asset name
 asset_name = None

diff --git a/...xpect_column_values_confidence_for_data_label_to_be_greater_than_or_equal_to_threshold.py b/...xpect_column_values_confidence_for_data_label_to_be_greater_than_or_equal_to_threshold.py
@@ -52,7 +52,7 @@ def _pandas(
             predict_options={"show_confidences": True},
         )
 
-        if data_label.upper() in labeler.label_mapping.keys():
+        if data_label.upper() in labeler.label_mapping:
             data_label_ind = labeler.label_mapping[data_label.upper()]
         else:
             raise ValueError(  # noqa: TRY003

diff --git a/...s/expect_column_values_confidence_for_data_label_to_be_less_than_or_equal_to_threshold.py b/...s/expect_column_values_confidence_for_data_label_to_be_less_than_or_equal_to_threshold.py
@@ -52,7 +52,7 @@ def _pandas(
             predict_options={"show_confidences": True},
         )
 
-        if data_label.upper() in labeler.label_mapping.keys():
+        if data_label.upper() in labeler.label_mapping:
             data_label_ind = labeler.label_mapping[data_label.upper()]
         else:
             raise ValueError(  # noqa: TRY003

diff --git a/.../tests/rule_based_profiler/data_assistant/test_data_profiler_structured_data_assistant.py b/.../tests/rule_based_profiler/data_assistant/test_data_profiler_structured_data_assistant.py
@@ -200,5 +200,5 @@ def test_profile_data_profiler_structured_data_assistant_result_batch_id_to_batc
             parameter_node[FULLY_QUALIFIED_PARAMETER_NAME_ATTRIBUTED_VALUE_KEY]
             if FULLY_QUALIFIED_PARAMETER_NAME_ATTRIBUTED_VALUE_KEY in parameter_node
             else {}
-        ).keys()
+        )
     )
diff --git a/contrib/cli/requirements.txt b/contrib/cli/requirements.txt
@@ -3,6 +3,6 @@ cookiecutter==2.1.1  # Project templating
 mypy==1.7.1            # Type checker
 pydantic>=1.0        # Needed for mypy plugin
 pytest>=5.3.5        # Test framework
-ruff==0.3.7        # Linting / code style / formatting
+ruff==0.4.2        # Linting / code style / formatting
 twine==3.7.1         # Packaging
 wheel==0.38.1        # Packaging
diff --git a/...at_expectations_experimental/expectations/expect_column_discrete_entropy_to_be_between.py b/...at_expectations_experimental/expectations/expect_column_discrete_entropy_to_be_between.py
@@ -285,9 +285,7 @@ def _prescriptive_renderer(
         **kwargs,
     ):
         runtime_configuration = runtime_configuration or {}
-        include_column_name = (
-            False if runtime_configuration.get("include_column_name") is False else True
-        )
+        include_column_name = runtime_configuration.get("include_column_name") is not False
         styling = runtime_configuration.get("styling")
         params = substitute_none_for_missing(
             configuration.kwargs,

diff --git a/...xpectations_experimental/expectations/expect_column_distribution_to_match_benfords_law.py b/...xpectations_experimental/expectations/expect_column_distribution_to_match_benfords_law.py
@@ -86,10 +86,7 @@ def _pandas(cls, column, **kwargs):
         for i in range(9):
             stat += ((listdata[i] - matchvalues[i]) ** 2) / (matchvalues[i])
 
-        if stat >= 5.071:
-            return False
-        else:
-            return True
+        return not stat >= 5.071
 
     # @metric_value(engine=SqlAlchemyExecutionEngine)
     # def _sqlalchemy(

diff --git a/...t_expectations_experimental/expectations/expect_column_values_after_split_to_be_in_set.py b/...t_expectations_experimental/expectations/expect_column_values_after_split_to_be_in_set.py
@@ -17,11 +17,7 @@
 def are_values_after_split_in_value_set(val: str, delimiter: str, value_set: List[str]) -> bool:
     all_split_values = [v.strip() for v in val.split(delimiter)]
 
-    for val in all_split_values:
-        if val not in value_set:
-            return False
-
-    return True
+    return all(val in value_set for val in all_split_values)
 
 
 # This class defines a Metric to support your Expectation.

diff --git a/...reat_expectations_experimental/expectations/expect_column_values_not_to_be_future_date.py b/...reat_expectations_experimental/expectations/expect_column_values_not_to_be_future_date.py
@@ -28,10 +28,7 @@ def is_not_a_future_date(date_in: str) -> bool:
         else:
             d = date_in
         d = d.date()
-        if d > today:
-            return False
-        else:
-            return True
+        return not d > today
     except Exception:
         return False
 

diff --git a/...great_expectations_experimental/expectations/expect_column_values_to_be_edtf_parseable.py b/...great_expectations_experimental/expectations/expect_column_values_to_be_edtf_parseable.py
@@ -390,9 +390,7 @@ def _prescriptive_renderer(
         **kwargs,
     ):
         runtime_configuration = runtime_configuration or {}
-        include_column_name = (
-            False if runtime_configuration.get("include_column_name") is False else True
-        )
+        include_column_name = runtime_configuration.get("include_column_name") is not False
         styling = runtime_configuration.get("styling")
         params = substitute_none_for_missing(
             configuration.kwargs,

diff --git a/...al/great_expectations_experimental/expectations/expect_column_values_to_be_valid_crc32.py b/...al/great_expectations_experimental/expectations/expect_column_values_to_be_valid_crc32.py
@@ -20,9 +20,7 @@ class ColumnValuesToBeValidCrc32(ColumnMapMetricProvider):
     @column_condition_partial(engine=PandasExecutionEngine)
     def _pandas(cls, column, **kwargs):
         def matches_crc32_regex(x):
-            if re.match(CRC32_REGEX, str(x)):
-                return True
-            return False
+            return bool(re.match(CRC32_REGEX, str(x)))
 
         return column.apply(lambda x: matches_crc32_regex(x) if x else False)
 

diff --git a/.../great_expectations_experimental/expectations/expect_column_values_to_match_xml_schema.py b/.../great_expectations_experimental/expectations/expect_column_values_to_match_xml_schema.py
@@ -152,9 +152,7 @@ def _prescriptive_renderer(
         **kwargs,
     ):
         runtime_configuration = runtime_configuration or {}
-        include_column_name = (
-            False if runtime_configuration.get("include_column_name") is False else True
-        )
+        include_column_name = runtime_configuration.get("include_column_name") is not False
         _ = runtime_configuration.get("styling")
         params = substitute_none_for_missing(
             configuration.kwargs,

diff --git a/...perimental/expectations/expect_column_values_to_not_be_null_and_column_to_not_be_empty.py b/...perimental/expectations/expect_column_values_to_not_be_null_and_column_to_not_be_empty.py
@@ -186,9 +186,7 @@ def _prescriptive_renderer(
         **kwargs,
     ):
         runtime_configuration = runtime_configuration or {}
-        include_column_name = (
-            False if runtime_configuration.get("include_column_name") is False else True
-        )
+        include_column_name = runtime_configuration.get("include_column_name") is not False
         styling = runtime_configuration.get("styling")
         params = substitute_none_for_missing(
             configuration.kwargs,

diff --git a/...tions_experimental/expectations/expect_column_values_to_not_contain_special_characters.py b/...tions_experimental/expectations/expect_column_values_to_not_contain_special_characters.py
@@ -48,10 +48,7 @@ def not_contain_special_character(val, *special_characters):
                 char for char in special_characters if char not in allowed_characters
             ]
 
-            for c in special_characters:
-                if c in str(val):
-                    return False
-            return True
+            return all(c not in str(val) for c in special_characters)
 
         return column.apply(not_contain_special_character, args=(list(string.punctuation)))
 
@@ -63,10 +60,7 @@ def not_contain_special_character(val, *special_characters):
                 char for char in list(string.punctuation) if char not in allowed_characters
             ]
 
-            for c in special_characters:
-                if c in str(val):
-                    return False
-            return True
+            return all(c not in str(val) for c in special_characters)
 
         # Register the UDF
         not_contain_special_character_udf = F.udf(
@@ -242,9 +236,7 @@ def _prescriptive_renderer(
         **kwargs,
     ):
         runtime_configuration = runtime_configuration or {}
-        include_column_name = (
-            False if runtime_configuration.get("include_column_name") is False else True
-        )
+        include_column_name = runtime_configuration.get("include_column_name") is not False
         styling = runtime_configuration.get("styling")
         params = substitute_none_for_missing(
             configuration.kwargs,

diff --git a/...ons_experimental/expectations/expect_day_count_to_be_close_to_equivalent_week_day_mean.py b/...ons_experimental/expectations/expect_day_count_to_be_close_to_equivalent_week_day_mean.py
@@ -272,7 +272,7 @@ def get_counts_per_day_as_dict(
     day_counts_dict = {i[0]: i[1] for i in counts_per_days}
 
     for day in all_days_list:
-        if day not in day_counts_dict.keys():
+        if day not in day_counts_dict:
             day_counts_dict.update({day: 0})
 
     return day_counts_dict

diff --git a/...tal/great_expectations_experimental/expectations/expect_multicolumn_values_to_be_equal.py b/...tal/great_expectations_experimental/expectations/expect_multicolumn_values_to_be_equal.py
@@ -268,9 +268,7 @@ def _prescriptive_renderer(
         **kwargs,
     ):
         runtime_configuration = runtime_configuration or {}
-        include_column_name = (
-            False if runtime_configuration.get("include_column_name") is False else True
-        )
+        include_column_name = runtime_configuration.get("include_column_name") is not False
         styling = runtime_configuration.get("styling")
         params = substitute_none_for_missing(
             configuration.kwargs,

diff --git a/...perimental/tests/rule_based_profiler/data_assistant/test_growth_numeric_data_assistant.py b/...perimental/tests/rule_based_profiler/data_assistant/test_growth_numeric_data_assistant.py
@@ -195,7 +195,7 @@ def test_growth_numeric_data_assistant_result_batch_id_to_batch_identifier_displ
             parameter_node[FULLY_QUALIFIED_PARAMETER_NAME_ATTRIBUTED_VALUE_KEY]
             if FULLY_QUALIFIED_PARAMETER_NAME_ATTRIBUTED_VALUE_KEY in parameter_node
             else {}
-        ).keys()
+        )
     )
 
 

diff --git a/...s_experimental/tests/rule_based_profiler/data_assistant/test_statistics_data_assistant.py b/...s_experimental/tests/rule_based_profiler/data_assistant/test_statistics_data_assistant.py
@@ -128,7 +128,7 @@ def test_statistics_data_assistant_result_batch_id_to_batch_identifier_display_n
             parameter_node[FULLY_QUALIFIED_PARAMETER_NAME_ATTRIBUTED_VALUE_KEY]
             if FULLY_QUALIFIED_PARAMETER_NAME_ATTRIBUTED_VALUE_KEY in parameter_node
             else {}
-        ).keys()
+        )
     )
 
 

diff --git a/...ons_ethical_ai_expectations/expectations/expect_table_linear_feature_importances_to_be.py b/...ons_ethical_ai_expectations/expectations/expect_table_linear_feature_importances_to_be.py
@@ -206,7 +206,7 @@ def _validate(
 
         if n_features:
             n_features_success = []
-            for i in importances.keys():
+            for i in importances:
                 if importances[i] >= threshold:
                     n_features_success.append(True)
             n_features_success = len(n_features_success) == int(n_features)

diff --git a/...tial_expectations/expectations/expect_column_average_to_be_within_range_of_given_point.py b/...tial_expectations/expectations/expect_column_average_to_be_within_range_of_given_point.py
@@ -171,9 +171,7 @@ def _prescriptive_renderer(
         ]
     ]:
         runtime_configuration = runtime_configuration or {}
-        include_column_name = (
-            False if runtime_configuration.get("include_column_name") is False else True
-        )
+        include_column_name = runtime_configuration.get("include_column_name") is not False
         styling = runtime_configuration.get("styling")
         params = substitute_none_for_missing(
             configuration.kwargs,

diff --git a/...tions_geospatial_expectations/expectations/expect_column_values_geometry_to_be_of_type.py b/...tions_geospatial_expectations/expectations/expect_column_values_geometry_to_be_of_type.py
@@ -161,9 +161,7 @@ def _prescriptive_renderer(
         ]
     ]:
         runtime_configuration = runtime_configuration or {}
-        include_column_name = (
-            False if runtime_configuration.get("include_column_name") is False else True
-        )
+        include_column_name = runtime_configuration.get("include_column_name") is not False
         styling = runtime_configuration.get("styling")
         params = substitute_none_for_missing(
             configuration.kwargs,

diff --git a/..._geospatial_expectations/expectations/expect_column_values_lat_lon_to_be_land_or_ocean.py b/..._geospatial_expectations/expectations/expect_column_values_lat_lon_to_be_land_or_ocean.py
@@ -167,9 +167,7 @@ def _prescriptive_renderer(
         ]
     ]:
         runtime_configuration = runtime_configuration or {}
-        include_column_name = (
-            False if runtime_configuration.get("include_column_name") is False else True
-        )
+        include_column_name = runtime_configuration.get("include_column_name") is not False
         styling = runtime_configuration.get("styling")
         params = substitute_none_for_missing(
             configuration.kwargs,

diff --git a/...ions_geospatial_expectations/expectations/expect_column_values_point_within_geo_region.py b/...ions_geospatial_expectations/expectations/expect_column_values_point_within_geo_region.py
@@ -273,9 +273,7 @@ def _prescriptive_renderer(
         **kwargs,
     ):
         runtime_configuration = runtime_configuration or {}
-        include_column_name = (
-            False if runtime_configuration.get("include_column_name") is False else True
-        )
+        include_column_name = runtime_configuration.get("include_column_name") is not False
         styling = runtime_configuration.get("styling")
         params = substitute_none_for_missing(
             configuration.kwargs,

diff --git a/...ial_expectations/expectations/expect_column_values_reverse_geocoded_lat_lon_to_contain.py b/...ial_expectations/expectations/expect_column_values_reverse_geocoded_lat_lon_to_contain.py
@@ -184,9 +184,7 @@ def _prescriptive_renderer(
         ]
     ]:
         runtime_configuration = runtime_configuration or {}
-        include_column_name = (
-            False if runtime_configuration.get("include_column_name") is False else True
-        )
+        include_column_name = runtime_configuration.get("include_column_name") is not False
         styling = runtime_configuration.get("styling")
         params = substitute_none_for_missing(
             configuration.kwargs,

diff --git a/...ns/expectations/expect_column_values_to_be_lat_lon_coordinates_in_range_of_given_point.py b/...ns/expectations/expect_column_values_to_be_lat_lon_coordinates_in_range_of_given_point.py
@@ -440,9 +440,7 @@ def _prescriptive_renderer(
         ]
     ]:
         runtime_configuration = runtime_configuration or {}
-        include_column_name = (
-            False if runtime_configuration.get("include_column_name") is False else True
-        )
+        include_column_name = runtime_configuration.get("include_column_name") is not False
         styling = runtime_configuration.get("styling")
         params = substitute_none_for_missing(
             configuration.kwargs,

diff --git a/...ns_geospatial_expectations/expectations/expect_column_values_to_be_nonempty_geometries.py b/...ns_geospatial_expectations/expectations/expect_column_values_to_be_nonempty_geometries.py
@@ -135,9 +135,7 @@ def _prescriptive_renderer(
         ]
     ]:
         runtime_configuration = runtime_configuration or {}
-        include_column_name = (
-            False if runtime_configuration.get("include_column_name") is False else True
-        )
+        include_column_name = runtime_configuration.get("include_column_name") is not False
         styling = runtime_configuration.get("styling")
         params = substitute_none_for_missing(
             configuration.kwargs,

diff --git a/...s_geospatial_expectations/expectations/expect_column_values_to_be_polygon_area_between.py b/...s_geospatial_expectations/expectations/expect_column_values_to_be_polygon_area_between.py
@@ -212,9 +212,7 @@ def _prescriptive_renderer(
         ]
     ]:
         runtime_configuration = runtime_configuration or {}
-        include_column_name = (
-            False if runtime_configuration.get("include_column_name") is False else True
-        )
+        include_column_name = runtime_configuration.get("include_column_name") is not False
         styling = runtime_configuration.get("styling")
         params = substitute_none_for_missing(
             configuration.kwargs,

diff --git a/..._expectations/expectations/expect_column_values_to_be_valid_degree_decimal_coordinates.py b/..._expectations/expectations/expect_column_values_to_be_valid_degree_decimal_coordinates.py
@@ -140,9 +140,7 @@ def _prescriptive_renderer(
         **kwargs,
     ):
         runtime_configuration = runtime_configuration or {}
-        include_column_name = (
-            False if runtime_configuration.get("include_column_name") is False else True
-        )
+        include_column_name = runtime_configuration.get("include_column_name") is not False
         styling = runtime_configuration.get("styling")
         params = substitute_none_for_missing(
             configuration.kwargs,

diff --git a/...ctations_geospatial_expectations/expectations/expect_column_values_to_be_valid_geojson.py b/...ctations_geospatial_expectations/expectations/expect_column_values_to_be_valid_geojson.py
@@ -151,9 +151,7 @@ def _prescriptive_renderer(
         ]
     ]:
         runtime_configuration = runtime_configuration or {}
-        include_column_name = (
-            False if runtime_configuration.get("include_column_name") is False else True
-        )
+        include_column_name = runtime_configuration.get("include_column_name") is not False
         styling = runtime_configuration.get("styling")
         params = substitute_none_for_missing(
             configuration.kwargs,

diff --git a/...ns_semantic_types_expectations/expectations/expect_column_values_bic_belong_to_country.py b/...ns_semantic_types_expectations/expectations/expect_column_values_bic_belong_to_country.py
@@ -21,10 +21,7 @@
 def bic_belong_to_country(bic: str, country_code: str) -> bool:
     try:
         bic = schwifty.BIC(bic)
-        if bic.country_code.upper() == country_code.upper():
-            return True
-        else:
-            return False
+        return bic.country_code.upper() == country_code.upper()
     except SchwiftyException:
         return False
 

diff --git a/..._types_expectations/expectations/expect_column_values_bitcoin_address_positive_balance.py b/..._types_expectations/expectations/expect_column_values_bitcoin_address_positive_balance.py
@@ -20,10 +20,7 @@ def has_btc_address_positive_balance(addr: str) -> bool:
         res = coinaddrvalidator.validate("btc", addr).valid
         if res is True:
             balance = blockcypher.get_total_balance(addr)
-            if balance > 0:
-                return True
-            else:
-                return False
+            return balance > 0
         else:
             return False
     except Exception:

diff --git a/..._semantic_types_expectations/expectations/expect_column_values_bitcoin_tx_is_confirmed.py b/..._semantic_types_expectations/expectations/expect_column_values_bitcoin_tx_is_confirmed.py
@@ -17,10 +17,7 @@
 def is_btc_tx_confirmed(tx: str) -> bool:
     try:
         cnt = blockcypher.get_num_confirmations(tx)
-        if cnt > 0:
-            return True
-        else:
-            return False
+        return cnt > 0
     except Exception as e:
         print(e)
         return False

diff --git a/...ic_types_expectations/expectations/expect_column_values_email_domain_is_not_disposable.py b/...ic_types_expectations/expectations/expect_column_values_email_domain_is_not_disposable.py
@@ -15,10 +15,7 @@
 
 
 def is_not_disposable(email_address: str) -> bool:
-    if email_address in blocklist:
-        return False
-    else:
-        return True
+    return email_address not in blocklist
 
 
 # This class defines a Metric to support your Expectation.

diff --git a/...ntic_types_expectations/expectations/expect_column_values_eth_address_positive_balance.py b/...ntic_types_expectations/expectations/expect_column_values_eth_address_positive_balance.py
@@ -20,10 +20,7 @@ def has_eth_address_positive_balance(addr: str) -> bool:
         res = coinaddrvalidator.validate("eth", addr).valid
         if res is True:
             balance = blockcypher.get_total_balance(addr, "eth")
-            if balance > 0:
-                return True
-            else:
-                return False
+            return balance > 0
         else:
             return False
     except Exception: