Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[MAINTENANCE] enable TRYceratops linting rules #9684

Merged
merged 6 commits into from
Apr 1, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
20 changes: 10 additions & 10 deletions assets/scripts/build_gallery.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ def execute_shell_command(command: str) -> int:
exception_message += (
f'{type(cpe).__name__}: "{cpe!s}". Traceback: "{exception_traceback}".'
)
logger.error(exception_message)
logger.error(exception_message) # noqa: TRY400

return status_code

Expand Down Expand Up @@ -296,7 +296,7 @@ def get_expectation_instances(expectations_info): # noqa: C901 - too complex
try:
importlib.import_module(*import_module_args)
except (ModuleNotFoundError, ImportError, Exception):
logger.error(f"Failed to load expectation_name: {expectation_name}")
logger.error(f"Failed to load expectation_name: {expectation_name}") # noqa: TRY400
print(traceback.format_exc())
expectation_tracebacks.write(
f"\n\n----------------\n{expectation_name} ({expectations_info[expectation_name]['package']})\n"
Expand All @@ -311,7 +311,7 @@ def get_expectation_instances(expectations_info): # noqa: C901 - too complex
expectation = create_expectation_instance(expectation_class)
expectation_instances[expectation_name] = expectation
except ExpectationNotFoundError:
logger.error(
logger.error( # noqa: TRY400
f"Failed to get Expectation implementation from registry: {expectation_name}"
)
print(traceback.format_exc())
Expand Down Expand Up @@ -348,7 +348,7 @@ def create_expectation_instance(expectation_class: type[Expectation]) -> Expecta
).resolve()
found = next(_TEST_DEFS_DIR.rglob(f"**/{expectation_class.expectation_type}.json"), None)
if not found:
raise Exception(
raise Exception( # noqa: TRY002, TRY003
f"No JSON test case found for Expectation {expectation_class.expectation_type}, cannot build gallery example."
)
with open(found) as fp:
Expand All @@ -362,9 +362,9 @@ def get_success_test_case(tests: list[dict]) -> dict:
try:
return next(test["in"] for test in tests if test["out"]["success"] is True)
except StopIteration:
raise ValueError("Error: Expectation test case has no valid success case.")
raise ValueError("Error: Expectation test case has no valid success case.") # noqa: TRY003
except IndexError:
raise ValueError("Error: Expectation test case is malformed.")
raise ValueError("Error: Expectation test case is malformed.") # noqa: TRY003


def combine_backend_results( # noqa: C901 - too complex
Expand Down Expand Up @@ -399,7 +399,7 @@ def combine_backend_results( # noqa: C901 - too complex
try:
diagnostic_object = diagnostic_objects[expectation_name]
except KeyError:
logger.error(f"No diagnostic obj for {expectation_name}")
logger.error(f"No diagnostic obj for {expectation_name}") # noqa: TRY400
bad_key_names.append(expectation_name)
continue
expectation_instance = expectation_instances[expectation_name]
Expand All @@ -411,7 +411,7 @@ def combine_backend_results( # noqa: C901 - too complex
]
]
except KeyError:
logger.error(f"No backend_test_result_counts for {expectation_name}")
logger.error(f"No backend_test_result_counts for {expectation_name}") # noqa: TRY400
bad_key_names.append(expectation_name)
continue
maturity_checklist_object = expectation_instance._get_maturity_checklist(
Expand Down Expand Up @@ -578,7 +578,7 @@ def build_gallery( # noqa: C901 - 17
expectation_docstrings.write("\n" + "." * 80 + "\n\n")
expectation_docstrings.write(f"{diagnostics['description']['docstring']}\n")
except Exception:
logger.error(f"Failed to run diagnostics for: {expectation_name}")
logger.error(f"Failed to run diagnostics for: {expectation_name}") # noqa: TRY400
print(traceback.format_exc())
expectation_tracebacks.write(
f"\n\n----------------\n{expectation_name} ({expectations_info[expectation_name]['package']})\n"
Expand Down Expand Up @@ -610,7 +610,7 @@ def build_gallery( # noqa: C901 - 17
}

except TypeError:
logger.error(f"Failed to create JSON for: {expectation_name}")
logger.error(f"Failed to create JSON for: {expectation_name}") # noqa: TRY400
print(traceback.format_exc())
expectation_tracebacks.write(
f"\n\n----------------\n[JSON write fail] {expectation_name} ({expectations_info[expectation_name]['package']})\n"
Expand Down
2 changes: 1 addition & 1 deletion assets/scripts/build_package_gallery.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ def gather_all_package_manifests(package_paths: List[str]) -> List[dict]:
f"Successfully serialized {package.package_name} to dict and appended to manifest list" # noqa: E501
)
except Exception as e:
logger.error(f"Something went wrong when syncing {path} and serializing to dict: {e}")
logger.error(f"Something went wrong when syncing {path} and serializing to dict: {e}") # noqa: TRY400
finally:
# Always ensure we revert back to the project root
os.chdir(root)
Expand Down
6 changes: 3 additions & 3 deletions ci/checks/check_integration_test_gets_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@
def check_dependencies(*deps: str) -> None:
for dep in deps:
if not shutil.which(dep):
raise Exception(f"Must have `{dep}` installed in PATH to run {__file__}")
raise Exception(f"Must have `{dep}` installed in PATH to run {__file__}") # noqa: TRY002, TRY003


def get_test_files(target_dir: pathlib.Path) -> Set[str]:
Expand Down Expand Up @@ -98,7 +98,7 @@ def get_test_files(target_dir: pathlib.Path) -> Set[str]:
)
return {s.strip("/") for s in res_test_files.stdout.splitlines()}
except subprocess.CalledProcessError as e:
raise RuntimeError(
raise RuntimeError( # noqa: TRY003
f"Command {e.cmd} returned with error (code {e.returncode}): {e.output}"
) from e

Expand Down Expand Up @@ -154,7 +154,7 @@ def get_test_files_in_test_suite(target_dir: pathlib.Path) -> Set[str]:
)
return {s.strip() for s in res_test_files_with_fixture_definitions.stdout.splitlines()}
except subprocess.CalledProcessError as e:
raise RuntimeError(
raise RuntimeError( # noqa: TRY003
f"Command {e.cmd} returned with error (code {e.returncode}): {e.output}"
) from e

Expand Down
6 changes: 3 additions & 3 deletions ci/checks/check_name_tag_snippets_referenced.py
Original file line number Diff line number Diff line change
Expand Up @@ -352,7 +352,7 @@
def check_dependencies(*deps: str) -> None:
for dep in deps:
if not shutil.which(dep):
raise Exception(f"Must have `{dep}` installed in PATH to run {__file__}")
raise Exception(f"Must have `{dep}` installed in PATH to run {__file__}") # noqa: TRY002, TRY003


def get_snippet_definitions(target_dir: pathlib.Path) -> List[str]:
Expand Down Expand Up @@ -380,7 +380,7 @@ def get_snippet_definitions(target_dir: pathlib.Path) -> List[str]:
)
return res_snippet_names.stdout.splitlines()
except subprocess.CalledProcessError as e:
raise RuntimeError(
raise RuntimeError( # noqa: TRY003
f"Command {e.cmd} returned with error (code {e.returncode}): {e.output}"
) from e

Expand Down Expand Up @@ -412,7 +412,7 @@ def get_snippets_used(target_dir: pathlib.Path) -> List[str]:
)
return res_snippet_used_names.stdout.splitlines()
except subprocess.CalledProcessError as e:
raise RuntimeError(
raise RuntimeError( # noqa: TRY003
f"Command {e.cmd} returned with error (code {e.returncode}): {e.output}"
) from e

Expand Down
4 changes: 2 additions & 2 deletions ci/checks/check_no_line_number_snippets.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
def check_dependencies(*deps: str) -> None:
for dep in deps:
if not shutil.which(dep):
raise Exception(f"Must have `{dep}` installed in PATH to run {__file__}")
raise Exception(f"Must have `{dep}` installed in PATH to run {__file__}") # noqa: TRY002, TRY003


def run_grep(target_dir: pathlib.Path) -> List[str]:
Expand All @@ -39,7 +39,7 @@ def run_grep(target_dir: pathlib.Path) -> List[str]:
capture_output=True,
)
except subprocess.CalledProcessError as e:
raise RuntimeError(
raise RuntimeError( # noqa: TRY003
f"Command {e.cmd} returned with error (code {e.returncode}): {e.output}"
) from e
return res.stdout.splitlines()
Expand Down
4 changes: 2 additions & 2 deletions ci/checks/check_only_name_tag_snippets.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
def check_dependencies(*deps: str) -> None:
for dep in deps:
if not shutil.which(dep):
raise Exception(f"Must have `{dep}` installed in PATH to run {__file__}")
raise Exception(f"Must have `{dep}` installed in PATH to run {__file__}") # noqa: TRY002, TRY003


def run_grep(target_dir: pathlib.Path) -> List[str]:
Expand Down Expand Up @@ -87,7 +87,7 @@ def run_grep(target_dir: pathlib.Path) -> List[str]:
set(res_positive.stdout.splitlines()).difference(set(res_negative.stdout.splitlines()))
)
except subprocess.CalledProcessError as e:
raise RuntimeError(
raise RuntimeError( # noqa: TRY003
f"Command {e.cmd} returned with error (code {e.returncode}): {e.output}"
) from e
return res
Expand Down
4 changes: 2 additions & 2 deletions ci/checks/validate_docs_snippets.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
def check_dependencies(*deps: str) -> None:
for dep in deps:
if not shutil.which(dep):
raise Exception(f"Must have `{dep}` installed in PATH to run {__file__}")
raise Exception(f"Must have `{dep}` installed in PATH to run {__file__}") # noqa: TRY002, TRY003


def run_docusaurus_build(target_dir: str) -> None:
Expand Down Expand Up @@ -45,7 +45,7 @@ def run_grep(target_dir: str) -> List[str]:
text=True,
)
except subprocess.CalledProcessError as e:
raise RuntimeError(f"Command {e.cmd} returned with error (code {e.returncode}): {e.output}")
raise RuntimeError(f"Command {e.cmd} returned with error (code {e.returncode}): {e.output}") # noqa: TRY003
return res.stdout.splitlines()


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ def _pandas(
if data_label.upper() in labeler.label_mapping.keys():
data_label_ind = labeler.label_mapping[data_label.upper()]
else:
raise ValueError(
raise ValueError( # noqa: TRY003
"""
The only values acceptable for the data label parameter are as follows:
['PAD', 'UNKNOWN', 'ADDRESS', 'BAN', 'CREDIT_CARD', 'DATE', 'TIME', 'DATETIME',\
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ def _pandas(
if data_label.upper() in labeler.label_mapping.keys():
data_label_ind = labeler.label_mapping[data_label.upper()]
else:
raise ValueError(
raise ValueError( # noqa: TRY003
"""
The only values acceptable for the data label parameter are as follows:
['PAD', 'UNKNOWN', 'ADDRESS', 'BAN', 'CREDIT_CARD', 'DATE', 'TIME', 'DATETIME',\
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ def _pandas(
first_profile_path = metric_value_kwargs["profile_path"]
first_profile = dp.Profiler.load(first_profile_path)
except FileNotFoundError:
raise ValueError(
raise ValueError( # noqa: TRY003
"'profile_path' does not point to a valid DataProfiler stored profile."
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ def _pandas(
)
return profile_report
except FileNotFoundError:
raise ValueError(
raise ValueError( # noqa: TRY003
"'profile_path' does not point to a valid DataProfiler stored profile."
)
except Exception as e:
Expand Down
6 changes: 3 additions & 3 deletions contrib/cli/great_expectations_contrib/package.py
Original file line number Diff line number Diff line change
Expand Up @@ -271,9 +271,9 @@ def _identify_user_package() -> str:

# A sanity check in case the user modifies the Cookiecutter template in unexpected ways
if len(packages) == 0:
raise FileNotFoundError("Could not find a user-defined package")
raise FileNotFoundError("Could not find a user-defined package") # noqa: TRY003
elif len(packages) > 1:
raise ValueError("Found more than one user-defined package")
raise ValueError("Found more than one user-defined package") # noqa: TRY003

return packages[0]

Expand All @@ -285,7 +285,7 @@ def _import_expectations_module(package: str) -> Any:
try:
expectations_module = importlib.import_module(f"{package}.expectations")
return expectations_module
except ModuleNotFoundError:
except ModuleNotFoundError: # noqa: TRY302
raise

@staticmethod
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -246,7 +246,7 @@ def _expected_list(
for x in range((end_date - start_date).days + 1)
]
except TypeError as ex:
raise InvalidExpectationConfigurationError(
raise InvalidExpectationConfigurationError( # noqa: TRY003
f"Expecting datetime when datetime_format is set\n{ex}"
)
# else - no datetime format, so we're assuming integers
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,8 @@ def _get_query_result(func, selectable, execution_engine: SqlAlchemyExecutionEng
exception_message: str = "An SQL syntax Exception occurred."
exception_traceback: str = traceback.format_exc()
exception_message += f'{type(pe).__name__}: "{pe!s}". Traceback: "{exception_traceback}".'
logger.error(exception_message)
raise pe
logger.error(exception_message) # noqa: TRY400
raise pe # noqa: TRY201

# @classmethod
# def _get_evaluation_dependencies(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ def _pandas(cls, column, level=None, **kwargs):
def is_parseable(val):
try:
if type(val) != str: # noqa: E721
raise TypeError(
raise TypeError( # noqa: TRY003
"Values passed to expect_column_values_to_be_edtf_parseable must be of type string.\nIf you want to validate a column of dates or timestamps, please call the expectation before converting from string format."
)

Expand All @@ -62,7 +62,7 @@ def is_parseable(val):
return False

if level is not None and type(level) != int: # noqa: E721
raise TypeError("level must be of type int.")
raise TypeError("level must be of type int.") # noqa: TRY003

return column.map(is_parseable)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ def _spark(
data=value_set, schema=df.schema[column_name].dataType
)
except TypeError as e:
raise InvalidExpectationConfigurationError(
raise InvalidExpectationConfigurationError( # noqa: TRY003
f"`value_set` must be of same type as `column` : {e}"
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def _pandas(self, _column, **kwargs):
if all(_column.str.isdigit()) is True:
temp_column = _column.astype(int)
else:
raise TypeError("Column must be a string-type capable of being cast to int.")
raise TypeError("Column must be a string-type capable of being cast to int.") # noqa: TRY003

series_diff = np.diff(temp_column)

Expand Down Expand Up @@ -75,7 +75,7 @@ def _spark(
if pyspark.types and isinstance(column_metadata["type"], pyspark.types.StringType):
column = F.col(column_name).cast(pyspark.types.IntegerType())
else:
raise TypeError("Column must be a string-type capable of being cast to int.")
raise TypeError("Column must be a string-type capable of being cast to int.") # noqa: TRY003

compute_domain_kwargs = metric_domain_kwargs

Expand All @@ -88,7 +88,7 @@ def _spark(
)

if any(np.array(df.select(column.isNull()).collect())):
raise TypeError("Column must be a string-type capable of being cast to int.")
raise TypeError("Column must be a string-type capable of being cast to int.") # noqa: TRY003

diff = column - F.lag(column).over(pyspark.Window.orderBy(F.lit("constant")))
diff = F.when(diff.isNull(), 1).otherwise(diff)
Expand Down Expand Up @@ -196,7 +196,7 @@ def _validate_success_key(
""""""
if param not in configuration.kwargs:
if required:
raise InvalidExpectationKwargsError(
raise InvalidExpectationKwargsError( # noqa: TRY003
f"Parameter {param} is required but was not found in configuration."
)
return
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ class ColumnValuesToChangeBetween(ColumnMapMetricProvider):
def _pandas(cls, column, from_value, to_value, **kwargs):
# throw an error if one of the values is not numeric
if not pd.to_numeric(column, errors="coerce").notnull().all():
raise TypeError("Column values must be numeric !")
raise TypeError("Column values must be numeric !") # noqa: TRY003

# calculate the difference of the current row with the previous.
# If previous is NaN fills with the initial value "from_value" to consider it true
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ class ColumnRuleFollowers(ColumnMapMetricProvider):
@column_condition_partial(engine=PandasExecutionEngine)
def _pandas(cls, column, rule, **kwargs):
if rule["ranges"]:
raise ValueError("Ranges must contain at least 1 variable!")
raise ValueError("Ranges must contain at least 1 variable!") # noqa: TRY003

return column.apply(lambda x: ColumnRuleFollowers._helper(x, rule))

Expand All @@ -47,15 +47,15 @@ def _helper(x, rule):
if x is None:
x = ""
if not isinstance(x, str):
raise TypeError(
raise TypeError( # noqa: TRY003
"Column values must be strings in order to use 'expect_column_values_to_follow_rule'"
)
for name, rnge in rule["ranges"].items():
if rnge[0] < rnge[1]:
strings[name] = str(x[rnge[0] : rnge[1]])
names += name + ","
else:
raise ValueError(
raise ValueError( # noqa: TRY003
"Unexpected range. Ensure that the second number in your range is larger than the first."
)

Expand Down