pyproject.toml

[tool.poetry]
name = "great_expectations_cloud"
version = "20240919.0.dev0"
description = "Great Expectations Cloud"
authors = ["The Great Expectations Team <team@greatexpectations.io>"]
repository = "https://github.com/great-expectations/cloud"
homepage = "https://greatexpectations.io"
readme = "README.md"
license = "Proprietary"
classifiers = [
    "Development Status :: 3 - Alpha",
    "Intended Audience :: Developers",
    "Intended Audience :: Science/Research",
    "Topic :: Scientific/Engineering",
    "Topic :: Scientific/Engineering :: Information Analysis",
    "Topic :: Software Development :: Quality Assurance",
    "Topic :: Software Development :: Testing",
]
# package data
include = [
    "py.typed", # exporting types https://peps.python.org/pep-0561/
]

[tool.poetry.dependencies]
python = ">=3.8,<3.12"
great-expectations = "^0.18.20,<0.19" # <0.19 instead of <v1.0 to include v1 pre-releases
pydantic = ">=2.8.1,<3"
pika = "^1.3.1"
# needed for metrics serialization
orjson = "^3.9.7, !=3.9.10" # TODO: remove inequality once dep resolution issue is resolved
# relying on packaging in agent code so declaring it explicitly here
packaging = ">=21.3,<25.0"
tenacity = ">=8.2.3,<10.0.0"
sqlalchemy = { version = ">=2.0", optional = false }
# optional dependencies
snowflake-sqlalchemy = { version = ">=1.6", optional = true }
snowflake-connector-python = { version = ">=3.3.1", optional = true }
psycopg2-binary = { version = "*", optional = true }
databricks-sql-connector = { version = ">=3.1.1", extras = ["sqlalchemy"], optional = true }

[tool.poetry.extras]
# https://python-poetry.org/docs/pyproject#extras
# these must reference an optional dependency in the dependencies section
snowflake = ["snowflake-sqlalchemy", "snowflake-connector-python"]
postgres = ["sqlalchemy", "psycopg2-binary"]
databricks = ["databricks-sql-connector"]

[tool.poetry.group.sql.dependencies]
great-expectations = { version = ">=0.18.19"}
snowflake-sqlalchemy = ">=1.6"
snowflake-connector-python = ">=3.3.1"
psycopg2-binary = "^2.9.9"
databricks-sql-connector = { version = ">=3.1.1", extras = ["sqlalchemy"]}

[tool.poetry.group.dev.dependencies]
coverage = { extras = ["toml"], version = "^7.5.1" }
freezegun = "^1.4.0"
invoke = "^2.2.0"
mypy = "1.11.2"
pre-commit = "^3.3.3"
pyfakefs = "^5.4.1"
pytest = ">=7.4,<9.0"
pytest-cov = ">=5"
pytest-icdiff = "*"
pytest-mock = "*"
responses = ">=0.23.1,<0.26.0"
ruff = "0.6.5"
tenacity = ">=8.2.3,<10.0.0"
tomlkit = ">=0.12.1,<0.14.0"
types-requests = "^2.31"
typing_extensions = ">=4.4.0"

[tool.poetry.scripts]
gx-agent = 'great_expectations_cloud.agent.cli:main'

[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"

[tool.mypy]
python_version = "3.8"
plugins = ["pydantic.mypy"]
files = ["great_expectations_cloud/", "tests/", "tasks.py"]
strict = true
warn_unused_configs = true
ignore_missing_imports = true
follow_imports = 'normal'
warn_redundant_casts = true
show_error_codes = true
implicit_reexport = true                                         # enabled due to strict mode
enable_error_code = [
    'ignore-without-code',
    'explicit-override',
    'possibly-undefined',
]


[[tool.mypy.overrides]]
module = "tests.*"
disable_error_code = ['no-untyped-def']

[tool.black]
target-version = ["py38", "py39", "py310", "py311"]
line-length = 100

[tool.ruff]
target-version = "py38"
# TODO: change this back to black default of 88 if desired
line-length = 100
lint.select = [
    # https://beta.ruff.rs/docs/rules/#pyflakes-f
    "F", # Pyflakes
    # https://beta.ruff.rs/docs/rules/#pycodestyle-e-w
    "E", # pycodestyle
    "W", # Warning
    # https://beta.ruff.rs/docs/rules/#flake8-comprehensions-c4
    # https://beta.ruff.rs/docs/rules/#mccabe-c90
    "C", # Complexity (mccabe+) & comprehensions
    # https://beta.ruff.rs/docs/rules/#pyupgrade-up
    "UP", # pyupgrade
    # https://beta.ruff.rs/docs/rules/#isort-i
    "I", # isort
    # https://beta.ruff.rs/docs/rules/#flake8-type-checking-tch
    "TCH", # flake8-type-checking-tch
    # https://beta.ruff.rs/docs/rules/#flake8-tidy-imports-tid
    "TID", # flake8-tidy-imports
    # https://docs.astral.sh/ruff/rules/#tryceratops-try
    # https://github.com/guilatrova/tryceratops
    "TRY", # tryceratops - pythonic exception handling
    # https://beta.ruff.rs/docs/rules/#flake8-pyi-pyi
    "PYI", # flake8-pyi - type stub files
    # https://beta.ruff.rs/docs/rules/#flake8-use-pathlib-pth
    "PTH", # use-pathlib - use pathlib for os.path and glob operations
    # https://beta.ruff.rs/docs/rules/#flake8-bugbear-b
    "B", # bugbear - common python bugs & design issues
    # https://beta.ruff.rs/docs/rules/#flake8-datetimez-dtz
    "DTZ", # flake8-datetimez-dtz - prevent use of tz naive datetimes
    # https://beta.ruff.rs/docs/rules/#pylint-pl
    "PL", # pylint
    # https://docs.astral.sh/ruff/rules/#ruff-specific-rules-ruf
    "RUF",
    # https://docs.astral.sh/ruff/rules/#flake8-bandit-s
    "S", # bandit - security
    # https://docs.astral.sh/ruff/rules/#flake8-async-async
    "ASYNC", # async rules
]
lint.ignore = [
    # formatting related ignores
    # https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules
    "W191", # tab-identation
    "E111", # indentation-with-invalid-multiple
    "E114", # indentation-with-invalid-multiple-comment
    "E117", # over-idented
    "E501", # line-too-long
    # https://beta.ruff.rs/docs/rules/#flake8-type-checking-tch
    # minimal cost for standard lib imports; keep this disabled
    "TCH003", # typing-only-standard-library-import
    # gives false positives if we use try imports and type-checking import
    "TID252", # Relative imports from parent modules are banned
    # https://beta.ruff.rs/docs/rules/#flake8-use-pathlib-pth
    "PTH123", # pathlib-open - this would force pathlib usage anytime open or with open was used.
]
# https://docs.astral.sh/ruff/linter/#fix-safety
lint.extend-safe-fixes = [
    # pydantic models use annotations at runtime and need to be accounted for
    # these are safe to apply automatically given settings for tool.ruff.flake8-type-checking
    "TCH", # https://beta.ruff.rs/docs/rules/#flake8-type-checking-tch
]

[tool.ruff.lint.flake8-type-checking]
# pydantic models use annotations at runtime
runtime-evaluated-base-classes = [
    # NOTE: ruff is unable to detect that these are subclasses of pydantic.BaseModel
    "pydantic.BaseModel",
    "pydantic.v1.BaseModel",
    "great_expectations_cloud.agent.models.AgentBaseExtraForbid",
    "great_expectations_cloud.agent.models.AgentBaseExtraIgnore",
    "great_expectations_cloud.agent.models.EventBase",
    "great_expectations.datasource.fluent.fluent_base_model.FluentBaseModel",
    "great_expectations.datasource.fluent.interfaces.Datasource",
    "great_expectations.datasource.fluent.sql_datasource.SQLDatasource",
]
runtime-evaluated-decorators = ["pydantic.dataclasses.dataclass"]

[tool.ruff.lint.isort]
required-imports = ["from __future__ import annotations"]

[tool.ruff.lint.mccabe]
max-complexity = 10

[tool.ruff.lint.flake8-tidy-imports.banned-api]
"os.environ".msg = """Please do not use os.environ, instead use a pydantic.BaseSettings model"""
"great_expectations.compatibility.pydantic".msg = "Import pydantic directly."
"great_expectations.compatibility.sqlalchemy".msg = "Import sqlalchemy directly."
"great_expectations.compatibility".msg = "Favor specific version imports over compatibility module."


[tool.ruff.lint.per-file-ignores]
"__init__.py" = [
    "F401", # unused import
    "F403", # star imports
]
"models.py" = [
    # TODO: remove these once our min python is 3.9 or greater
    # or once these rule respect runtime-evaluated-base-classes
    "UP006", # non-pep585-annotation - pydantic models use annotations at runtime
    "UP007", # non-pep604-annotation - pydantic models use annotations at runtime
]
"tasks.py" = [
    "S101", # https://docs.astral.sh/ruff/rules/assert/
]

[tool.pytest.ini_options]
addopts = "--cov=great_expectations_cloud"
markers = [
    "agentjobs",
    "unit: mark a test as a unit test i.e. no external dependencies.",
    "integration: test that relies on a running GX Cloud and mercury instance.",
]
log_level = "info"
filterwarnings = [
    "error", # Turn all warnings not explicitly filtered below into errors
    # Add any warnings to globally ignore here (this should be used sparingly, it is better to handle warnings
    # upstream e.g. in GX Core or explicitly where they are raised):
    # this is from azure dependency
    # These ignores should be removed once we handle this warning in GX Core
    # Full warning:
    # DeprecationWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html
    "ignore:pkg_resources is deprecated as an API:DeprecationWarning",
    # Full warning:
    # DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('ruamel')`.
    # Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages
    "ignore:Deprecated call to:DeprecationWarning",
    # Full warning:
    # pytest.PytestCollectionWarning: cannot collect test class 'TestConnectionError' because it has a __init__ constructor (from: tests/agent/actions/test_draft_datasource_config.py)
    "ignore:cannot collect test class 'TestConnectionError':pytest.PytestCollectionWarning",
    # Full warning:
    # DeprecationWarning: module 'sre_constants' is deprecated
    "ignore:module 'sre_constants' is deprecated:DeprecationWarning",
    # Full warning:
    # DeprecationWarning: module 'sre_parse' is deprecated
    "ignore:module 'sre_parse' is deprecated:DeprecationWarning",
    # Full warning:
    # IAM_ROLE_NAME_RE = re.compile('[A-Za-z0-9+=,.@\-_]{1,64}')"
    # DeprecationWarning: invalid escape sequence \-
    "ignore:invalid escape sequence",
    # Full warning:
    # PoetryVersionOutdated: The latest version of poetry is latest_poetry_version: 1.8.0 but the poetry.lock file was generated using 1.7.1.
    "once:.*:tests.test_project.PoetryVersionOutdated",
    # Full warning:
    # sqlalchemy.exc.SAWarning: The GenericFunction 'flatten' is already registered and is going to be overridden.
    # this is apparently a known issue with snowflake connector 1.6
    "module: The GenericFunction 'flatten' is already registered and is going to be overridden:sqlalchemy.exc.SAWarning",
    #  RuntimeWarning: coroutine 'FastStream.run' was never awaited. We are already handling FastStream.run() as part of asyncio
    "ignore: coroutine 'FastStream.run' was never awaited"
]

[tool.coverage.report]
# https://coverage.readthedocs.io/en/7.5.1/config.html
# Regexes for lines to exclude from consideration
exclude_also = [
    # type-checking imports don't exist at runtime
    "if TYPE_CHECKING:",
    # Don't complain if tests don't hit NotImplementedError:
    "raise NotImplementedError",
    # Don't complain if non-runnable code isn't run:
    "if __name__ == .__main__.:",
    # Don't complain about abstract methods, they aren't run:
    "@(abc\\.)?abstractmethod",
]