From c66d221e4117a4aa319aa8050a4e8def71413307 Mon Sep 17 00:00:00 2001 From: Petro Date: Wed, 1 Dec 2021 15:09:34 +0000 Subject: [PATCH 1/7] enh: moving setup.py to setup.cfg --- setup.cfg | 8 ++++++++ setup.py | 36 ------------------------------------ 2 files changed, 8 insertions(+), 36 deletions(-) delete mode 100644 setup.py diff --git a/setup.cfg b/setup.cfg index 7446cd2..aa3acfc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,8 +7,16 @@ description = Sqlalchemy adapter for Firebolt long_description = file: README.md long_description_content_type = text/markdown url = https://github.com/firebolt-db/firebolt-sqlalchemy +download_url = https://github.com/firebolt-db/firebolt-sqlalchemy/archive/refs/tags/0.0.9.tar.gz project_urls = Bug Tracker = https://github.com/firebolt-db/firebolt-sqlalchemy +install_requires = + sqlalchemy>=1.0.0 + requests + datetime +entry_points = + sqlalchemy.dialects : + firebolt = firebolt_db.firebolt_dialect:FireboltDialect classifiers = Programming Language :: Python :: 3 License :: OSI Approved :: MIT License diff --git a/setup.py b/setup.py deleted file mode 100644 index dc56de2..0000000 --- a/setup.py +++ /dev/null @@ -1,36 +0,0 @@ -import setuptools - -with open("README.md", "r", encoding="utf-8") as fh: - long_description = fh.read() - -setuptools.setup( - name="firebolt-sqlalchemy", - version="0.0.9", - author="Firebolt", - author_email="pypi@firebolt.io", - description="Sqlalchemy adapter for Firebolt", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/firebolt-db/firebolt-sqlalchemy", - download_url="https://github.com/firebolt-db/firebolt-sqlalchemy/archive/refs/tags/0.0.9.tar.gz", - project_urls={ - "Bug Tracker": "https://github.com/firebolt-db/firebolt-sqlalchemy", - }, - install_requires=[ - 'sqlalchemy>=1.0.0', - "firebolt-sdk" - ], - entry_points={ - "sqlalchemy.dialects": [ - "firebolt = firebolt_db.firebolt_dialect:FireboltDialect" - ], - }, - classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - ], - package_dir={"": "src"}, - packages=setuptools.find_packages(where="src"), - python_requires=">=3.6", -) \ No newline at end of file From 9dd2aef6ab558ee31b72a6e26e894ca0c14d52bd Mon Sep 17 00:00:00 2001 From: Petro Date: Wed, 1 Dec 2021 15:06:25 +0000 Subject: [PATCH 2/7] Improving gitignore --- .gitignore | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 9966719..1e4eecc 100644 --- a/.gitignore +++ b/.gitignore @@ -64,4 +64,41 @@ share/python-wheels/ *.egg-info/ .installed.cfg *.egg -MANIFEST \ No newline at end of file +MANIFEST + +# Byte-compiled / optimized / DLL files +__pycache__/ + +# Unit tests / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ +coverage-reports/* + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# VS Code +.vscode/* +*.code-workspace + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json \ No newline at end of file From 1d014e45a4f546d0957a6c927eae2ab1d75a2cd2 Mon Sep 17 00:00:00 2001 From: Petro Date: Wed, 1 Dec 2021 15:24:09 +0000 Subject: [PATCH 3/7] Adding pre-commit hooks --- .pre-commit-config.yaml | 47 +++++++++++++++++++++++++++++++++++ setup.cfg | 42 ++++++++++++++++++------------- src/firebolt_db/__init__.py | 5 ++-- tests/integration/conftest.py | 1 - 4 files changed, 74 insertions(+), 21 deletions(-) create mode 100755 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100755 index 0000000..8682406 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,47 @@ +repos: + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: check-merge-conflict + + - repo: https://github.com/asottile/setup-cfg-fmt + rev: v1.17.0 + hooks: + - id: setup-cfg-fmt + + - repo: https://github.com/myint/autoflake + rev: v1.4 + hooks: + - id: autoflake + args: + - -i + - -r + - --remove-all-unused-imports + - --ignore-init-module-imports + - --remove-duplicate-keys + - --remove-unused-variables + + - repo: https://github.com/PyCQA/isort + rev: 5.9.3 + hooks: + - id: isort + + - repo: https://github.com/psf/black + rev: 21.7b0 + hooks: + - id: black + language-version: 3.9 + + - repo: https://github.com/dfm/black_nbconvert + rev: v0.3.0 + hooks: + - id: black_nbconvert + + - repo: https://gitlab.com/pycqa/flake8 + rev: 3.7.9 + hooks: + - id: flake8 + additional_dependencies: + - flake8-tidy-imports + - flake8-quotes diff --git a/setup.cfg b/setup.cfg index aa3acfc..1ad3907 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,32 +1,40 @@ [metadata] -name = firebolt-sqlalchemy +name = firebolt_sqlalchemy version = 0.0.9 -author = Firebolt -author_email = pypi@firebolt.io description = Sqlalchemy adapter for Firebolt long_description = file: README.md long_description_content_type = text/markdown url = https://github.com/firebolt-db/firebolt-sqlalchemy -download_url = https://github.com/firebolt-db/firebolt-sqlalchemy/archive/refs/tags/0.0.9.tar.gz -project_urls = - Bug Tracker = https://github.com/firebolt-db/firebolt-sqlalchemy -install_requires = - sqlalchemy>=1.0.0 - requests - datetime -entry_points = - sqlalchemy.dialects : - firebolt = firebolt_db.firebolt_dialect:FireboltDialect +author = Firebolt +author_email = pypi@firebolt.io +license = Apache-2.0 +license_file = LICENSE classifiers = - Programming Language :: Python :: 3 + License :: OSI Approved :: Apache Software License License :: OSI Approved :: MIT License Operating System :: OS Independent + Programming Language :: Python :: 3 + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 +download_url = https://github.com/firebolt-db/firebolt-sqlalchemy/archive/refs/tags/0.0.9.tar.gz +entry_points = + sqlalchemy.dialects : + firebolt = firebolt_db.firebolt_dialect:FireboltDialect +install_requires = + sqlalchemy>=1.0.0 + requests + datetime +project_urls = + Bug Tracker = https://github.com/firebolt-db/firebolt-sqlalchemy [options] -package_dir = - = src packages = find: python_requires = >=3.6 +package_dir = + = src [options.packages.find] -where = src \ No newline at end of file +where = src diff --git a/src/firebolt_db/__init__.py b/src/firebolt_db/__init__.py index b031390..d3f12ab 100644 --- a/src/firebolt_db/__init__.py +++ b/src/firebolt_db/__init__.py @@ -1,4 +1,3 @@ -from firebolt.db import connect from firebolt.common.exception import ( DatabaseError, DataError, @@ -11,7 +10,7 @@ ProgrammingError, Warning, ) - +from firebolt.db import connect __all__ = [ "connect", @@ -34,4 +33,4 @@ apilevel = "2.0" # Threads may share the module and connections threadsafety = 1 -paramstyle = "pyformat" \ No newline at end of file +paramstyle = "pyformat" diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index c80ff61..1dafaf9 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,7 +1,6 @@ from logging import getLogger from os import environ -import firebolt as firebolt_sdk from pytest import fixture from sqlalchemy import create_engine from sqlalchemy.dialects import registry From efa5d9b6caa9afec7369f6f34453a4c3c3ca709c Mon Sep 17 00:00:00 2001 From: Petro Date: Thu, 2 Dec 2021 08:23:36 +0000 Subject: [PATCH 4/7] Adding mypy and correcting checks --- .github/workflows/python-app.yml | 48 +++++++++++++++++++++++++++++ pyproject.toml | 20 +++++++++++- setup.cfg | 27 ++++++++++++++-- src/firebolt_db/firebolt_dialect.py | 48 ++++++++++++++++------------- 4 files changed, 119 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/python-app.yml diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml new file mode 100644 index 0000000..328faf2 --- /dev/null +++ b/.github/workflows/python-app.yml @@ -0,0 +1,48 @@ +# This workflow will install Python dependencies, run pre-commit checks, and run tests with a single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: PR verify + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install ".[dev]" + + - name: Run pre-commit checks + uses: pre-commit/action@v2.0.3 + with: + extra_args: --all-files + + - name: Type check with mypy + run: | + mypy src + + - name: Test with pytest + run: | + pytest tests/unit + + - name: Convetional Commits Pull Request Title + uses: jef/conventional-commits-pr-action@v1.0.0 + if: github.event_name == 'pull_request' + with: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/pyproject.toml b/pyproject.toml index b5a3c46..8f9cc7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,4 +3,22 @@ requires = [ "setuptools>=42", "wheel" ] -build-backend = "setuptools.build_meta" \ No newline at end of file +build-backend = "setuptools.build_meta" + +# configure isort to be compatible with black +# source: https://black.readthedocs.io/en/stable/compatible_configs.html#configuration +[tool.isort] +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true + +[tool.black] +exclude = ''' +/( + \.git + | \.venv + | dist +)/ +''' \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 1ad3907..3afe4e1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,8 +25,7 @@ entry_points = firebolt = firebolt_db.firebolt_dialect:FireboltDialect install_requires = sqlalchemy>=1.0.0 - requests - datetime + firebolt-sdk project_urls = Bug Tracker = https://github.com/firebolt-db/firebolt-sqlalchemy @@ -38,3 +37,27 @@ package_dir = [options.packages.find] where = src + +[options.extras_require] +dev = + devtools==0.7.0 + mypy==0.910 + pre-commit==2.15.0 + pytest==6.2.5 + sqlalchemy-stubs + +[mypy] +disallow_untyped_defs = True +show_error_codes = True +plugins = sqlmypy + +[pydantic-mypy] +warn_required_dynamic_aliases = True +warn_untyped_fields = True + +[flake8] +max-line-length = 88 +per-file-ignores = __init__.py:F401 +ignore = E203, W503 +ban-relative-imports = True +inline-quotes = " diff --git a/src/firebolt_db/firebolt_dialect.py b/src/firebolt_db/firebolt_dialect.py index 549fc7e..6f07f96 100644 --- a/src/firebolt_db/firebolt_dialect.py +++ b/src/firebolt_db/firebolt_dialect.py @@ -1,11 +1,11 @@ import os +from types import ModuleType from typing import Any, Dict, List, Optional, Tuple, Union import sqlalchemy.types as sqltypes from sqlalchemy.engine import Connection as AlchemyConnection from sqlalchemy.engine import ExecutionContext, default from sqlalchemy.engine.url import URL -from sqlalchemy.pool.base import _ConnectionFairy from sqlalchemy.sql import compiler from sqlalchemy.types import ( BIGINT, @@ -48,7 +48,7 @@ class ARRAY(sqltypes.TypeEngine): class UniversalSet(set): - def __contains__(self, item): + def __contains__(self, item: Any) -> bool: return True @@ -61,14 +61,15 @@ class FireboltCompiler(compiler.SQLCompiler): class FireboltTypeCompiler(compiler.GenericTypeCompiler): - def visit_ARRAY(self, type, **kw): + def visit_ARRAY(self, type: sqltypes.TypeEngine, **kw: int) -> str: return "Array(%s)" % type class FireboltDialect(default.DefaultDialect): """ FireboltDialect defines the behavior of Firebolt database and DB-API combination. - It is responsible for metadata definition and firing queries for receiving Database schema and table information. + It is responsible for metadata definition and firing queries for receiving Database + schema and table information. """ name = "firebolt" @@ -88,12 +89,14 @@ class FireboltDialect(default.DefaultDialect): description_encoding = None supports_native_boolean = True - def __init__(self, context: Optional[ExecutionContext] = None, *args, **kwargs): + def __init__( + self, context: Optional[ExecutionContext] = None, *args: Any, **kwargs: Any + ): super(FireboltDialect, self).__init__(*args, **kwargs) self.context: Union[ExecutionContext, Dict] = context or {} @classmethod - def dbapi(cls): + def dbapi(cls) -> ModuleType: return firebolt_db # Build firebolt-sdk compatible connection arguments. @@ -110,7 +113,9 @@ def create_connect_args(self, url: URL) -> Tuple[List, Dict]: kwargs["api_endpoint"] = os.environ["FIREBOLT_BASE_URL"] return ([], kwargs) - def get_schema_names(self, connection: AlchemyConnection, **kwargs) -> List[str]: + def get_schema_names( + self, connection: AlchemyConnection, **kwargs: Any + ) -> List[str]: query = "select schema_name from information_schema.databases" result = connection.execute(query) return [row.schema_name for row in result] @@ -133,7 +138,7 @@ def has_table( return result.fetchone().exists_ def get_table_names( - self, connection: AlchemyConnection, schema: Optional[str] = None, **kwargs + self, connection: AlchemyConnection, schema: Optional[str] = None, **kwargs: Any ) -> List[str]: query = "select table_name from information_schema.tables" if schema: @@ -145,7 +150,7 @@ def get_table_names( return [row.table_name for row in result] def get_view_names( - self, connection: AlchemyConnection, schema: Optional[str] = None, **kwargs + self, connection: AlchemyConnection, schema: Optional[str] = None, **kwargs: Any ) -> List[str]: return [] @@ -154,7 +159,7 @@ def get_table_options( connection: AlchemyConnection, table_name: str, schema: Optional[str] = None, - **kwargs + **kwargs: Any ) -> Dict: return {} @@ -163,7 +168,7 @@ def get_columns( connection: AlchemyConnection, table_name: str, schema: Optional[str] = None, - **kwargs + **kwargs: Any ) -> List[Dict]: query = """ select column_name, @@ -196,7 +201,7 @@ def get_pk_constraint( connection: AlchemyConnection, table_name: str, schema: Optional[str] = None, - **kwargs + **kwargs: Any ) -> Dict: return {"constrained_columns": [], "name": None} @@ -205,7 +210,7 @@ def get_foreign_keys( connection: AlchemyConnection, table_name: str, schema: Optional[str] = None, - **kwargs + **kwargs: Any ) -> List[Dict]: return [] @@ -214,7 +219,7 @@ def get_check_constraints( connection: AlchemyConnection, table_name: str, schema: Optional[str] = None, - **kwargs + **kwargs: Any ) -> List[Dict]: return [] @@ -223,7 +228,7 @@ def get_table_comment( connection: AlchemyConnection, table_name: str, schema: Optional[str] = None, - **kwargs + **kwargs: Any ) -> Dict: return {"text": ""} @@ -232,7 +237,7 @@ def get_indexes( connection: AlchemyConnection, table_name: str, schema: Optional[str] = None, - **kwargs + **kwargs: Any ) -> List[Dict]: return [] @@ -241,7 +246,7 @@ def get_unique_constraints( connection: AlchemyConnection, table_name: str, schema: Optional[str] = None, - **kwargs + **kwargs: Any ) -> List[Dict]: return [] @@ -250,11 +255,11 @@ def get_view_definition( connection: AlchemyConnection, view_name: str, schema: Optional[str] = None, - **kwargs + **kwargs: Any ) -> str: pass - def do_rollback(self, dbapi_connection: _ConnectionFairy): + def do_rollback(self, dbapi_connection: AlchemyConnection) -> None: pass def _check_unicode_returns( @@ -270,11 +275,12 @@ def _check_unicode_returns( def _check_unicode_description(self, connection: AlchemyConnection) -> bool: """ - Same as _check_unicode_returns this might be redundant as there's no reference to it in the sqlalchemy repo + Same as _check_unicode_returns this might be redundant as there's + no reference to it in the sqlalchemy repo. """ return True - def do_commit(self, dbapi_connection: _ConnectionFairy): + def do_commit(self, dbapi_connection: AlchemyConnection) -> None: pass From de7485d1cf43cddf02a64826398c29d0705c5761 Mon Sep 17 00:00:00 2001 From: Petro Date: Thu, 2 Dec 2021 08:29:58 +0000 Subject: [PATCH 5/7] refactor unit tests --- tests/unit/test_firebolt_dialect.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_firebolt_dialect.py b/tests/unit/test_firebolt_dialect.py index 912edb6..d726aab 100644 --- a/tests/unit/test_firebolt_dialect.py +++ b/tests/unit/test_firebolt_dialect.py @@ -1,7 +1,6 @@ import os from unittest import mock -import firebolt.db # Firebolt sdk import sqlalchemy from conftest import MockDBApi from sqlalchemy.engine import url @@ -24,7 +23,8 @@ def test_create_dialect(self, dialect: FireboltDialect): assert dialect.driver == "firebolt" assert issubclass(dialect.preparer, FireboltIdentifierPreparer) assert issubclass(dialect.statement_compiler, FireboltCompiler) - # SQLAlchemy's DefaultDialect creates an instance of type_compiler behind the scenes + # SQLAlchemy's DefaultDialect creates an instance of + # type_compiler behind the scenes assert isinstance(dialect.type_compiler, FireboltTypeCompiler) assert dialect.context == {} From d72d1635b09ff42d33a8b6f51e8c44c588713fc3 Mon Sep 17 00:00:00 2001 From: Petro Date: Thu, 2 Dec 2021 08:33:39 +0000 Subject: [PATCH 6/7] moving install requires --- setup.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 3afe4e1..f78531d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,14 +23,14 @@ download_url = https://github.com/firebolt-db/firebolt-sqlalchemy/archive/refs/t entry_points = sqlalchemy.dialects : firebolt = firebolt_db.firebolt_dialect:FireboltDialect -install_requires = - sqlalchemy>=1.0.0 - firebolt-sdk project_urls = Bug Tracker = https://github.com/firebolt-db/firebolt-sqlalchemy [options] packages = find: +install_requires = + firebolt-sdk + sqlalchemy>=1.0.0 python_requires = >=3.6 package_dir = = src From f1a881c1871253d26f7d79209ef7c55e7f1926ee Mon Sep 17 00:00:00 2001 From: Petro Date: Fri, 3 Dec 2021 14:54:15 +0000 Subject: [PATCH 7/7] Adding temporary mypy override before sdk release --- .github/workflows/python-app.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 328faf2..e882995 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -35,7 +35,8 @@ jobs: - name: Type check with mypy run: | - mypy src + # Remove missing import override after firebolt-sdk release 0.2.1+ + mypy --ignore-missing-imports src - name: Test with pytest run: |