From 33dfee8f6baf8c112d9c60c330865bb5aaa2a386 Mon Sep 17 00:00:00 2001 From: Apurva Anand Date: Fri, 24 Sep 2021 23:03:33 +0530 Subject: [PATCH 1/7] Revised and new pytest cases --- tests/test_fireboltconnector.py | 101 +++++++++++++++++++++++--------- tests/test_fireboltdialect.py | 75 ++++++++++++++++++------ 2 files changed, 129 insertions(+), 47 deletions(-) diff --git a/tests/test_fireboltconnector.py b/tests/test_fireboltconnector.py index 2d2621b..b3fca76 100644 --- a/tests/test_fireboltconnector.py +++ b/tests/test_fireboltconnector.py @@ -8,7 +8,7 @@ @pytest.fixture def get_connection(): - return firebolt_connector.connect('aapurva@sigmoidanalytics.com', 'Apurva111', 'Sigmoid_Alchemy') + return firebolt_connector.connect('localhost', 8123, 'aapurva@sigmoidanalytics.com', 'Apurva111', 'Sigmoid_Alchemy') class TestConnect: @@ -17,7 +17,9 @@ def test_connect_success(self): user_email = "aapurva@sigmoidanalytics.com" password = "Apurva111" db_name = "Sigmoid_Alchemy" - connection = firebolt_connector.connect(user_email, password, db_name) + host = "localhost" + port = "8123" + connection = firebolt_connector.connect(host, port, user_email, password, db_name) assert connection.access_token assert connection.engine_url @@ -25,19 +27,23 @@ def test_connect_invalid_credentials(self): user_email = "aapurva@sigmoidanalytics.com" password = "wrongpassword" db_name = "Sigmoid_Alchemy" + host = "localhost" + port = "8123" with pytest.raises(exceptions.InvalidCredentialsError): - firebolt_connector.connect(user_email, password, db_name) + firebolt_connector.connect(host, port, user_email, password, db_name) def test_connect_invalid_database(self): user_email = "aapurva@sigmoidanalytics.com" password = "Apurva111" db_name = "wrongdatabase" + host = "localhost" + port = "8123" with pytest.raises(exceptions.SchemaNotFoundError): - firebolt_connector.connect(user_email, password, db_name) + firebolt_connector.connect(host, port, user_email, password, db_name) def test_get_description_from_row_valid_rows(): - row = {'id': 1, 'name': 'John', 'is_eligible': True} + row = {'id': 1, 'name': 'John', 'is_eligible': True, 'some_array': [2, 4]} result = firebolt_connector.get_description_from_row(row) assert result[0][0] == 'id' assert result[0][1] == firebolt_connector.Type.NUMBER @@ -48,10 +54,13 @@ def test_get_description_from_row_valid_rows(): assert result[2][0] == 'is_eligible' assert result[2][1] == firebolt_connector.Type.BOOLEAN assert not result[2][6] + assert result[3][0] == 'some_array' + assert result[3][1] == firebolt_connector.Type.ARRAY + assert not result[3][6] def test_get_description_from_row_invalid_rows(): - row = {'id': []} + row = {'id': {}} with pytest.raises(Exception): firebolt_connector.get_description_from_row(row) @@ -68,8 +77,13 @@ def test_get_type(): assert firebolt_connector.get_type(value_2_2) == 2 assert firebolt_connector.get_type(value_3_1) == 3 assert firebolt_connector.get_type(value_3_2) == 3 + assert firebolt_connector.get_type(value_4) == 4 + + +def test_get_type_invalid_type(): + value = {} with pytest.raises(Exception): - firebolt_connector.get_type(value_4) + firebolt_connector.get_type(value) class TestConnection: @@ -81,11 +95,11 @@ def test_cursor(self, get_connection): assert len(connection.cursors) > 0 assert type(cursor) == firebolt_connector.Cursor - def test_execute(self, get_connection): - connection = get_connection - query = "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.DATABASES" - cursor = connection.execute(query) - assert type(cursor._results) == itertools.chain + # def test_execute(self, get_connection): + # connection = get_connection + # query = "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.DATABASES" + # cursor = connection.execute(query) + # assert type(cursor._results) == itertools.chain def test_commit(self): pass @@ -106,25 +120,54 @@ def test_rowcount(self, get_connection): cursor = connection.cursor().execute(query) assert cursor.rowcount == 10 + def test_close(self, get_connection): + connection = get_connection + cursor = connection.cursor() + if not cursor.closed: + cursor.close() + assert cursor.closed - def test_close(self): - pass - - def test_execute(self): - pass - - def test_stream_query(self): - pass - - def test_fetchone(self): - pass + def test_execute(self, get_connection): + query = 'select * from lineitem ' \ + 'where l_orderkey=3184321 and l_partkey=65945' + connection = get_connection + cursor = connection.cursor() + assert not cursor._results + cursor.execute(query) + assert cursor.rowcount == 1 - def test_fetchmany(self): - pass + def test_executemany(self, get_connection): + query = "select * from lineitem limit 10" + connection = get_connection + cursor = connection.cursor() + with pytest.raises(exceptions.NotSupportedError): + cursor.executemany(query) - def test_fetchall(self): - pass + def test_fetchone(self, get_connection): + query = "select * from lineitem limit 10" + connection = get_connection + cursor = connection.cursor() + assert not cursor._results + cursor.execute(query) + result = cursor.fetchone() + assert isinstance(result, tuple) + def test_fetchmany(self, get_connection): + query = "select * from lineitem limit 10" + connection = get_connection + cursor = connection.cursor() + assert not cursor._results + cursor.execute(query) + result = cursor.fetchmany(3) + assert isinstance(result, list) + assert len(result) == 3 -def test_rows_from_chunks(): - pass + def test_fetchall(self, get_connection): + query = "select * from lineitem limit 10" + connection = get_connection + cursor = connection.cursor() + assert not cursor._results + cursor.execute(query) + result = cursor.fetchall() + assert isinstance(result, list) + assert len(result) == 10 diff --git a/tests/test_fireboltdialect.py b/tests/test_fireboltdialect.py index 674de9d..315218e 100644 --- a/tests/test_fireboltdialect.py +++ b/tests/test_fireboltdialect.py @@ -1,29 +1,68 @@ -from firebolt_db import firebolt_dialect, firebolt_connector +import pytest +from firebolt_db import firebolt_dialect +from sqlalchemy.engine import url +from sqlalchemy import create_engine -class ConnectionDetails(): - user_email = "aapurva@sigmoidanalytics.com" - password = "Apurva111" - db_name = "Sigmoid_Alchemy" - connection = firebolt_connector.connect(user_email, password, db_name) - fireboltDialect = firebolt_dialect.FireboltDialect -class TestFireboltDialect: +@pytest.fixture +def get_engine(): + return create_engine("firebolt://aapurva@sigmoidanalytics.com:Apurva111@host/Sigmoid_Alchemy") + + +dialect = firebolt_dialect.FireboltDialect() +class TestFireboltDialect: def test_create_connect_args(self): - None + connection_url = "test_engine://test_user@email:test_password@test_host_name/test_db_name" + u = url.make_url(connection_url) + result_list, result_dict = dialect.create_connect_args(u) + assert result_dict["host"] == "test_host_name" + assert result_dict["port"] == 5432 + assert result_dict["username"] == "test_user@email" + assert result_dict["password"] == "test_password" + assert result_dict["db_name"] == "test_db_name" + assert result_dict["context"] == {} + assert not result_dict["header"] + + def test_get_schema_names(self, get_engine): + engine = get_engine + results = dialect.get_schema_names(engine) + assert 'Sigmoid_Alchemy' in results + + def test_has_table(self, get_engine): + table = 'lineitem' + schema = 'Sigmoid_Alchemy' + engine = get_engine + results = dialect.has_table(engine, table, schema) + assert results == 1 - def test_get_schema_names(self): - # result = fireboltDialect.get_schema_names() - None + def test_get_table_names(self, get_engine): + schema = 'Sigmoid_Alchemy' + engine = get_engine + results = dialect.get_table_names(engine, schema) + assert len(results) > 0 - def test_has_table(self): - None + def test_get_columns(self, get_engine): + table = 'lineitem' + schema = 'Sigmoid_Alchemy' + engine = get_engine + results = dialect.get_columns(engine, table, schema) + assert len(results) > 0 + row = results[0] + assert isinstance(row, dict) + row_keys = list(row.keys()) + assert row_keys[0] == "name" + assert row_keys[1] == "type" + assert row_keys[2] == "nullable" + assert row_keys[3] == "default" - def test_get_table_names(self): - None - def get_columns(self): - None \ No newline at end of file +def test_get_is_nullable(): + assert firebolt_dialect.get_is_nullable("YES") + assert firebolt_dialect.get_is_nullable("yes") + assert not firebolt_dialect.get_is_nullable("NO") + assert not firebolt_dialect.get_is_nullable("no") + assert not firebolt_dialect.get_is_nullable("ABC") From 4635510913048d61652fe0d5ea77790529c761ed Mon Sep 17 00:00:00 2001 From: raghavsharma Date: Mon, 27 Sep 2021 13:27:07 +0530 Subject: [PATCH 2/7] Added code fix for token expiry --- src/firebolt_db/firebolt_api_service.py | 2 +- src/firebolt_db/firebolt_connector.py | 3 ++- tests/test_firebolt_api_service.py | 19 +++++++++++++------ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/firebolt_db/firebolt_api_service.py b/src/firebolt_db/firebolt_api_service.py index caf1925..81e9d90 100644 --- a/src/firebolt_db/firebolt_api_service.py +++ b/src/firebolt_db/firebolt_api_service.py @@ -12,7 +12,7 @@ class FireboltApiService: @staticmethod @memoized - def get_connection(user_email, password, db_name): + def get_connection(user_email, password, db_name, date): """ Retrieve Authorisation details for connection This method internally calls methods to get access token, refresh token and engine URL. diff --git a/src/firebolt_db/firebolt_connector.py b/src/firebolt_db/firebolt_connector.py index 88cce71..37def62 100644 --- a/src/firebolt_db/firebolt_connector.py +++ b/src/firebolt_db/firebolt_connector.py @@ -10,6 +10,7 @@ import itertools import json from collections import namedtuple, OrderedDict +from datetime import date from firebolt_db.firebolt_api_service import FireboltApiService from firebolt_db import exceptions @@ -122,7 +123,7 @@ def __init__(self, self._username = username self._password = password self._db_name = db_name - connection_details = FireboltApiService.get_connection(username, password, db_name) + connection_details = FireboltApiService.get_connection(username, password, db_name, date.today()) self.access_token = connection_details[0] self.engine_url = connection_details[1] diff --git a/tests/test_firebolt_api_service.py b/tests/test_firebolt_api_service.py index 869562c..12a4bc9 100644 --- a/tests/test_firebolt_api_service.py +++ b/tests/test_firebolt_api_service.py @@ -1,3 +1,5 @@ +from datetime import date + from firebolt_db.firebolt_api_service import FireboltApiService from tests import constants from requests.exceptions import HTTPError @@ -12,7 +14,8 @@ class TestFireboltApiService: def test_get_connection_success(self): - response = FireboltApiService.get_connection(constants.username, constants.password, constants.db_name) + response = FireboltApiService.get_connection(constants.username, constants.password, + constants.db_name, date.today()) if type(response) == HTTPError: assert response.response.status_code == 503 else: @@ -20,11 +23,11 @@ def test_get_connection_success(self): def test_get_connection_invalid_credentials(self): with pytest.raises(Exception) as e_info: - response = FireboltApiService.get_connection('username', 'password', constants.db_name)[0] + response = FireboltApiService.get_connection('username', 'password', constants.db_name, date.today())[0] def test_get_connection_invalid_schema_name(self): with pytest.raises(Exception) as e_info: - response = FireboltApiService.get_connection(constants.username, constants.password, 'db_name')[1] + response = FireboltApiService.get_connection(constants.username, constants.password, 'db_name', date.today())[1] def test_get_access_token_success(self): assert access_token["access_token"] != "" @@ -71,9 +74,13 @@ def test_run_query_invalid_schema(self): engine_url, 'db_name', constants.query) def test_run_query_invalid_header(self): - with pytest.raises(Exception) as e_info: - response = FireboltApiService.run_query('header', access_token["refresh_token"], engine_url, constants.db_name, - constants.query) != {} + try: + response = FireboltApiService.run_query('header', access_token["refresh_token"], + engine_url, constants.db_name, + constants.query) + assert response != "" + except exceptions.InternalError as e_info: + assert e_info != "" def test_run_query_invalid_query(self): with pytest.raises(Exception) as e_info: From 2a8aa32ab2a41ce2e0e0111eaa9dff6db283ca53 Mon Sep 17 00:00:00 2001 From: raghavsharma Date: Tue, 28 Sep 2021 17:29:19 +0530 Subject: [PATCH 3/7] Changes as per pypi registration --- README.md | 6 ++---- setup.py | 4 ++-- src/firebolt_db/firebolt_api_service.py | 8 +------- temp_test_script.py | 7 +++++++ 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 4b717ca..f2c4811 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,3 @@ -# firebolt-sqlalchemy 0.0.1 +# sqlalchemy_adapter -This is a simple example package. You can use -[Github-flavored Markdown](https://guides.github.com/features/mastering-markdown/) -to write your content. \ No newline at end of file +This is the 'alpha' package. Expect updates in future. \ No newline at end of file diff --git a/setup.py b/setup.py index 5b3e40b..35d0435 100644 --- a/setup.py +++ b/setup.py @@ -4,11 +4,11 @@ long_description = fh.read() setuptools.setup( - name="firebolt-sqlalchemy", + name="sqlalchemy_adapter", version="0.0.1", author="Raghav Sharma", author_email="raghavs@sigmoidanalytics.com", - description="Package for Sqlalchemy adapter for Firebolt-Superset integration", + description="Sqlalchemy adapter for Firebolt", long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/raghavSharmaSigmoid/sqlalchemy_adapter", diff --git a/src/firebolt_db/firebolt_api_service.py b/src/firebolt_db/firebolt_api_service.py index 81e9d90..006f7ee 100644 --- a/src/firebolt_db/firebolt_api_service.py +++ b/src/firebolt_db/firebolt_api_service.py @@ -202,13 +202,7 @@ def run_query(access_token, refresh_token, engine_url, db_name, query): query_response.raise_for_status() except HTTPError as http_err: - if http_err.response.status_code == 401: - access_token = FireboltApiService.get_access_token_via_refresh(refresh_token) - header = {'Authorization': "Bearer " + access_token} - query_response = requests.post(url="https://" + engine_url, params={'database': db_name}, - headers=header, files={"query": (None, query)}) - else: - payload = { + payload = { "error": "DB-API Exception", "errorMessage": http_err.response.text, } diff --git a/temp_test_script.py b/temp_test_script.py index 83d7887..1707be7 100644 --- a/temp_test_script.py +++ b/temp_test_script.py @@ -8,6 +8,13 @@ # connection = connect('localhost',8123,'aapurva@sigmoidanalytics.com', 'Apurva111', 'Sigmoid_Alchemy') +# if http_err.response.status_code == 401: + # access_token = FireboltApiService.get_access_token_via_refresh(refresh_token) + # header = {'Authorization': "Bearer " + access_token} + # query_response = requests.post(url="https://" + engine_url, params={'database': db_name}, + # headers=header, files={"query": (None, query)}) + # else: + # Test for end to end # query = 'select * from lineitem limit 10' # cursor = connection.cursor() From afad34f5855eb06f0fd1b815c7464bd87459f77f Mon Sep 17 00:00:00 2001 From: raghavsharma Date: Tue, 28 Sep 2021 17:42:08 +0530 Subject: [PATCH 4/7] Changed package name to 'firebolt-sqlalchemy' --- README.md | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f2c4811..019ece2 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,3 @@ -# sqlalchemy_adapter +# firebolt-sqlalchemy This is the 'alpha' package. Expect updates in future. \ No newline at end of file diff --git a/setup.py b/setup.py index 35d0435..f5a8429 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ long_description = fh.read() setuptools.setup( - name="sqlalchemy_adapter", + name="firebolt-sqlalchemy", version="0.0.1", author="Raghav Sharma", author_email="raghavs@sigmoidanalytics.com", From a45319c45cfed7b88e569a1c27a20c884bc40e05 Mon Sep 17 00:00:00 2001 From: raghavsharma Date: Tue, 28 Sep 2021 17:59:23 +0530 Subject: [PATCH 5/7] Changed repo name to 'firebolt-sqlalchemy' and added setup.cfg --- setup.cfg | 24 ++++++++++++++++++++++++ setup.py | 4 ++-- 2 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..39c0acb --- /dev/null +++ b/setup.cfg @@ -0,0 +1,24 @@ +[metadata] +name = firebolt-sqlalchemy +version = 0.0.1 +author = Raghav Sharma +author_email = raghavs@sigmoidanalytics.com +description = Sqlalchemy adapter for Firebolt +long_description = file: README.md +long_description_content_type = text/markdown +url = https://github.com/raghavSharmaSigmoid/firebolt-sqlalchemy +project_urls = + Bug Tracker = https://github.com/raghavSharmaSigmoid/firebolt-sqlalchemy +classifiers = + Programming Language :: Python :: 3 + License :: OSI Approved :: MIT License + Operating System :: OS Independent + +[options] +package_dir = + = src +packages = find: +python_requires = >=3.6 + +[options.packages.find] +where = src \ No newline at end of file diff --git a/setup.py b/setup.py index f5a8429..8c7c707 100644 --- a/setup.py +++ b/setup.py @@ -11,9 +11,9 @@ description="Sqlalchemy adapter for Firebolt", long_description=long_description, long_description_content_type="text/markdown", - url="https://github.com/raghavSharmaSigmoid/sqlalchemy_adapter", + url="https://github.com/raghavSharmaSigmoid/firebolt-sqlalchemy", project_urls={ - "Bug Tracker": "https://github.com/raghavSharmaSigmoid/sqlalchemy_adapter", + "Bug Tracker": "https://github.com/raghavSharmaSigmoid/firebolt-sqlalchemy", }, install_requires=[ 'sqlalchemy>=1.0.0', From 229ea1beb97eae6257b9d5c15a1859977f95ff97 Mon Sep 17 00:00:00 2001 From: raghavsharma Date: Tue, 28 Sep 2021 18:46:38 +0530 Subject: [PATCH 6/7] Added necessary imports under 'install_requires' in setup.py --- setup.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/setup.py b/setup.py index 8c7c707..304c4cc 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,11 @@ install_requires=[ 'sqlalchemy>=1.0.0', "requests" + "json" + "itertools" + "collections" + "datetime" + "functools" ], entry_points={ "sqlalchemy.dialects": [ From 22b08edb02787619595a101b55fd33473d00a5ad Mon Sep 17 00:00:00 2001 From: raghavsharma Date: Tue, 28 Sep 2021 18:55:54 +0530 Subject: [PATCH 7/7] removed temp_test_script.py --- temp_test_script.py | 61 --------------------------------------------- 1 file changed, 61 deletions(-) delete mode 100644 temp_test_script.py diff --git a/temp_test_script.py b/temp_test_script.py deleted file mode 100644 index 1707be7..0000000 --- a/temp_test_script.py +++ /dev/null @@ -1,61 +0,0 @@ -# Temporary file to share manual testing code -# This is a temporary test file to test end to end function of adapter -# To use this file, copy this file to a folder outside firebolt_db -# Comment out the type of test you want to run - -# from firebolt_db.firebolt_connector import connect -from firebolt_db.firebolt_dialect import FireboltDialect - -# connection = connect('localhost',8123,'aapurva@sigmoidanalytics.com', 'Apurva111', 'Sigmoid_Alchemy') - -# if http_err.response.status_code == 401: - # access_token = FireboltApiService.get_access_token_via_refresh(refresh_token) - # header = {'Authorization': "Bearer " + access_token} - # query_response = requests.post(url="https://" + engine_url, params={'database': db_name}, - # headers=header, files={"query": (None, query)}) - # else: - -# Test for end to end -# query = 'select * from lineitem limit 10' -# cursor = connection.cursor() -# response = cursor.execute(query) -# # print(response.fetchmany(3)) -# # print(response.fetchone()) -# print(response.fetchall()) - -# Test for dialect -# dialect = FireboltDialect() - -# schemas = dialect.get_schema_names(connection) -# print("Schema Names") -# print(schemas.fetchone()) - -# tables = dialect.get_table_names(connection,"Sigmoid_Alchemy").fetchall() -# print("Table names") -# print(tables) - - -from sqlalchemy import create_engine -from sqlalchemy.dialects import registry -registry.register("firebolt", "src.firebolt_db.firebolt_dialect", "FireboltDialect") - -engine = create_engine("firebolt://aapurva@sigmoidanalytics.com:Apurva111@host/Sigmoid_Alchemy") -print(type(engine)) - -dialect = FireboltDialect() - -schemas = dialect.get_schema_names(engine) -print(schemas) -# print("Schema Names") -# print(schemas.fetchone()) - -schemas = dialect.get_table_names(engine,"Sigmoid_Alchemy") -print(schemas) - -schemas = dialect.get_columns(engine,"lineitem","Sigmoid_Alchemy") -print(schemas) - -connection = engine.connect() -schemas = dialect.get_schema_names(connection) -print(schemas) -