From 00351067b0a756f09343b37af02e11440c6aa109 Mon Sep 17 00:00:00 2001 From: Anton Kukushkin <3997468+kukushking@users.noreply.github.com> Date: Mon, 18 Jul 2022 12:04:24 +0100 Subject: [PATCH 1/4] rds data api - rethrow exception from the client --- awswrangler/data_api/rds.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/awswrangler/data_api/rds.py b/awswrangler/data_api/rds.py index 0cbc0b590..cfb17218c 100644 --- a/awswrangler/data_api/rds.py +++ b/awswrangler/data_api/rds.py @@ -88,9 +88,7 @@ def _execute_statement(self, sql: str, database: Optional[str] = None) -> str: if response is None: self.logger.exception("Maximum BadRequestException retries reached for query %s", sql) - raise self.client.exceptions.BadRequestException( - f"Query failed - BadRequestException received after {total_tries} tries and sleeping {total_sleep}s" - ) from last_exception + raise last_exception request_id: str = uuid.uuid4().hex self.results[request_id] = response From 50be3feb3a73cb57acadb765be61e17ad09fc526 Mon Sep 17 00:00:00 2001 From: Anton Kukushkin <3997468+kukushking@users.noreply.github.com> Date: Mon, 18 Jul 2022 12:04:48 +0100 Subject: [PATCH 2/4] test infra - optional db parameters --- tests/conftest.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9300b107c..6bbdd74df 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -130,32 +130,32 @@ def workgroup3(bucket, kms_key): @pytest.fixture(scope="session") def databases_parameters(cloudformation_outputs, db_password): parameters = dict(postgresql={}, mysql={}, redshift={}, sqlserver={}, mysql_serverless={}, oracle={}) - parameters["postgresql"]["host"] = cloudformation_outputs["PostgresqlAddress"] + parameters["postgresql"]["host"] = cloudformation_outputs.get("PostgresqlAddress") parameters["postgresql"]["port"] = 3306 parameters["postgresql"]["schema"] = "public" parameters["postgresql"]["database"] = "postgres" - parameters["mysql"]["host"] = cloudformation_outputs["MysqlAddress"] + parameters["mysql"]["host"] = cloudformation_outputs.get("MysqlAddress") parameters["mysql"]["port"] = 3306 parameters["mysql"]["schema"] = "test" parameters["mysql"]["database"] = "test" - parameters["redshift"]["secret_arn"] = cloudformation_outputs["RedshiftSecretArn"] - parameters["redshift"]["host"] = cloudformation_outputs["RedshiftAddress"] - parameters["redshift"]["port"] = cloudformation_outputs["RedshiftPort"] - parameters["redshift"]["identifier"] = cloudformation_outputs["RedshiftIdentifier"] + parameters["redshift"]["secret_arn"] = cloudformation_outputs.get("RedshiftSecretArn") + parameters["redshift"]["host"] = cloudformation_outputs.get("RedshiftAddress") + parameters["redshift"]["port"] = cloudformation_outputs.get("RedshiftPort") + parameters["redshift"]["identifier"] = cloudformation_outputs.get("RedshiftIdentifier") parameters["redshift"]["schema"] = "public" parameters["redshift"]["database"] = "test" - parameters["redshift"]["role"] = cloudformation_outputs["RedshiftRole"] + parameters["redshift"]["role"] = cloudformation_outputs.get("RedshiftRole") parameters["password"] = db_password parameters["user"] = "test" - parameters["sqlserver"]["host"] = cloudformation_outputs["SqlServerAddress"] + parameters["sqlserver"]["host"] = cloudformation_outputs.get("SqlServerAddress") parameters["sqlserver"]["port"] = 1433 parameters["sqlserver"]["schema"] = "dbo" parameters["sqlserver"]["database"] = "test" - parameters["mysql_serverless"]["secret_arn"] = cloudformation_outputs["MysqlServerlessSecretArn"] + parameters["mysql_serverless"]["secret_arn"] = cloudformation_outputs.get("MysqlServerlessSecretArn") parameters["mysql_serverless"]["schema"] = "test" parameters["mysql_serverless"]["database"] = "test" - parameters["mysql_serverless"]["arn"] = cloudformation_outputs["MysqlServerlessClusterArn"] - parameters["oracle"]["host"] = cloudformation_outputs["OracleAddress"] + parameters["mysql_serverless"]["arn"] = cloudformation_outputs.get("MysqlServerlessClusterArn") + parameters["oracle"]["host"] = cloudformation_outputs.get("OracleAddress") parameters["oracle"]["port"] = 1521 parameters["oracle"]["schema"] = "TEST" parameters["oracle"]["database"] = "ORCL" From b9f8a6e0c995d02674413479ebf10647f422d72b Mon Sep 17 00:00:00 2001 From: Anton Kukushkin <3997468+kukushking@users.noreply.github.com> Date: Mon, 18 Jul 2022 12:11:42 +0100 Subject: [PATCH 3/4] test case --- tests/test_data_api.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_data_api.py b/tests/test_data_api.py index 5f36db9ce..19a7390dc 100644 --- a/tests/test_data_api.py +++ b/tests/test_data_api.py @@ -1,4 +1,5 @@ import boto3 +import botocore import pandas as pd import pytest @@ -129,3 +130,10 @@ def test_data_api_mysql_column_subset_select(mysql_serverless_connector, mysql_s ) expected_dataframe = pd.DataFrame([["test"]], columns=["name"]) pd.testing.assert_frame_equal(dataframe, expected_dataframe) + + +def test_data_api_exception(mysql_serverless_connector, mysql_serverless_table): + with pytest.raises(boto3.client("rds-data").exceptions.BadRequestException): + wr.data_api.rds.read_sql_query( + f"CUPCAKE", con=mysql_serverless_connector + ) From 7917504e056c63aa07a337065143d9994cc59b53 Mon Sep 17 00:00:00 2001 From: Anton Kukushkin <3997468+kukushking@users.noreply.github.com> Date: Mon, 18 Jul 2022 12:24:44 +0100 Subject: [PATCH 4/4] mypy --- awswrangler/data_api/rds.py | 2 +- tests/test_data_api.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/awswrangler/data_api/rds.py b/awswrangler/data_api/rds.py index cfb17218c..55b2e719e 100644 --- a/awswrangler/data_api/rds.py +++ b/awswrangler/data_api/rds.py @@ -88,7 +88,7 @@ def _execute_statement(self, sql: str, database: Optional[str] = None) -> str: if response is None: self.logger.exception("Maximum BadRequestException retries reached for query %s", sql) - raise last_exception + raise last_exception # type: ignore request_id: str = uuid.uuid4().hex self.results[request_id] = response diff --git a/tests/test_data_api.py b/tests/test_data_api.py index 19a7390dc..4130029ea 100644 --- a/tests/test_data_api.py +++ b/tests/test_data_api.py @@ -1,5 +1,4 @@ import boto3 -import botocore import pandas as pd import pytest @@ -134,6 +133,4 @@ def test_data_api_mysql_column_subset_select(mysql_serverless_connector, mysql_s def test_data_api_exception(mysql_serverless_connector, mysql_serverless_table): with pytest.raises(boto3.client("rds-data").exceptions.BadRequestException): - wr.data_api.rds.read_sql_query( - f"CUPCAKE", con=mysql_serverless_connector - ) + wr.data_api.rds.read_sql_query("CUPCAKE", con=mysql_serverless_connector)