From 4744fb3ebd3c8b8a847c569d50fa67c28c18bc2b Mon Sep 17 00:00:00 2001 From: jonavellecuerdo Date: Wed, 20 Sep 2023 13:35:12 -0400 Subject: [PATCH 1/2] IN-923 Raise exceptions for failed connection tests Why these changes are being introduced: * Raising exceptions for any failed connection tests stops the application from proceeding with the workflow once a connection test fails. This logic makes sense as the (1) a successful connection to the Data Warehouse is required for any Carbon run and (2) a successful connection to the Elements FTP server is required for any run Carbon run that involves FTP. How this addresses that need: * Exceptions are raised in the connection test functions of carbon.database.DatabaseEngine and carbon.app.DatabaseToFtpPipe. * CLI tests for failed connection tests are properly configured. Side effects of this change: * None Relevant ticket(s): * https://mitlibraries.atlassian.net/browse/IN-923 --- carbon/app.py | 1 + carbon/database.py | 1 + tests/test_cli.py | 20 ++++++++++++++------ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/carbon/app.py b/carbon/app.py index 83e1af4..5879475 100644 --- a/carbon/app.py +++ b/carbon/app.py @@ -244,6 +244,7 @@ def run_connection_test(self) -> None: f"Failed to connect to the Symplectic Elements FTP server: {error}" ) logger.exception(error_message) + raise else: logger.info("Successfully connected to the Symplectic Elements FTP server") ftps.quit() diff --git a/carbon/database.py b/carbon/database.py index b36f8cf..49e45ad 100644 --- a/carbon/database.py +++ b/carbon/database.py @@ -114,6 +114,7 @@ def run_connection_test(self) -> None: except Exception as error: error_message = f"Failed to connect to the Data Warehouse: {error}" logger.exception(error_message) + raise else: dbapi_connection = connection.connection version = ( diff --git a/tests/test_cli.py b/tests/test_cli.py index d5a218e..6f521fd 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -188,11 +188,19 @@ def test_cli_connection_tests_success(caplog, functional_engine, runner): assert "Successfully connected to the Symplectic Elements FTP server" in caplog.text -def test_cli_connection_tests_fail( - caplog, ftp_server, monkeypatch, nonfunctional_engine, runner +def test_cli_database_connection_test_fails(caplog, nonfunctional_engine, runner): + with patch("carbon.cli.DatabaseEngine") as mocked_engine: + mocked_engine.return_value = nonfunctional_engine + result = runner.invoke(main, ["--run_connection_tests"]) + assert result.exit_code == 1 + + assert "Failed to connect to the Data Warehouse" in caplog.text + + +def test_cli_ftp_connection_test_fails( + caplog, ftp_server, functional_engine, monkeypatch, runner ): ftp_socket, _ = ftp_server - monkeypatch.setenv( "SYMPLECTIC_FTP_JSON", ( @@ -202,10 +210,10 @@ def test_cli_connection_tests_fail( '"SYMPLECTIC_FTP_PASS": "invalid_password"}' ), ) + with patch("carbon.cli.DatabaseEngine") as mocked_engine: - mocked_engine.return_value = nonfunctional_engine + mocked_engine.return_value = functional_engine result = runner.invoke(main, ["--run_connection_tests"]) - assert result.exit_code == 0 + assert result.exit_code == 1 - assert "Failed to connect to the Data Warehouse" in caplog.text assert "Failed to connect to the Symplectic Elements FTP server" in caplog.text From 5b758b553337aeba195d976e3a4008265664955d Mon Sep 17 00:00:00 2001 From: jonavellecuerdo Date: Wed, 20 Sep 2023 15:00:25 -0400 Subject: [PATCH 2/2] IN-925 Add Makefile commands for running 'people' and 'articles' feed Why these changes are being introduced: * Provide an easy way to execute runs of the 'people' and 'articles' feed as an ECS task. How this addresses that need: * Add Makefile commands to run 'people' and 'articles' feed Side effects of this change: * None Relevant ticket(s): * https://mitlibraries.atlassian.net/browse/IN-925 --- Makefile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ffe4100..b65d13e 100644 --- a/Makefile +++ b/Makefile @@ -104,4 +104,10 @@ run-connection-tests-with-docker: # run connection tests from local docker insta docker run -v ./.env:/.env carbon-dev --run_connection_tests run-connection-tests-with-ecs-stage: # use after the Data Warehouse password is changed every year to confirm that the new password works - aws ecs run-task --cluster carbon-ecs-stage --task-definition carbon-ecs-stage-people --launch-type="FARGATE" --region us-east-1 --network-configuration '{"awsvpcConfiguration": {"subnets": ["subnet-05df31ac28dd1a4b0","subnet-04cfa272d4f41dc8a"], "securityGroups": ["sg-0f11e2619db7da196"],"assignPublicIp": "DISABLED"}}' --overrides '{"containerOverrides": [ {"name": "carbon-ecs-stage", "command": ["--run_connection_tests"]}]}' + aws ecs run-task --cluster carbon-ecs-stage --task-definition carbon-ecs-stage-people --launch-type="FARGATE" --region us-east-1 --network-configuration '{"awsvpcConfiguration": {"subnets": ["subnet-05df31ac28dd1a4b0","subnet-04cfa272d4f41dc8a"], "securityGroups": ["sg-0f11e2619db7da196"],"assignPublicIp": "DISABLED"}}' --overrides '{"containerOverrides": [ {"name": "carbon-ecs-stage", "command": ["--run_connection_tests", "--ignore_sns_logging"]}]}' + +run-articles-feed-with-ecs-stage: # run 'articles' feed + aws ecs run-task --cluster carbon-ecs-stage --task-definition carbon-ecs-stage-articles --launch-type="FARGATE" --region us-east-1 --network-configuration '{"awsvpcConfiguration": {"subnets": ["subnet-05df31ac28dd1a4b0","subnet-04cfa272d4f41dc8a"], "securityGroups": ["sg-0f11e2619db7da196"],"assignPublicIp": "DISABLED"}}' --overrides '{"containerOverrides": [ {"name": "carbon-ecs-stage", "command": ["--ignore_sns_logging"]}]}' + +run-people-feed-with-ecs-stage: # run 'people' feed + aws ecs run-task --cluster carbon-ecs-stage --task-definition carbon-ecs-stage-people --launch-type="FARGATE" --region us-east-1 --network-configuration '{"awsvpcConfiguration": {"subnets": ["subnet-05df31ac28dd1a4b0","subnet-04cfa272d4f41dc8a"], "securityGroups": ["sg-0f11e2619db7da196"],"assignPublicIp": "DISABLED"}}' --overrides '{"containerOverrides": [ {"name": "carbon-ecs-stage", "command": ["--ignore_sns_logging"]}]}' \ No newline at end of file