From 3aabd92137507622b4cd8c4bd29f613bc6478455 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 31 Aug 2021 11:48:03 +0300 Subject: [PATCH] Fix facebook marketing source SAT fail on invalid config file (#5621) * Fix facebook marketing source SAT fail on invalid config file --- .pre-commit-config.yaml | 8 ++++---- airbyte-cdk/python/CHANGELOG.md | 3 +++ .../python/airbyte_cdk/sources/utils/schema_helpers.py | 7 ++----- airbyte-cdk/python/setup.py | 2 +- airbyte-cdk/python/unit_tests/test_entrypoint.py | 4 ++-- .../source_acceptance_test/tests/test_core.py | 2 +- .../source-facebook-marketing/acceptance-test-config.yml | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8392679c8a476..ce71aaf602a3e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,13 +6,13 @@ repos: args: ["--tmpl=LICENSE", "--ext=py", "-f"] - repo: https://github.com/ambv/black - rev: stable + rev: 20.8b1 hooks: - id: black args: ["--line-length=140"] - repo: https://github.com/timothycrosley/isort - rev: 4.3.21 + rev: 5.6.4 hooks: - id: isort args: ["--settings-path=tools/python/.isort.cfg"] @@ -24,8 +24,8 @@ repos: types_or: [yaml, json] exclude: ^.github/ - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v1.2.3 + - repo: https://gitlab.com/pycqa/flake8 + rev: 3.8.4 hooks: - id: flake8 args: ["--config=tools/python/.flake8"] diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index e8a1e8e920880..fec3fe46cc109 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.1.14 +If the input config file does not comply with spec schema, raise an exception instead of `system.exit`. + ## 0.1.13 Fix defect with user defined backoff time retry attempts, number of retries logic fixed diff --git a/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py b/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py index 8c3b96105d4d8..5e4a6d91cf841 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py @@ -26,7 +26,6 @@ import json import os import pkgutil -import sys from typing import Any, Dict, Mapping import pkg_resources @@ -133,8 +132,7 @@ def get_schema(self, name: str) -> dict: def check_config_against_spec_or_exit(config: Mapping[str, Any], spec: ConnectorSpecification, logger: AirbyteLogger): """ Check config object against spec. In case of spec is invalid, throws - SystemExit exception causing application to make system exit call with - errorcode 1 + an exception with validation error description. :param config - config loaded from file specified over command line :param spec - spec object generated by connector :param logger - Airbyte logger for reporting validation error @@ -143,5 +141,4 @@ def check_config_against_spec_or_exit(config: Mapping[str, Any], spec: Connector try: validate(instance=config, schema=spec_schema) except ValidationError as validation_error: - logger.error("Config validation error: " + validation_error.message) - sys.exit(1) + raise Exception("Config validation error: " + validation_error.message) diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 38a119df0e502..5a32d2fc21d3c 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -35,7 +35,7 @@ setup( name="airbyte-cdk", - version="0.1.13", + version="0.1.14", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", diff --git a/airbyte-cdk/python/unit_tests/test_entrypoint.py b/airbyte-cdk/python/unit_tests/test_entrypoint.py index efe1fef51628e..82c219cd71f84 100644 --- a/airbyte-cdk/python/unit_tests/test_entrypoint.py +++ b/airbyte-cdk/python/unit_tests/test_entrypoint.py @@ -156,9 +156,9 @@ def test_config_validate(entrypoint: AirbyteEntrypoint, mocker, config_mock, sch messages = list(entrypoint.run(parsed_args)) assert [_wrap_message(check_value)] == messages else: - with pytest.raises(SystemExit) as ex_info: + with pytest.raises(Exception) as ex_info: list(entrypoint.run(parsed_args)) - assert ex_info.value.code == 1 + assert "Config validation error:" in str(ex_info.value) def test_run_check(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock): diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py index 8316ba6a5e22c..b16066b2905e3 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py @@ -97,7 +97,7 @@ def test_check(self, connector_config, inputs: ConnectionTestConfig, docker_runn docker_runner.call_check(config=connector_config) assert err.value.exit_status != 0, "Connector should exit with error code" - assert "Traceback" in err.value.stderr.decode("utf-8"), "Connector should print exception" + assert "Traceback" in err.value.stderr, "Connector should print exception" @pytest.mark.default_timeout(30) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml b/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml index 70eed799e56d3..2e84915bed60d 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml @@ -8,7 +8,7 @@ tests: - config_path: "secrets/config.json" status: "succeed" - config_path: "integration_tests/invalid_config.json" - status: "failed" + status: "exception" discovery: - config_path: "secrets/config.json" basic_read: