diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index c730776d3fd11..b5b71c15558f6 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -17,7 +17,7 @@ - name: Amazon Ads sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246 dockerRepository: airbyte/source-amazon-ads - dockerImageTag: 0.1.18 + dockerImageTag: 0.1.19 documentationUrl: https://docs.airbyte.io/integrations/sources/amazon-ads icon: amazonads.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 95c732a60e9b0..c1e96e54aec14 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -87,7 +87,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-amazon-ads:0.1.18" +- dockerImage: "airbyte/source-amazon-ads:0.1.19" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/amazon-ads" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile index ed940909275c5..ab4778154832e 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.18 +LABEL io.airbyte.version=0.1.19 LABEL io.airbyte.name=airbyte/source-amazon-ads diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py index 302d5925ec904..a03c781355e3a 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py @@ -4,10 +4,9 @@ import logging -import os from typing import Any, List, Mapping, Optional, Tuple -from airbyte_cdk.connector import _WriteConfigProtocol +import pendulum from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator @@ -36,16 +35,19 @@ # Oauth 2.0 authentication URL for amazon TOKEN_URL = "https://api.amazon.com/auth/o2/token" +CONFIG_DATE_FORMAT = "YYYY-MM-DD" class SourceAmazonAds(AbstractSource): - def configure(self: _WriteConfigProtocol, config: Mapping[str, Any], temp_dir: str) -> Mapping[str, Any]: + def _validate_and_transform(self, config: Mapping[str, Any]): + start_date = config.get("start_date") + if start_date: + config["start_date"] = pendulum.from_format(start_date, CONFIG_DATE_FORMAT).date() + else: + config["start_date"] = None if not config.get("region"): source_spec = self.spec(logging.getLogger("airbyte")) - default_region = source_spec.connectionSpecification["properties"]["region"]["default"] - config["region"] = default_region - config_path = os.path.join(temp_dir, "config.json") - self.write_config(config, config_path) + config["region"] = source_spec.connectionSpecification["properties"]["region"]["default"] return config def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: @@ -54,6 +56,10 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> :param logger: logger object :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ + try: + config = self._validate_and_transform(config) + except Exception as e: + return False, str(e) # Check connection by sending list of profiles request. Its most simple # request, not require additional parameters and usually has few data # in response body. @@ -67,6 +73,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: :param config: A Mapping of the user input configuration as defined in the connector spec. :return list of streams for current source """ + config = self._validate_and_transform(config) auth = self._make_authenticator(config) stream_args = {"config": config, "authenticator": auth} # All data for individual Amazon Ads stream divided into sets of data for diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index 594d5c413fd30..35c8fab7d269e 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -96,7 +96,6 @@ class ReportStream(BasicAmazonAdsStream, ABC): # (Service limits section) # Format used to specify metric generation date over Amazon Ads API. REPORT_DATE_FORMAT = "YYYYMMDD" - CONFIG_DATE_FORMAT = "YYYY-MM-DD" cursor_field = "reportDate" def __init__(self, config: Mapping[str, Any], profiles: List[Profile], authenticator: Oauth2Authenticator): @@ -106,10 +105,7 @@ def __init__(self, config: Mapping[str, Any], profiles: List[Profile], authentic self._model = self._generate_model() self.report_wait_timeout = config.get("report_wait_timeout", 30) self.report_generation_maximum_retries = config.get("report_generation_max_retries", 5) - # Set start date from config file - self._start_date = config.get("start_date") - if self._start_date: - self._start_date = pendulum.from_format(self._start_date, self.CONFIG_DATE_FORMAT).date() + self._start_date: Optional[Date] = config.get("start_date") super().__init__(config, profiles) @property diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py index 727f81d7001f5..54cfe29dfefbf 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py @@ -2,6 +2,8 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from copy import deepcopy + from pytest import fixture @@ -10,12 +12,22 @@ def config(): return { "client_id": "test_client_id", "client_secret": "test_client_secret", - "scope": "test_scope", "refresh_token": "test_refresh", "region": "NA", } +@fixture +def config_gen(config): + def inner(**kwargs): + new_config = deepcopy(config) + # WARNING, no support deep dictionaries + new_config.update(kwargs) + return {k: v for k, v in new_config.items() if v is not ...} + + return inner + + @fixture def profiles_response(): return """ diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py index 9c106407b25c6..985f79f66552c 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py @@ -8,6 +8,7 @@ from functools import partial from unittest import mock +import pendulum import pytest import responses from airbyte_cdk.models import SyncMode @@ -16,6 +17,7 @@ from pytest import raises from requests.exceptions import ConnectionError from source_amazon_ads.schemas.profile import AccountInfo, Profile +from source_amazon_ads.source import CONFIG_DATE_FORMAT from source_amazon_ads.streams import ( SponsoredBrandsReportStream, SponsoredBrandsVideoReportStream, @@ -339,10 +341,10 @@ def test_display_report_stream_slices_incremental(config): def test_get_start_date(config): profiles = make_profiles() - config["start_date"] = "2021-07-10" + config["start_date"] = pendulum.from_format("2021-07-10", CONFIG_DATE_FORMAT).date() stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock()) assert stream.get_start_date(profiles[0], {}) == Date(2021, 7, 10) - config["start_date"] = "2021-05-10" + config["start_date"] = pendulum.from_format("2021-05-10", CONFIG_DATE_FORMAT).date() stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock()) assert stream.get_start_date(profiles[0], {}) == Date(2021, 6, 1) @@ -368,7 +370,7 @@ def test_stream_slices_different_timezones(config): def test_stream_slices_lazy_evaluation(config): with freeze_time("2022-06-01T23:50:00+00:00") as frozen_datetime: - config["start_date"] = "2021-05-10" + config["start_date"] = pendulum.from_format("2021-05-10", CONFIG_DATE_FORMAT).date() profile1 = Profile(profileId=1, timezone="UTC", accountInfo=AccountInfo(marketplaceStringId="", id="", type="seller")) profile2 = Profile(profileId=2, timezone="UTC", accountInfo=AccountInfo(marketplaceStringId="", id="", type="seller")) @@ -491,7 +493,7 @@ def test_read_incremental_without_records_start_date(config): ) profiles = make_profiles() - config["start_date"] = "2020-12-25" + config["start_date"] = pendulum.from_format("2020-12-25", CONFIG_DATE_FORMAT).date() stream = SponsoredDisplayReportStream(config, profiles, authenticator=mock.MagicMock()) with freeze_time("2021-01-02 12:00:00") as frozen_datetime: @@ -514,7 +516,7 @@ def test_read_incremental_with_records_start_date(config): ) profiles = make_profiles() - config["start_date"] = "2020-12-25" + config["start_date"] = pendulum.from_format("2020-12-25", CONFIG_DATE_FORMAT).date() stream = SponsoredDisplayReportStream(config, profiles, authenticator=mock.MagicMock()) with freeze_time("2021-01-02 12:00:00") as frozen_datetime: diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py index 52876c71b9f72..4637ad670fa45 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py @@ -7,6 +7,8 @@ from jsonschema import Draft4Validator from source_amazon_ads import SourceAmazonAds +from .utils import command_check, url_strip_query + def setup_responses(): responses.add( @@ -39,12 +41,33 @@ def test_spec(): @responses.activate -def test_check(config): +def test_check(config_gen): setup_responses() source = SourceAmazonAds() - assert source.check(None, config) == AirbyteConnectionStatus(status=Status.SUCCEEDED) + + assert command_check(source, config_gen(start_date=...)) == AirbyteConnectionStatus(status=Status.SUCCEEDED) assert len(responses.calls) == 2 + assert command_check(source, config_gen(start_date="")) == AirbyteConnectionStatus(status=Status.SUCCEEDED) + assert len(responses.calls) == 4 + + assert source.check(None, config_gen(start_date="2022-02-20")) == AirbyteConnectionStatus(status=Status.SUCCEEDED) + assert len(responses.calls) == 6 + + assert command_check(source, config_gen(start_date="2022-20-02")) == AirbyteConnectionStatus( + status=Status.FAILED, message="'month must be in 1..12'" + ) + assert len(responses.calls) == 6 + + assert command_check(source, config_gen(start_date="no date")) == AirbyteConnectionStatus( + status=Status.FAILED, message="'String does not match format YYYY-MM-DD'" + ) + assert len(responses.calls) == 6 + + assert command_check(source, config_gen(region=...)) == AirbyteConnectionStatus(status=Status.SUCCEEDED) + assert len(responses.calls) == 8 + assert url_strip_query(responses.calls[7].request.url) == "https://advertising-api.amazon.com/v2/profiles" + @responses.activate def test_source_streams(config): diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py index 6c1c110d35dfe..71f5d8566f31c 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py @@ -3,9 +3,14 @@ # from typing import Any, Iterator, MutableMapping +from unittest import mock +from urllib.parse import urlparse, urlunparse from airbyte_cdk.models import SyncMode +from airbyte_cdk.models.airbyte_protocol import ConnectorSpecification +from airbyte_cdk.sources import Source from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit, split_config def read_incremental(stream_instance: Stream, stream_state: MutableMapping[str, Any]) -> Iterator[dict]: @@ -23,3 +28,18 @@ def read_incremental(stream_instance: Stream, stream_state: MutableMapping[str, if hasattr(stream_instance, "state"): stream_state.clear() stream_state.update(stream_instance.state) + + +def command_check(source: Source, config): + logger = mock.MagicMock() + connector_config, _ = split_config(config) + if source.check_config_against_spec: + source_spec: ConnectorSpecification = source.spec(logger) + check_config_against_spec_or_exit(connector_config, source_spec) + return source.check(logger, config) + + +def url_strip_query(url): + parsed_result = urlparse(url) + parsed_result = parsed_result._replace(query="") + return urlunparse(parsed_result) diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md index d2be14f8eb0af..3f19f9795a576 100644 --- a/docs/integrations/sources/amazon-ads.md +++ b/docs/integrations/sources/amazon-ads.md @@ -90,6 +90,7 @@ Information about expected report generation waiting time you may find [here](ht | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------| +| 0.1.19 | 2022-08-31 | [16191](https://github.com/airbytehq/airbyte/pull/16191) | Improved connector's input configuration validation | | 0.1.18 | 2022-08-25 | [15951](https://github.com/airbytehq/airbyte/pull/15951) | Skip API error "Tactic T00020 is not supported for report API in marketplace A1C3SOZRARQ6R3." | | 0.1.17 | 2022-08-24 | [15921](https://github.com/airbytehq/airbyte/pull/15921) | Skip API error "Report date is too far in the past." | | 0.1.16 | 2022-08-23 | [15822](https://github.com/airbytehq/airbyte/pull/15822) | Set default value for 'region' if needed |