🎉 Source Amazon Ads: improve config.start_date validation (#16191)

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -17,7 +17,7 @@
 - name: Amazon Ads
   sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246
   dockerRepository: airbyte/source-amazon-ads
-  dockerImageTag: 0.1.18
+  dockerImageTag: 0.1.19
   documentationUrl: https://docs.airbyte.io/integrations/sources/amazon-ads
   icon: amazonads.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -87,7 +87,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-amazon-ads:0.1.18"
+- dockerImage: "airbyte/source-amazon-ads:0.1.19"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/amazon-ads"
     connectionSpecification:

airbyte-integrations/connectors/source-amazon-ads/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.18
+LABEL io.airbyte.version=0.1.19
 LABEL io.airbyte.name=airbyte/source-amazon-ads

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py

@@ -4,10 +4,9 @@
 
 
 import logging
-import os
 from typing import Any, List, Mapping, Optional, Tuple
 
-from airbyte_cdk.connector import _WriteConfigProtocol
+import pendulum
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator
@@ -36,16 +35,19 @@
 
 # Oauth 2.0 authentication URL for amazon
 TOKEN_URL = "https://api.amazon.com/auth/o2/token"
+CONFIG_DATE_FORMAT = "YYYY-MM-DD"
 
 
 class SourceAmazonAds(AbstractSource):
-    def configure(self: _WriteConfigProtocol, config: Mapping[str, Any], temp_dir: str) -> Mapping[str, Any]:
+    def _validate_and_transform(self, config: Mapping[str, Any]):
+        start_date = config.get("start_date")
+        if start_date:
+            config["start_date"] = pendulum.from_format(start_date, CONFIG_DATE_FORMAT).date()
+        else:
+            config["start_date"] = None
         if not config.get("region"):
             source_spec = self.spec(logging.getLogger("airbyte"))
-            default_region = source_spec.connectionSpecification["properties"]["region"]["default"]
-            config["region"] = default_region
-        config_path = os.path.join(temp_dir, "config.json")
-        self.write_config(config, config_path)
+            config["region"] = source_spec.connectionSpecification["properties"]["region"]["default"]
         return config
 
     def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]:
@@ -54,6 +56,10 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) ->
         :param logger:  logger object
         :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise.
         """
+        try:
+            config = self._validate_and_transform(config)
+        except Exception as e:
+            return False, str(e)
         # Check connection by sending list of profiles request. Its most simple
         # request, not require additional parameters and usually has few data
         # in response body.
@@ -67,6 +73,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         :param config: A Mapping of the user input configuration as defined in the connector spec.
         :return list of streams for current source
         """
+        config = self._validate_and_transform(config)
         auth = self._make_authenticator(config)
         stream_args = {"config": config, "authenticator": auth}
         # All data for individual Amazon Ads stream divided into sets of data for

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py

@@ -96,7 +96,6 @@ class ReportStream(BasicAmazonAdsStream, ABC):
     # (Service limits section)
     # Format used to specify metric generation date over Amazon Ads API.
     REPORT_DATE_FORMAT = "YYYYMMDD"
-    CONFIG_DATE_FORMAT = "YYYY-MM-DD"
     cursor_field = "reportDate"
 
     def __init__(self, config: Mapping[str, Any], profiles: List[Profile], authenticator: Oauth2Authenticator):
@@ -106,10 +105,7 @@ def __init__(self, config: Mapping[str, Any], profiles: List[Profile], authentic
         self._model = self._generate_model()
         self.report_wait_timeout = config.get("report_wait_timeout", 30)
         self.report_generation_maximum_retries = config.get("report_generation_max_retries", 5)
-        # Set start date from config file
-        self._start_date = config.get("start_date")
-        if self._start_date:
-            self._start_date = pendulum.from_format(self._start_date, self.CONFIG_DATE_FORMAT).date()
+        self._start_date: Optional[Date] = config.get("start_date")
         super().__init__(config, profiles)
 
     @property

airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py

@@ -2,6 +2,8 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
+from copy import deepcopy
+
 from pytest import fixture
 
 
@@ -10,12 +12,22 @@ def config():
     return {
         "client_id": "test_client_id",
         "client_secret": "test_client_secret",
-        "scope": "test_scope",
         "refresh_token": "test_refresh",
         "region": "NA",
     }
 
 
+@fixture
+def config_gen(config):
+    def inner(**kwargs):
+        new_config = deepcopy(config)
+        # WARNING, no support deep dictionaries
+        new_config.update(kwargs)
+        return {k: v for k, v in new_config.items() if v is not ...}
+
+    return inner
+
+
 @fixture
 def profiles_response():
     return """

airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py

@@ -8,6 +8,7 @@
 from functools import partial
 from unittest import mock
 
+import pendulum
 import pytest
 import responses
 from airbyte_cdk.models import SyncMode
@@ -16,6 +17,7 @@
 from pytest import raises
 from requests.exceptions import ConnectionError
 from source_amazon_ads.schemas.profile import AccountInfo, Profile
+from source_amazon_ads.source import CONFIG_DATE_FORMAT
 from source_amazon_ads.streams import (
     SponsoredBrandsReportStream,
     SponsoredBrandsVideoReportStream,
@@ -339,10 +341,10 @@ def test_display_report_stream_slices_incremental(config):
 def test_get_start_date(config):
     profiles = make_profiles()
 
-    config["start_date"] = "2021-07-10"
+    config["start_date"] = pendulum.from_format("2021-07-10", CONFIG_DATE_FORMAT).date()
     stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock())
     assert stream.get_start_date(profiles[0], {}) == Date(2021, 7, 10)
-    config["start_date"] = "2021-05-10"
+    config["start_date"] = pendulum.from_format("2021-05-10", CONFIG_DATE_FORMAT).date()
     stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock())
     assert stream.get_start_date(profiles[0], {}) == Date(2021, 6, 1)
 
@@ -368,7 +370,7 @@ def test_stream_slices_different_timezones(config):
 
 def test_stream_slices_lazy_evaluation(config):
     with freeze_time("2022-06-01T23:50:00+00:00") as frozen_datetime:
-        config["start_date"] = "2021-05-10"
+        config["start_date"] = pendulum.from_format("2021-05-10", CONFIG_DATE_FORMAT).date()
         profile1 = Profile(profileId=1, timezone="UTC", accountInfo=AccountInfo(marketplaceStringId="", id="", type="seller"))
         profile2 = Profile(profileId=2, timezone="UTC", accountInfo=AccountInfo(marketplaceStringId="", id="", type="seller"))
 
@@ -491,7 +493,7 @@ def test_read_incremental_without_records_start_date(config):
     )
 
     profiles = make_profiles()
-    config["start_date"] = "2020-12-25"
+    config["start_date"] = pendulum.from_format("2020-12-25", CONFIG_DATE_FORMAT).date()
     stream = SponsoredDisplayReportStream(config, profiles, authenticator=mock.MagicMock())
 
     with freeze_time("2021-01-02 12:00:00") as frozen_datetime:
@@ -514,7 +516,7 @@ def test_read_incremental_with_records_start_date(config):
     )
 
     profiles = make_profiles()
-    config["start_date"] = "2020-12-25"
+    config["start_date"] = pendulum.from_format("2020-12-25", CONFIG_DATE_FORMAT).date()
     stream = SponsoredDisplayReportStream(config, profiles, authenticator=mock.MagicMock())
 
     with freeze_time("2021-01-02 12:00:00") as frozen_datetime:

airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py

@@ -7,6 +7,8 @@
 from jsonschema import Draft4Validator
 from source_amazon_ads import SourceAmazonAds
 
+from .utils import command_check, url_strip_query
+
 
 def setup_responses():
     responses.add(
@@ -39,12 +41,33 @@ def test_spec():
 
 
 @responses.activate
-def test_check(config):
+def test_check(config_gen):
     setup_responses()
     source = SourceAmazonAds()
-    assert source.check(None, config) == AirbyteConnectionStatus(status=Status.SUCCEEDED)
+
+    assert command_check(source, config_gen(start_date=...)) == AirbyteConnectionStatus(status=Status.SUCCEEDED)
     assert len(responses.calls) == 2
 
+    assert command_check(source, config_gen(start_date="")) == AirbyteConnectionStatus(status=Status.SUCCEEDED)
+    assert len(responses.calls) == 4
+
+    assert source.check(None, config_gen(start_date="2022-02-20")) == AirbyteConnectionStatus(status=Status.SUCCEEDED)
+    assert len(responses.calls) == 6
+
+    assert command_check(source, config_gen(start_date="2022-20-02")) == AirbyteConnectionStatus(
+        status=Status.FAILED, message="'month must be in 1..12'"
+    )
+    assert len(responses.calls) == 6
+
+    assert command_check(source, config_gen(start_date="no date")) == AirbyteConnectionStatus(
+        status=Status.FAILED, message="'String does not match format YYYY-MM-DD'"
+    )
+    assert len(responses.calls) == 6
+
+    assert command_check(source, config_gen(region=...)) == AirbyteConnectionStatus(status=Status.SUCCEEDED)
+    assert len(responses.calls) == 8
+    assert url_strip_query(responses.calls[7].request.url) == "https://advertising-api.amazon.com/v2/profiles"
+
 
 @responses.activate
 def test_source_streams(config):

airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py

@@ -3,9 +3,14 @@
 #
 
 from typing import Any, Iterator, MutableMapping
+from unittest import mock
+from urllib.parse import urlparse, urlunparse
 
 from airbyte_cdk.models import SyncMode
+from airbyte_cdk.models.airbyte_protocol import ConnectorSpecification
+from airbyte_cdk.sources import Source
 from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit, split_config
 
 
 def read_incremental(stream_instance: Stream, stream_state: MutableMapping[str, Any]) -> Iterator[dict]:
@@ -23,3 +28,18 @@ def read_incremental(stream_instance: Stream, stream_state: MutableMapping[str,
         if hasattr(stream_instance, "state"):
             stream_state.clear()
             stream_state.update(stream_instance.state)
+
+
+def command_check(source: Source, config):
+    logger = mock.MagicMock()
+    connector_config, _ = split_config(config)
+    if source.check_config_against_spec:
+        source_spec: ConnectorSpecification = source.spec(logger)
+        check_config_against_spec_or_exit(connector_config, source_spec)
+    return source.check(logger, config)
+
+
+def url_strip_query(url):
+    parsed_result = urlparse(url)
+    parsed_result = parsed_result._replace(query="")
+    return urlunparse(parsed_result)

docs/integrations/sources/amazon-ads.md

@@ -90,6 +90,7 @@ Information about expected report generation waiting time you may find [here](ht
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                           |
 |:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------|
+| 0.1.19 | 2022-08-31 | [16191](https://github.com/airbytehq/airbyte/pull/16191)    | Improved connector's input configuration validation                                                               |
 | 0.1.18 | 2022-08-25 | [15951](https://github.com/airbytehq/airbyte/pull/15951)    | Skip API error "Tactic T00020 is not supported for report API in marketplace A1C3SOZRARQ6R3."                     |
 | 0.1.17 | 2022-08-24 | [15921](https://github.com/airbytehq/airbyte/pull/15921)    | Skip API error "Report date is too far in the past."                                                              |
 | 0.1.16 | 2022-08-23 | [15822](https://github.com/airbytehq/airbyte/pull/15822)    | Set default value for 'region' if needed                                                                          |