diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index c5b914f0b3228..9c17bed250599 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -57,7 +57,7 @@ - name: Amazon Ads sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246 dockerRepository: airbyte/source-amazon-ads - dockerImageTag: 0.1.27 + dockerImageTag: 0.1.28 documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-ads icon: amazonads.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 451360ad448f8..da171a636a08d 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -804,7 +804,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-amazon-ads:0.1.27" +- dockerImage: "airbyte/source-amazon-ads:0.1.28" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/amazon-ads" connectionSpecification: @@ -880,6 +880,16 @@ type: "array" uniqueItems: true order: 7 + look_back_window: + title: "Look Back Window" + description: "The amount of days to go back in time to get the updated data\ + \ from Amazon Ads" + examples: + - 3 + - 10 + type: "integer" + default: 3 + order: 8 required: - "client_id" - "client_secret" diff --git a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile index c886f587e32f2..630c894b5326f 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.27 +LABEL io.airbyte.version=0.1.28 LABEL io.airbyte.name=airbyte/source-amazon-ads diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json index 9400deac956e6..2dbf91ddb1744 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json @@ -64,6 +64,14 @@ "type": "array", "uniqueItems": true, "order": 7 + }, + "look_back_window": { + "title": "Look Back Window", + "description": "The amount of days to go back in time to get the updated data from Amazon Ads", + "default": 3, + "examples": [3, 10], + "order": 8, + "type": "integer" } }, "required": ["client_id", "client_secret", "refresh_token"], diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py index 5810971db1d52..7466c746817a1 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py @@ -52,6 +52,9 @@ def _validate_and_transform(self, config: Mapping[str, Any]): if not config.get("region"): source_spec = self.spec(logging.getLogger("airbyte")) config["region"] = source_spec.connectionSpecification["properties"]["region"]["default"] + if not config.get("look_back_window"): + source_spec = self.spec(logging.getLogger("airbyte")) + config["look_back_window"] = source_spec.connectionSpecification["properties"]["look_back_window"]["default"] return config def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml index d77e1120f72fb..c91e8263600c7 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml @@ -77,6 +77,15 @@ connectionSpecification: type: array uniqueItems: true order: 7 + look_back_window: + title: "Look Back Window" + description: "The amount of days to go back in time to get the updated data from Amazon Ads" + examples: + - 3 + - 10 + type: "integer" + default: 3 + order: 8 required: - client_id - client_secret diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index 0cbad9c7dcd10..7359bbbc884b7 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -90,8 +90,6 @@ class ReportStream(BasicAmazonAdsStream, ABC): """ primary_key = ["profileId", "recordType", "reportDate", "updatedAt"] - # Amazon ads updates the data for the next 3 days - LOOK_BACK_WINDOW = 3 # https://advertising.amazon.com/API/docs/en-us/reporting/v2/faq#what-is-the-available-report-history-for-the-version-2-reporting-api REPORTING_PERIOD = 60 # (Service limits section) @@ -118,6 +116,7 @@ def __init__(self, config: Mapping[str, Any], profiles: List[Profile], authentic self._session = requests.Session() self._model = self._generate_model() self._start_date: Optional[Date] = config.get("start_date") + self._look_back_window: int = config["look_back_window"] # Timeout duration in minutes for Reports. Default is 180 minutes. self.report_wait_timeout: int = get_typed_env("REPORT_WAIT_TIMEOUT", 180) # Maximum retries Airbyte will attempt for fetching report data. Default is 5. @@ -329,7 +328,7 @@ def get_updated_state(self, current_stream_state: Dict[str, Any], latest_data: M def _update_state(self, profile: Profile, report_date: str): report_date = pendulum.from_format(report_date, self.REPORT_DATE_FORMAT).date() - look_back_date = pendulum.today(tz=profile.timezone).date().subtract(days=self.LOOK_BACK_WINDOW - 1) + look_back_date = pendulum.today(tz=profile.timezone).date().subtract(days=self._look_back_window - 1) start_date = self.get_start_date(profile, self._state) updated_state = max(min(report_date, look_back_date), start_date).format(self.REPORT_DATE_FORMAT) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py index 6cf2617a71c27..2ac66a0eb488a 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py @@ -15,6 +15,7 @@ def config(): "client_secret": "test_client_secret", "refresh_token": "test_refresh", "region": "NA", + "look_back_window": 3 } diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py index 5a28cf92975e5..2d648396bf474 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py @@ -68,6 +68,8 @@ def test_check(config_gen): assert len(responses.calls) == 8 assert url_strip_query(responses.calls[7].request.url) == "https://advertising-api.amazon.com/v2/profiles" + assert command_check(source, config_gen(look_back_window=...)) == AirbyteConnectionStatus(status=Status.SUCCEEDED) + @responses.activate def test_source_streams(config): diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md index cf501d1b2bfce..062e7e0249a4d 100644 --- a/docs/integrations/sources/amazon-ads.md +++ b/docs/integrations/sources/amazon-ads.md @@ -94,6 +94,7 @@ Information about expected report generation waiting time you may find [here](ht | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| +| 0.1.28 | 2023-01-18 | [19491](https://github.com/airbytehq/airbyte/pull/19491) | Add option to customize look back window value | 0.1.27 | 2023-01-05 | [21082](https://github.com/airbytehq/airbyte/pull/21082) | Fix bug with handling: "Report date is too far in the past." - partial revert of #20662 | | 0.1.26 | 2022-12-19 | [20662](https://github.com/airbytehq/airbyte/pull/20662) | Fix bug with handling: "Report date is too far in the past." | | 0.1.25 | 2022-11-08 | [18985](https://github.com/airbytehq/airbyte/pull/18985) | Remove "report_wait_timeout", "report_generation_max_retries" from config |