From 43b1175ffe0cd3e233793f5d8518fb75a76efbbf Mon Sep 17 00:00:00 2001 From: dandpz Date: Wed, 16 Nov 2022 12:54:32 +0100 Subject: [PATCH 01/10] add new configurable field --- .../init/src/main/resources/seed/source_specs.yaml | 9 +++++++++ .../source-amazon-ads/integration_tests/spec.json | 8 ++++++++ .../source-amazon-ads/source_amazon_ads/spec.yaml | 9 +++++++++ .../streams/report_streams/report_streams.py | 3 ++- 4 files changed, 28 insertions(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index bdab07061c955..360321437662e 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -824,6 +824,15 @@ type: "array" uniqueItems: true order: 7 + look_back_window: + title: "Look Back Window" + description: "The amount of days to go back in time to get the updated data from Amazon Ads" + examples: + - 3 + - 10 + type: "integer" + default: 3 + order: 8 required: - "client_id" - "client_secret" diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json index 9400deac956e6..2dbf91ddb1744 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json @@ -64,6 +64,14 @@ "type": "array", "uniqueItems": true, "order": 7 + }, + "look_back_window": { + "title": "Look Back Window", + "description": "The amount of days to go back in time to get the updated data from Amazon Ads", + "default": 3, + "examples": [3, 10], + "order": 8, + "type": "integer" } }, "required": ["client_id", "client_secret", "refresh_token"], diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml index d77e1120f72fb..c91e8263600c7 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml @@ -77,6 +77,15 @@ connectionSpecification: type: array uniqueItems: true order: 7 + look_back_window: + title: "Look Back Window" + description: "The amount of days to go back in time to get the updated data from Amazon Ads" + examples: + - 3 + - 10 + type: "integer" + default: 3 + order: 8 required: - client_id - client_secret diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index c6228d52d334e..05906d677bc53 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -118,6 +118,7 @@ def __init__(self, config: Mapping[str, Any], profiles: List[Profile], authentic self._session = requests.Session() self._model = self._generate_model() self._start_date: Optional[Date] = config.get("start_date") + self._look_back_window: int = config.get("look_back_window", self.LOOK_BACK_WINDOW) # Timeout duration in minutes for Reports. Default is 180 minutes. self.report_wait_timeout: int = get_typed_env("REPORT_WAIT_TIMEOUT", 180) # Maximum retries Airbyte will attempt for fetching report data. Default is 5. @@ -329,7 +330,7 @@ def get_updated_state(self, current_stream_state: Dict[str, Any], latest_data: M def _update_state(self, profile: Profile, report_date: str): report_date = pendulum.from_format(report_date, self.REPORT_DATE_FORMAT).date() - look_back_date = pendulum.today(tz=profile.timezone).date().subtract(days=self.LOOK_BACK_WINDOW - 1) + look_back_date = pendulum.today(tz=profile.timezone).date().subtract(days=self._look_back_window - 1) start_date = self.get_start_date(profile, self._state) updated_state = max(min(report_date, look_back_date), start_date).format(self.REPORT_DATE_FORMAT) From 66eb96c6d718354a662ca3d20b9b4d975a13f5b1 Mon Sep 17 00:00:00 2001 From: dandpz Date: Wed, 16 Nov 2022 12:54:44 +0100 Subject: [PATCH 02/10] update docker image version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- airbyte-integrations/connectors/source-amazon-ads/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index a5cfb7ce70576..96c36a6383ea1 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -54,7 +54,7 @@ - name: Amazon Ads sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246 dockerRepository: airbyte/source-amazon-ads - dockerImageTag: 0.1.25 + dockerImageTag: 0.1.26 documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-ads icon: amazonads.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 360321437662e..da2e0bc2f5dac 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -748,7 +748,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-amazon-ads:0.1.25" +- dockerImage: "airbyte/source-amazon-ads:0.1.26" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/amazon-ads" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile index a47bfb1d7151e..ae0a5ce88f265 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.25 +LABEL io.airbyte.version=0.1.26 LABEL io.airbyte.name=airbyte/source-amazon-ads From 5e6f604d1867c60a5d060d6fee86da81d6617a04 Mon Sep 17 00:00:00 2001 From: dandpz Date: Wed, 16 Nov 2022 18:20:43 +0100 Subject: [PATCH 03/10] amazon-ads.md updated --- docs/integrations/sources/amazon-ads.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md index 1a8a06fc5682e..c67353fc267d2 100644 --- a/docs/integrations/sources/amazon-ads.md +++ b/docs/integrations/sources/amazon-ads.md @@ -87,6 +87,7 @@ Information about expected report generation waiting time you may find [here](ht | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| +| 0.1.26 | 2022-11-16 | [19491](https://github.com/airbytehq/airbyte/pull/19491) | Add option to customize look back window value | 0.1.25 | 2022-11-08 | [18985](https://github.com/airbytehq/airbyte/pull/18985) | Remove "report_wait_timeout", "report_generation_max_retries" from config | | 0.1.24 | 2022-10-19 | [17475](https://github.com/airbytehq/airbyte/pull/17475) | Add filters for state on brand, product and display campaigns | | 0.1.23 | 2022-09-06 | [16342](https://github.com/airbytehq/airbyte/pull/16342) | Add attribution reports | From d8c46fa18e81bc58da99238361edd724f0406a4b Mon Sep 17 00:00:00 2001 From: dandpz Date: Thu, 19 Jan 2023 10:41:07 +0100 Subject: [PATCH 04/10] remove unnecessary constant --- .../streams/report_streams/report_streams.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index 05906d677bc53..67e991b8ec24f 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -90,8 +90,6 @@ class ReportStream(BasicAmazonAdsStream, ABC): """ primary_key = ["profileId", "recordType", "reportDate", "updatedAt"] - # Amazon ads updates the data for the next 3 days - LOOK_BACK_WINDOW = 3 # https://advertising.amazon.com/API/docs/en-us/reporting/v2/faq#what-is-the-available-report-history-for-the-version-2-reporting-api REPORTING_PERIOD = 60 # (Service limits section) @@ -118,7 +116,7 @@ def __init__(self, config: Mapping[str, Any], profiles: List[Profile], authentic self._session = requests.Session() self._model = self._generate_model() self._start_date: Optional[Date] = config.get("start_date") - self._look_back_window: int = config.get("look_back_window", self.LOOK_BACK_WINDOW) + self._look_back_window: int = config.get("look_back_window") # Timeout duration in minutes for Reports. Default is 180 minutes. self.report_wait_timeout: int = get_typed_env("REPORT_WAIT_TIMEOUT", 180) # Maximum retries Airbyte will attempt for fetching report data. Default is 5. From c0b0f56138364964092d9dc35f1bcf4dd865380c Mon Sep 17 00:00:00 2001 From: dandpz Date: Fri, 20 Jan 2023 10:28:22 +0100 Subject: [PATCH 05/10] rollback changes inside seed files --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 56f5fec5dbaba..e7dfad1c3c127 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -57,7 +57,7 @@ - name: Amazon Ads sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246 dockerRepository: airbyte/source-amazon-ads - dockerImageTag: 0.1.28 + dockerImageTag: 0.1.27 documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-ads icon: amazonads.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 657c5ba3f89f5..020b58d13ed39 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -700,7 +700,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-amazon-ads:0.1.28" +- dockerImage: "airbyte/source-amazon-ads:0.1.27" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/amazon-ads" connectionSpecification: From d2386e54215ee7c22909e674fa9e555c3b0dd7f0 Mon Sep 17 00:00:00 2001 From: dandpz Date: Fri, 20 Jan 2023 10:38:37 +0100 Subject: [PATCH 06/10] fully revert changes --- .../init/src/main/resources/seed/source_specs.yaml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 020b58d13ed39..dc9f71b621a75 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -776,15 +776,6 @@ type: "array" uniqueItems: true order: 7 - look_back_window: - title: "Look Back Window" - description: "The amount of days to go back in time to get the updated data from Amazon Ads" - examples: - - 3 - - 10 - type: "integer" - default: 3 - order: 8 required: - "client_id" - "client_secret" From 0a0494505390714d4a4d638e5c4a7a0daafb9422 Mon Sep 17 00:00:00 2001 From: dandpz Date: Fri, 20 Jan 2023 10:42:45 +0100 Subject: [PATCH 07/10] pre load lbw default value --- .../connectors/source-amazon-ads/source_amazon_ads/source.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py index 5810971db1d52..b53cc24352965 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py @@ -52,6 +52,9 @@ def _validate_and_transform(self, config: Mapping[str, Any]): if not config.get("region"): source_spec = self.spec(logging.getLogger("airbyte")) config["region"] = source_spec.connectionSpecification["properties"]["region"]["default"] + if not config.get('look_back_window'): + source_spec = self.spec(logging.getLogger("airbyte")) + config["look_back_window"] = source_spec.connectionSpecification["properties"]["look_back_window"]["default"] return config def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: From 509da214c4b57fae83e488ba598294054d4842fd Mon Sep 17 00:00:00 2001 From: dandpz Date: Fri, 20 Jan 2023 11:01:35 +0100 Subject: [PATCH 08/10] fix value access --- .../source_amazon_ads/streams/report_streams/report_streams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index 9716aa37d7d9e..7359bbbc884b7 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -116,7 +116,7 @@ def __init__(self, config: Mapping[str, Any], profiles: List[Profile], authentic self._session = requests.Session() self._model = self._generate_model() self._start_date: Optional[Date] = config.get("start_date") - self._look_back_window: int = config.get("look_back_window") + self._look_back_window: int = config["look_back_window"] # Timeout duration in minutes for Reports. Default is 180 minutes. self.report_wait_timeout: int = get_typed_env("REPORT_WAIT_TIMEOUT", 180) # Maximum retries Airbyte will attempt for fetching report data. Default is 5. From 78170060d4fa634c55265d272c023c854b465d58 Mon Sep 17 00:00:00 2001 From: dandpz Date: Fri, 20 Jan 2023 12:46:54 +0100 Subject: [PATCH 09/10] update config fixture --- .../connectors/source-amazon-ads/unit_tests/conftest.py | 1 + .../connectors/source-amazon-ads/unit_tests/test_source.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py index 6cf2617a71c27..2ac66a0eb488a 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py @@ -15,6 +15,7 @@ def config(): "client_secret": "test_client_secret", "refresh_token": "test_refresh", "region": "NA", + "look_back_window": 3 } diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py index 5a28cf92975e5..2d648396bf474 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py @@ -68,6 +68,8 @@ def test_check(config_gen): assert len(responses.calls) == 8 assert url_strip_query(responses.calls[7].request.url) == "https://advertising-api.amazon.com/v2/profiles" + assert command_check(source, config_gen(look_back_window=...)) == AirbyteConnectionStatus(status=Status.SUCCEEDED) + @responses.activate def test_source_streams(config): From eb48a4112a3f323392bd0edebebaa83a76eb93bb Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Thu, 26 Jan 2023 17:14:58 +0000 Subject: [PATCH 10/10] auto-bump connector version --- .../src/main/resources/seed/source_definitions.yaml | 2 +- .../init/src/main/resources/seed/source_specs.yaml | 12 +++++++++++- .../source-amazon-ads/source_amazon_ads/source.py | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 308b63c11004e..a1b7a3183587d 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -57,7 +57,7 @@ - name: Amazon Ads sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246 dockerRepository: airbyte/source-amazon-ads - dockerImageTag: 0.1.27 + dockerImageTag: 0.1.28 documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-ads icon: amazonads.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 51e04cb4fe40f..9601868c86fce 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -804,7 +804,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-amazon-ads:0.1.27" +- dockerImage: "airbyte/source-amazon-ads:0.1.28" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/amazon-ads" connectionSpecification: @@ -880,6 +880,16 @@ type: "array" uniqueItems: true order: 7 + look_back_window: + title: "Look Back Window" + description: "The amount of days to go back in time to get the updated data\ + \ from Amazon Ads" + examples: + - 3 + - 10 + type: "integer" + default: 3 + order: 8 required: - "client_id" - "client_secret" diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py index b53cc24352965..7466c746817a1 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py @@ -52,7 +52,7 @@ def _validate_and_transform(self, config: Mapping[str, Any]): if not config.get("region"): source_spec = self.spec(logging.getLogger("airbyte")) config["region"] = source_spec.connectionSpecification["properties"]["region"]["default"] - if not config.get('look_back_window'): + if not config.get("look_back_window"): source_spec = self.spec(logging.getLogger("airbyte")) config["look_back_window"] = source_spec.connectionSpecification["properties"]["look_back_window"]["default"] return config