From fad169a51519ec6a4dac78ae0a09466d6b7d5f48 Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Fri, 3 Feb 2023 04:37:53 +0100 Subject: [PATCH 1/9] Migrate products_report stream to API v3 --- .../connectors/source-amazon-ads/Dockerfile | 2 +- .../streams/report_streams/products_report.py | 495 ++++++++++-------- .../streams/report_streams/report_streams.py | 21 +- docs/integrations/sources/amazon-ads.md | 3 +- 4 files changed, 280 insertions(+), 241 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile index 1c1d83c84e2a3..5f9e193d240e2 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.29 +LABEL io.airbyte.version=1.0.0 LABEL io.airbyte.name=airbyte/source-amazon-ads diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py index ad06f0aae6f0a..020a1d04777c2 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py @@ -3,269 +3,304 @@ # -from copy import copy - -from .report_streams import RecordType, ReportStream +from http import HTTPStatus +from .report_streams import ReportStream, ReportInfo +from typing import List METRICS_MAP = { "campaigns": [ - "bidPlus", - "campaignName", - "campaignId", - "campaignStatus", - "campaignBudget", - "campaignRuleBasedBudget", - "applicableBudgetRuleId", - "applicableBudgetRuleName", - "impressions", - "clicks", - "cost", - "attributedConversions1d", - "attributedConversions7d", - "attributedConversions14d", - "attributedConversions30d", - "attributedConversions1dSameSKU", - "attributedConversions7dSameSKU", - "attributedConversions14dSameSKU", - "attributedConversions30dSameSKU", - "attributedUnitsOrdered1d", - "attributedUnitsOrdered7d", - "attributedUnitsOrdered14d", - "attributedUnitsOrdered30d", - "attributedSales1d", - "attributedSales7d", - "attributedSales14d", - "attributedSales30d", - "attributedSales1dSameSKU", - "attributedSales7dSameSKU", - "attributedSales14dSameSKU", - "attributedSales30dSameSKU", - "attributedUnitsOrdered1dSameSKU", - "attributedUnitsOrdered7dSameSKU", - "attributedUnitsOrdered14dSameSKU", - "attributedUnitsOrdered30dSameSKU", - ], + "campaignName", + "campaignId", + "campaignStatus", + "campaignBudgetAmount", + "campaignRuleBasedBudgetAmount", + "campaignApplicableBudgetRuleId", + "campaignApplicableBudgetRuleName", + "impressions", + "clicks", + "cost", + "purchases1d", + "purchases7d", + "purchases14d", + "purchases30d", + "purchasesSameSku1d", + "purchasesSameSku7d", + "purchasesSameSku14d", + "purchasesSameSku30d", + "unitsSoldClicks1d", + "unitsSoldClicks7d", + "unitsSoldClicks14d", + "unitsSoldClicks30d", + "sales1d", + "sales7d", + "sales14d", + "sales30d", + "attributedSalesSameSku1d", + "attributedSalesSameSku7d", + "attributedSalesSameSku14d", + "attributedSalesSameSku30d", + "unitsSoldSameSku1d", + "unitsSoldSameSku7d", + "unitsSoldSameSku14d", + "unitsSoldSameSku30d", + ], "adGroups": [ - "campaignName", - "campaignId", - "adGroupName", - "adGroupId", - "impressions", - "clicks", - "cost", - "attributedConversions1d", - "attributedConversions7d", - "attributedConversions14d", - "attributedConversions30d", - "attributedConversions1dSameSKU", - "attributedConversions7dSameSKU", - "attributedConversions14dSameSKU", - "attributedConversions30dSameSKU", - "attributedUnitsOrdered1d", - "attributedUnitsOrdered7d", - "attributedUnitsOrdered14d", - "attributedUnitsOrdered30d", - "attributedSales1d", - "attributedSales7d", - "attributedSales14d", - "attributedSales30d", - "attributedSales1dSameSKU", - "attributedSales7dSameSKU", - "attributedSales14dSameSKU", - "attributedSales30dSameSKU", - "attributedUnitsOrdered1dSameSKU", - "attributedUnitsOrdered7dSameSKU", - "attributedUnitsOrdered14dSameSKU", - "attributedUnitsOrdered30dSameSKU", - ], + "campaignName", + "campaignId", + "adGroupName", + "adGroupId", + "impressions", + "clicks", + "cost", + "purchases1d", + "purchases7d", + "purchases14d", + "purchases30d", + "purchasesSameSku1d", + "purchasesSameSku7d", + "purchasesSameSku14d", + "purchasesSameSku30d", + "unitsSoldClicks1d", + "unitsSoldClicks7d", + "unitsSoldClicks14d", + "unitsSoldClicks30d", + "sales1d", + "sales7d", + "sales14d", + "sales30d", + "attributedSalesSameSku1d", + "attributedSalesSameSku7d", + "attributedSalesSameSku14d", + "attributedSalesSameSku30d", + "unitsSoldSameSku1d", + "unitsSoldSameSku7d", + "unitsSoldSameSku14d", + "unitsSoldSameSku30d", + ], "keywords": [ - "campaignName", - "campaignId", - "adGroupName", - "adGroupId", - "keywordId", - "keywordText", - "matchType", - "impressions", - "clicks", - "cost", - "attributedConversions1d", - "attributedConversions7d", - "attributedConversions14d", - "attributedConversions30d", - "attributedConversions1dSameSKU", - "attributedConversions7dSameSKU", - "attributedConversions14dSameSKU", - "attributedConversions30dSameSKU", - "attributedUnitsOrdered1d", - "attributedUnitsOrdered7d", - "attributedUnitsOrdered14d", - "attributedUnitsOrdered30d", - "attributedSales1d", - "attributedSales7d", - "attributedSales14d", - "attributedSales30d", - "attributedSales1dSameSKU", - "attributedSales7dSameSKU", - "attributedSales14dSameSKU", - "attributedSales30dSameSKU", - "attributedUnitsOrdered1dSameSKU", - "attributedUnitsOrdered7dSameSKU", - "attributedUnitsOrdered14dSameSKU", - "attributedUnitsOrdered30dSameSKU", - ], + "campaignName", + "campaignId", + "adGroupName", + "adGroupId", + "keywordId", + "keyword", + "matchType", + "impressions", + "clicks", + "cost", + "purchases1d", + "purchases7d", + "purchases14d", + "purchases30d", + "purchasesSameSku1d", + "purchasesSameSku7d", + "purchasesSameSku14d", + "purchasesSameSku30d", + "unitsSoldClicks1d", + "unitsSoldClicks7d", + "unitsSoldClicks14d", + "unitsSoldClicks30d", + "sales1d", + "sales7d", + "sales14d", + "sales30d", + "attributedSalesSameSku1d", + "attributedSalesSameSku7d", + "attributedSalesSameSku14d", + "attributedSalesSameSku30d", + "unitsSoldSameSku1d", + "unitsSoldSameSku7d", + "unitsSoldSameSku14d", + "unitsSoldSameSku30d", + ], + "targets": [ + "campaignName", + "campaignId", + "adGroupName", + "adGroupId", + "keywordId", + "keyword", + "targeting", + "keywordType", + "impressions", + "clicks", + "cost", + "purchases1d", + "purchases7d", + "purchases14d", + "purchases30d", + "purchasesSameSku1d", + "purchasesSameSku7d", + "purchasesSameSku14d", + "purchasesSameSku30d", + "unitsSoldClicks1d", + "unitsSoldClicks7d", + "unitsSoldClicks14d", + "unitsSoldClicks30d", + "sales1d", + "sales7d", + "sales14d", + "sales30d", + "attributedSalesSameSku1d", + "attributedSalesSameSku7d", + "attributedSalesSameSku14d", + "attributedSalesSameSku30d", + "unitsSoldSameSku1d", + "unitsSoldSameSku7d", + "unitsSoldSameSku14d", + "unitsSoldSameSku30d", + ], "productAds": [ - "campaignName", - "campaignId", - "adGroupName", - "adGroupId", - "adId", - "impressions", - "clicks", - "cost", - "currency", - "asin", - "attributedConversions1d", - "attributedConversions7d", - "attributedConversions14d", - "attributedConversions30d", - "attributedConversions1dSameSKU", - "attributedConversions7dSameSKU", - "attributedConversions14dSameSKU", - "attributedConversions30dSameSKU", - "attributedUnitsOrdered1d", - "attributedUnitsOrdered7d", - "attributedUnitsOrdered14d", - "attributedUnitsOrdered30d", - "attributedSales1d", - "attributedSales7d", - "attributedSales14d", - "attributedSales30d", - "attributedSales1dSameSKU", - "attributedSales7dSameSKU", - "attributedSales14dSameSKU", - "attributedSales30dSameSKU", - "attributedUnitsOrdered1dSameSKU", - "attributedUnitsOrdered7dSameSKU", - "attributedUnitsOrdered14dSameSKU", - "attributedUnitsOrdered30dSameSKU", - ], + "campaignName", + "campaignId", + "adGroupName", + "adGroupId", + "adId", + "impressions", + "clicks", + "cost", + "campaignBudgetCurrencyCode", + "advertisedAsin", + "purchases1d", + "purchases7d", + "purchases14d", + "purchases30d", + "purchasesSameSku1d", + "purchasesSameSku7d", + "purchasesSameSku14d", + "purchasesSameSku30d", + "unitsSoldClicks1d", + "unitsSoldClicks7d", + "unitsSoldClicks14d", + "unitsSoldClicks30d", + "sales1d", + "sales7d", + "sales14d", + "sales30d", + "attributedSalesSameSku1d", + "attributedSalesSameSku7d", + "attributedSalesSameSku14d", + "attributedSalesSameSku30d", + "unitsSoldSameSku1d", + "unitsSoldSameSku7d", + "unitsSoldSameSku14d", + "unitsSoldSameSku30d", + ], "asins_keywords": [ "campaignName", "campaignId", "adGroupName", "adGroupId", "keywordId", - "keywordText", - "adId", - "asin", - "otherAsin", - "sku", - "currency", + "keyword", + "advertisedAsin", + "purchasedAsin", + "advertisedSku", + "campaignBudgetCurrencyCode", "matchType", - "attributedUnitsOrdered1d", - "attributedUnitsOrdered7d", - "attributedUnitsOrdered14d", - "attributedUnitsOrdered30d", - "attributedUnitsOrdered1dOtherSKU", - "attributedUnitsOrdered7dOtherSKU", - "attributedUnitsOrdered14dOtherSKU", - "attributedUnitsOrdered30dOtherSKU", - "attributedSales1dOtherSKU", - "attributedSales7dOtherSKU", - "attributedSales14dOtherSKU", - "attributedSales30dOtherSKU", + "unitsSoldClicks1d", + "unitsSoldClicks7d", + "unitsSoldClicks14d", + "unitsSoldClicks30d", + "unitsSoldOtherSku1d", + "unitsSoldOtherSku7d", + "unitsSoldOtherSku14d", + "unitsSoldOtherSku30d", + "salesOtherSku1d", + "salesOtherSku7d", + "salesOtherSku14d", + "salesOtherSku30d", ], "asins_targets": [ "campaignName", "campaignId", "adGroupName", "adGroupId", - "adId", - "asin", - "otherAsin", - "sku", - "currency", + "advertisedAsin", + "purchasedAsin", + "advertisedSku", + "campaignBudgetCurrencyCode", "matchType", - "attributedUnitsOrdered1d", - "attributedUnitsOrdered7d", - "attributedUnitsOrdered14d", - "attributedUnitsOrdered30d", - "attributedUnitsOrdered1dOtherSKU", - "attributedUnitsOrdered7dOtherSKU", - "attributedUnitsOrdered14dOtherSKU", - "attributedUnitsOrdered30dOtherSKU", - "attributedSales1dOtherSKU", - "attributedSales7dOtherSKU", - "attributedSales14dOtherSKU", - "attributedSales30dOtherSKU", - "targetId", - "targetingText", - "targetingType", - ], - "targets": [ - "campaignName", - "campaignId", - "adGroupName", - "adGroupId", - "targetId", - "targetingExpression", - "targetingText", - "targetingType", - "impressions", - "clicks", - "cost", - "attributedConversions1d", - "attributedConversions7d", - "attributedConversions14d", - "attributedConversions30d", - "attributedConversions1dSameSKU", - "attributedConversions7dSameSKU", - "attributedConversions14dSameSKU", - "attributedConversions30dSameSKU", - "attributedUnitsOrdered1d", - "attributedUnitsOrdered7d", - "attributedUnitsOrdered14d", - "attributedUnitsOrdered30d", - "attributedSales1d", - "attributedSales7d", - "attributedSales14d", - "attributedSales30d", - "attributedSales1dSameSKU", - "attributedSales7dSameSKU", - "attributedSales14dSameSKU", - "attributedSales30dSameSKU", - "attributedUnitsOrdered1dSameSKU", - "attributedUnitsOrdered7dSameSKU", - "attributedUnitsOrdered14dSameSKU", - "attributedUnitsOrdered30dSameSKU", - ], + "unitsSoldClicks1d", + "unitsSoldClicks7d", + "unitsSoldClicks14d", + "unitsSoldClicks30d", + "unitsSoldOtherSku1d", + "unitsSoldOtherSku7d", + "unitsSoldOtherSku14d", + "unitsSoldOtherSku30d", + "salesOtherSku1d", + "salesOtherSku7d", + "salesOtherSku14d", + "salesOtherSku30d", + "keywordId", + "targeting", + "keywordType", + ] } class SponsoredProductsReportStream(ReportStream): """ https://advertising.amazon.com/API/docs/en-us/sponsored-products/2-0/openapi#/Reports + https://advertising.amazon.com/API/docs/en-us/reporting/v3/migration-guide + https://advertising.amazon.com/API/docs/en-us/reporting/v3/report-types#sponsored-products """ + API_VERSION = "reporting" # v3 + REPORT_DATE_FORMAT = "YYYY-MM-DD" + ad_product = "SPONSORED_PRODUCTS" + report_is_created = HTTPStatus.OK + metrics_map = METRICS_MAP + def report_init_endpoint(self, record_type: str) -> str: - return f"/v2/sp/{record_type}/report" + return f"/{self.API_VERSION}/reports" - metrics_map = METRICS_MAP + def _download_report(self, report_info: ReportInfo, url: str) -> List[dict]: + """ + Download and parse report result + """ + return super()._download_report(None, url) def _get_init_report_body(self, report_date: str, record_type: str, profile): metrics_list = self.metrics_map[record_type] + + reportTypeId = "spCampaigns" + group_by = ["campaign"] + filters = [] + + if record_type == "adGroups": + group_by.append("adGroup") + + elif record_type == "productAds": + reportTypeId = "spAdvertisedProduct" + group_by = ["advertiser"] + + elif "asin" in record_type: + reportTypeId = "spPurchasedProduct" + group_by = ["asin"] + + elif record_type == "keywords" or record_type == "targets": + group_by = ["targeting"] + reportTypeId = "spTargeting" + filters = [{"field": "keywordType", "values": ["TARGETING_EXPRESSION", "TARGETING_EXPRESSION_PREDEFINED"]}] + + if record_type == "keywords": + filters = [{"field": "keywordType", "values": ["BROAD", "PHRASE", "EXACT"]}] + body = { - "reportDate": report_date, + "name": f"{record_type} report {report_date}", + "startDate": report_date, + "endDate": report_date, + "configuration": { + "adProduct": self.ad_product, + "groupBy": group_by, + "columns": metrics_list, + "reportTypeId": reportTypeId, + "filters": filters, + "timeUnit": "SUMMARY", + "format": "GZIP_JSON" + } } - if RecordType.ASINS in record_type: - body["campaignType"] = "sponsoredProducts" - if profile.accountInfo.type == "vendor": - metrics_list = copy(metrics_list) - metrics_list.remove("sku") - - # adId is automatically added to the report by amazon and requesting adId causes an amazon error - if "adId" in metrics_list: - metrics_list.remove("adId") - return {**body, "metrics": ",".join(metrics_list)} + return body diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index 7359bbbc884b7..877d1f5f121b3 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -36,6 +36,7 @@ class RecordType(str, Enum): class Status(str, Enum): IN_PROGRESS = "IN_PROGRESS" SUCCESS = "SUCCESS" + COMPLETED = "COMPLETED" FAILURE = "FAILURE" @@ -47,6 +48,7 @@ class ReportInitResponse(BaseModel): class ReportStatus(BaseModel): status: str location: Optional[str] + url: Optional[str] @dataclass @@ -88,7 +90,7 @@ class ReportStream(BasicAmazonAdsStream, ABC): """ Common base class for report streams """ - + API_VERSION = "v2" primary_key = ["profileId", "recordType", "reportDate", "updatedAt"] # https://advertising.amazon.com/API/docs/en-us/reporting/v2/faq#what-is-the-available-report-history-for-the-version-2-reporting-api REPORTING_PERIOD = 60 @@ -96,6 +98,7 @@ class ReportStream(BasicAmazonAdsStream, ABC): # Format used to specify metric generation date over Amazon Ads API. REPORT_DATE_FORMAT = "YYYYMMDD" cursor_field = "reportDate" + report_is_created = HTTPStatus.ACCEPTED ERRORS = [ (400, "KDP authors do not have access to Sponsored Brands functionality"), @@ -138,7 +141,7 @@ def read_records( collects metrics for all profiles and record types. Amazon Ads metric generation works in async way: First we need to initiate creating report for specific profile/record type/date and then constantly check for report - generation status - when it will have "SUCCESS" status then download the + generation status - when it will have "SUCCESS" or "COMPLETED" status then download the report and parse result. """ @@ -198,7 +201,7 @@ def _try_read_records(self, report_infos): if report_status == Status.FAILURE: message = f"Report for {report_info.profile_id} with {report_info.record_type} type generation failed" raise ReportGenerationFailure(message) - elif report_status == Status.SUCCESS: + elif report_status == Status.SUCCESS or report_status == Status.COMPLETED: try: report_info.metric_objects = self._download_report(report_info, download_url) except requests.HTTPError as error: @@ -210,7 +213,7 @@ def _try_read_records(self, report_infos): raise ReportGenerationInProgress(message) def _incomplete_report_infos(self, report_infos): - return [r for r in report_infos if r.status != Status.SUCCESS] + return [r for r in report_infos if r.status != Status.SUCCESS and r.status != Status.COMPLETED] def _generate_model(self): """ @@ -228,7 +231,7 @@ def _get_auth_headers(self, profile_id: int): "Amazon-Advertising-API-ClientId": self._client_id, "Amazon-Advertising-API-Scope": str(profile_id), **self._authenticator.get_auth_header(), - } + } if profile_id else {} @abstractmethod def report_init_endpoint(self, record_type: str) -> str: @@ -249,7 +252,7 @@ def _check_status(self, report_info: ReportInfo) -> Tuple[Status, str]: """ Check report status and return download link if report generated successfuly """ - check_endpoint = f"/v2/reports/{report_info.report_id}" + check_endpoint = f"/{self.API_VERSION}/reports/{report_info.report_id}" resp = self._send_http_request(urljoin(self._url, check_endpoint), report_info.profile_id) try: @@ -257,7 +260,7 @@ def _check_status(self, report_info: ReportInfo) -> Tuple[Status, str]: except ValueError as error: raise ReportStatusFailure(error) - return resp.status, resp.location + return resp.status, resp.location or resp.url @backoff.on_exception( backoff.expo, @@ -371,7 +374,7 @@ def _init_reports(self, profile: Profile, report_date: str) -> List[ReportInfo]: profile.profileId, report_init_body, ) - if response.status_code != HTTPStatus.ACCEPTED: + if response.status_code != self.report_is_created: error_msg = f"Unexpected HTTP status code {response.status_code} when registering {record_type}, {type(self).__name__} for {profile.profileId} profile: {response.text}" if self._skip_known_errors(response): self.logger.warning(error_msg) @@ -401,7 +404,7 @@ def _download_report(self, report_info: ReportInfo, url: str) -> List[dict]: """ Download and parse report result """ - response = self._send_http_request(url, report_info.profile_id) + response = self._send_http_request(url, report_info.profile_id) if report_info else self._send_http_request(url, None) response.raise_for_status() raw_string = decompress(response.content).decode("utf") return json.loads(raw_string) diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md index 79efcff41b576..af6c3d2919167 100644 --- a/docs/integrations/sources/amazon-ads.md +++ b/docs/integrations/sources/amazon-ads.md @@ -94,8 +94,9 @@ Information about expected report generation waiting time you may find [here](ht | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| +| 1.0.0 | 2023-02-03 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Migrate `products_report` stream to API v3 | | 0.1.29 | 2023-01-27 | [22038](https://github.com/airbytehq/airbyte/pull/22038) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 0.1.28 | 2023-01-18 | [19491](https://github.com/airbytehq/airbyte/pull/19491) | Add option to customize look back window value +| 0.1.28 | 2023-01-18 | [19491](https://github.com/airbytehq/airbyte/pull/19491) | Add option to customize look back window value | | 0.1.27 | 2023-01-05 | [21082](https://github.com/airbytehq/airbyte/pull/21082) | Fix bug with handling: "Report date is too far in the past." - partial revert of #20662 | | 0.1.26 | 2022-12-19 | [20662](https://github.com/airbytehq/airbyte/pull/20662) | Fix bug with handling: "Report date is too far in the past." | | 0.1.25 | 2022-11-08 | [18985](https://github.com/airbytehq/airbyte/pull/18985) | Remove "report_wait_timeout", "report_generation_max_retries" from config | From d7be2b2d383f3bfc37adcb01b89c07de6ac11ed7 Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Fri, 3 Feb 2023 04:41:49 +0100 Subject: [PATCH 2/9] Updated PR number --- docs/integrations/sources/amazon-ads.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md index af6c3d2919167..a70efad3c86eb 100644 --- a/docs/integrations/sources/amazon-ads.md +++ b/docs/integrations/sources/amazon-ads.md @@ -94,7 +94,7 @@ Information about expected report generation waiting time you may find [here](ht | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| -| 1.0.0 | 2023-02-03 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Migrate `products_report` stream to API v3 | +| 1.0.0 | 2023-02-03 | [22355](https://github.com/airbytehq/airbyte/pull/22355) | Migrate `products_report` stream to API v3 | | 0.1.29 | 2023-01-27 | [22038](https://github.com/airbytehq/airbyte/pull/22038) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 0.1.28 | 2023-01-18 | [19491](https://github.com/airbytehq/airbyte/pull/19491) | Add option to customize look back window value | | 0.1.27 | 2023-01-05 | [21082](https://github.com/airbytehq/airbyte/pull/21082) | Fix bug with handling: "Report date is too far in the past." - partial revert of #20662 | From 4e259def62a48ce390c8ccfda6b157c232af4782 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 30 Mar 2023 23:33:30 +0300 Subject: [PATCH 3/9] fix Signed-off-by: Your Name --- .../streams/report_streams/products_report.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py index 664b54a522040..a2e268eb0a7f0 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py @@ -245,9 +245,9 @@ "adGroups": "adGroupId", "keywords": "keywordId", "productAds": "adId", - "asins_keywords": "asin", - "asins_targets": "asin", - "targets": "targetId", + "asins_keywords": "advertisedAsin", + "asins_targets": "advertisedAsin", + "targets": "keywordId", } From bdf06ed6fd313ee7e2af7255a4829a47811e1d67 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Thu, 30 Mar 2023 20:40:30 +0000 Subject: [PATCH 4/9] fix tests Signed-off-by: Sergey Chvalyuk --- .../unit_tests/test_report_streams.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py index c8afe17f3ee71..d5febcc003bca 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py @@ -103,15 +103,20 @@ def setup_responses(init_response=None, init_response_products=None, init_respon if init_response_products: responses.add( responses.POST, - re.compile(r"https://advertising-api.amazon.com/v2/sp/[a-zA-Z]+/report"), + re.compile(r"https://advertising-api.amazon.com/reporting/reports"), body=init_response_products, - status=202, + status=200, ) if init_response_brands: responses.add( responses.POST, re.compile(r"https://advertising-api.amazon.com/v2/hsa/[a-zA-Z]+/report"), body=init_response_brands, status=202 ) if status_response: + responses.add( + responses.GET, + re.compile(r"https://advertising-api.amazon.com/reporting/reports/[^/]+$"), + body=status_response, + ) responses.add( responses.GET, re.compile(r"https://advertising-api.amazon.com/v2/reports/[^/]+$"), @@ -178,7 +183,7 @@ def test_products_report_stream(config): profiles = make_profiles(profile_type="vendor") stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock()) - stream_slice = {"profile": profiles[0], "reportDate": "20210725", "retry_count": 3} + stream_slice = {"profile": profiles[0], "reportDate": "2021-07-25", "retry_count": 3} metrics = [m for m in stream.read_records(SyncMode.incremental, stream_slice=stream_slice)] assert len(metrics) == METRICS_COUNT * len(stream.metrics_map) @@ -384,9 +389,9 @@ def test_get_start_date(config): profile_id = str(profiles[0].profileId) stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock()) - assert stream.get_start_date(profiles[0], {profile_id: {"reportDate": "20210810"}}) == Date(2021, 8, 10) + assert stream.get_start_date(profiles[0], {profile_id: {"reportDate": "2021-08-10"}}) == Date(2021, 8, 10) stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock()) - assert stream.get_start_date(profiles[0], {profile_id: {"reportDate": "20210510"}}) == Date(2021, 6, 1) + assert stream.get_start_date(profiles[0], {profile_id: {"reportDate": "2021-05-10"}}) == Date(2021, 6, 1) config.pop("start_date") stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock()) @@ -399,7 +404,7 @@ def test_stream_slices_different_timezones(config): profile2 = Profile(profileId=2, timezone="UTC", accountInfo=AccountInfo(marketplaceStringId="", id="", type="seller")) stream = SponsoredProductsReportStream(config, [profile1, profile2], authenticator=mock.MagicMock()) slices = list(stream.stream_slices(SyncMode.incremental, cursor_field=stream.cursor_field, stream_state={})) - assert slices == [{"profile": profile1, "reportDate": "20210731"}, {"profile": profile2, "reportDate": "20210801"}] + assert slices == [{"profile": profile1, "reportDate": "2021-07-31"}, {"profile": profile2, "reportDate": "2021-08-01"}] def test_stream_slices_lazy_evaluation(config): @@ -728,7 +733,7 @@ def test_products_report_stream_with_custom_record_types(config_gen, custom_reco profiles = make_profiles(profile_type="vendor") stream = SponsoredProductsReportStream(config_gen(report_record_types=custom_record_types), profiles, authenticator=mock.MagicMock()) - stream_slice = {"profile": profiles[0], "reportDate": "20210725", "retry_count": 3} + stream_slice = {"profile": profiles[0], "reportDate": "2021-07-25", "retry_count": 3} records = list(stream.read_records(SyncMode.incremental, stream_slice=stream_slice)) for record in records: print(record) From efd126b90d4791df51cbccd753dce79b761bedf2 Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Fri, 31 Mar 2023 00:04:52 +0200 Subject: [PATCH 5/9] Fixed unttests --- .../connectors/source-amazon-ads/README.md | 2 +- .../unit_tests/test_report_streams.py | 51 +++++++++++-------- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/README.md b/airbyte-integrations/connectors/source-amazon-ads/README.md index 610c77fb435d0..acd73da4c15c1 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/README.md +++ b/airbyte-integrations/connectors/source-amazon-ads/README.md @@ -81,7 +81,7 @@ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integrat Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. First install test dependencies into your virtual environment: ``` -pip install .[tests] +pip install .'[tests]' ``` ### Unit Tests To run unit tests locally, from the connector directory run: diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py index a72a5f709e33f..b409c04e565d8 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py @@ -76,6 +76,12 @@ def setup_responses(init_response=None, init_response_products=None, init_respon body=init_response_products, status=202, ) + responses.add( + responses.POST, + re.compile(r"https://advertising-api.amazon.com/reporting/reports"), + body=init_response_products, + status=200, + ) if init_response_brands: responses.add( responses.POST, re.compile(r"https://advertising-api.amazon.com/v2/hsa/[a-zA-Z]+/report"), body=init_response_brands, status=202 @@ -86,6 +92,11 @@ def setup_responses(init_response=None, init_response_products=None, init_respon re.compile(r"https://advertising-api.amazon.com/v2/reports/[^/]+$"), body=status_response, ) + responses.add( + responses.GET, + re.compile(r"https://advertising-api.amazon.com/reporting/reports"), + body=status_response, + ) if metric_response: responses.add( responses.GET, @@ -147,7 +158,7 @@ def test_products_report_stream(config): profiles = make_profiles(profile_type="vendor") stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock()) - stream_slice = {"profile": profiles[0], "reportDate": "20210725", "retry_count": 3} + stream_slice = {"profile": profiles[0], "reportDate": "2021-07-25", "retry_count": 3} metrics = [m for m in stream.read_records(SyncMode.incremental, stream_slice=stream_slice)] assert len(metrics) == METRICS_COUNT * len(stream.metrics_map) @@ -353,9 +364,9 @@ def test_get_start_date(config): profile_id = str(profiles[0].profileId) stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock()) - assert stream.get_start_date(profiles[0], {profile_id: {"reportDate": "20210810"}}) == Date(2021, 8, 10) + assert stream.get_start_date(profiles[0], {profile_id: {"reportDate": "2021-08-10"}}) == Date(2021, 8, 10) stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock()) - assert stream.get_start_date(profiles[0], {profile_id: {"reportDate": "20210510"}}) == Date(2021, 6, 1) + assert stream.get_start_date(profiles[0], {profile_id: {"reportDate": "2021-05-10"}}) == Date(2021, 6, 1) config.pop("start_date") stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock()) @@ -368,7 +379,7 @@ def test_stream_slices_different_timezones(config): profile2 = Profile(profileId=2, timezone="UTC", accountInfo=AccountInfo(marketplaceStringId="", id="", type="seller")) stream = SponsoredProductsReportStream(config, [profile1, profile2], authenticator=mock.MagicMock()) slices = list(stream.stream_slices(SyncMode.incremental, cursor_field=stream.cursor_field, stream_state={})) - assert slices == [{"profile": profile1, "reportDate": "20210731"}, {"profile": profile2, "reportDate": "20210801"}] + assert slices == [{"profile": profile1, "reportDate": "2021-07-31"}, {"profile": profile2, "reportDate": "2021-08-01"}] def test_stream_slices_lazy_evaluation(config): @@ -386,19 +397,19 @@ def test_stream_slices_lazy_evaluation(config): frozen_datetime.tick(delta=timedelta(minutes=10)) assert slices == [ - {"profile": profile1, "reportDate": "20220527"}, - {"profile": profile2, "reportDate": "20220528"}, - {"profile": profile1, "reportDate": "20220528"}, - {"profile": profile2, "reportDate": "20220529"}, - {"profile": profile1, "reportDate": "20220529"}, - {"profile": profile2, "reportDate": "20220530"}, - {"profile": profile1, "reportDate": "20220530"}, - {"profile": profile2, "reportDate": "20220531"}, - {"profile": profile1, "reportDate": "20220531"}, - {"profile": profile2, "reportDate": "20220601"}, - {"profile": profile1, "reportDate": "20220601"}, - {"profile": profile2, "reportDate": "20220602"}, - {"profile": profile1, "reportDate": "20220602"}, + {"profile": profile1, "reportDate": "2022-05-27"}, + {"profile": profile2, "reportDate": "2022-05-28"}, + {"profile": profile1, "reportDate": "2022-05-28"}, + {"profile": profile2, "reportDate": "2022-05-29"}, + {"profile": profile1, "reportDate": "2022-05-29"}, + {"profile": profile2, "reportDate": "2022-05-30"}, + {"profile": profile1, "reportDate": "2022-05-30"}, + {"profile": profile2, "reportDate": "2022-05-31"}, + {"profile": profile1, "reportDate": "2022-05-31"}, + {"profile": profile2, "reportDate": "2022-06-01"}, + {"profile": profile1, "reportDate": "2022-06-01"}, + {"profile": profile2, "reportDate": "2022-06-02"}, + {"profile": profile1, "reportDate": "2022-06-02"}, ] @@ -407,10 +418,10 @@ def test_get_date_range_lazy_evaluation(): with freeze_time("2022-06-01T12:00:00+00:00") as frozen_datetime: date_range = list(get_date_range(start_date=Date(2022, 5, 29), timezone="UTC")) - assert date_range == ["20220529", "20220530", "20220531", "20220601"] + assert date_range == ["2022-05-29", "2022-05-30", "2022-05-31", "2022-06-01"] date_range = list(get_date_range(start_date=Date(2022, 6, 1), timezone="UTC")) - assert date_range == ["20220601"] + assert date_range == ["2022-06-01"] date_range = list(get_date_range(start_date=Date(2022, 6, 2), timezone="UTC")) assert date_range == [] @@ -419,7 +430,7 @@ def test_get_date_range_lazy_evaluation(): for date in get_date_range(start_date=Date(2022, 5, 29), timezone="UTC"): date_range.append(date) frozen_datetime.tick(delta=timedelta(hours=3)) - assert date_range == ["20220529", "20220530", "20220531", "20220601", "20220602"] + assert date_range == ["2022-05-29", "2022-05-30", "2022-05-31", "2022-06-01", "2022-06-02"] @responses.activate From 0ed84e6af2d1676bbe50387077bd3b0484b4eb6d Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Fri, 31 Mar 2023 00:38:59 +0200 Subject: [PATCH 6/9] Fix Metric response --- .../unit_tests/test_report_streams.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py index fb0b3c1770e42..26702713b8763 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py @@ -3,7 +3,7 @@ # import re -from base64 import b64decode +from base64 import b64decode, b64encode from datetime import timedelta from functools import partial from unittest import mock @@ -41,6 +41,7 @@ "adId": "665320125", "targetId": "791320341", "asin": "G000PSH142", + "advertisedAsin": "G000PSH142", "keywordBid": "511234974", "keywordId": "965783021" }, @@ -51,6 +52,7 @@ "adId": "665320125", "targetId": "791320341", "asin": "G000PSH142", + "advertisedAsin": "G000PSH142", "keywordBid": "511234974", "keywordId": "965783021" }, @@ -61,6 +63,7 @@ "adId": "665320125", "targetId": "791320341", "asin": "G000PSH142", + "advertisedAsin": "G000PSH142", "keywordBid": "511234974", "keywordId": "965783021" }, @@ -71,6 +74,7 @@ "adId": "665320125", "targetId": "791320341", "asin": "G000PSH142", + "advertisedAsin": "G000PSH142", "keywordBid": "511234974", "keywordId": "965783021" }, @@ -81,6 +85,7 @@ "adId": "665320125", "targetId": "791320341", "asin": "G000PSH142", + "advertisedAsin": "G000PSH142", "keywordBid": "511234974", "keywordId": "965783021" } @@ -88,10 +93,12 @@ """ METRIC_RESPONSE = b64decode( """ -H4sIANnqymMC/92SsYrCQBBA+3zFsrWBmdnZ7K6lTbSRgyvFYjFBwl2iJIqI+O+3p2aPEyxSmmKLnceb4jHJKhHiEp -4QcuPrva+2zaKQU0HIYCyTnfyHS1+XAcsu/L/LtB+nTZinUZIPyxd5uzvubxtlxg5Q8R97jDOtCJB0Dw6+3ZaHOzQO -A1SM0eqq5hfkAPDxOUemnnyV59OuLWbVTdSIpNgZfsL3tS7TxioglAFeJy8aMGtgbWlIgt4ZRwENDpmtGnQFURpHA1 -JokGDYGURpHA2s0woyYBjSIErv1SBZJz+HyV3zFgUAAA== + H4sIAAAAAAAACuWSPWvDMBCGd/8KoTmGu9PJkrq1S5olBDqWDqIWwbRxgu + 02hJD/HjX+ooUMXutBg95Hzwle7jUR4hyPEPLd7w6+2JarXD4IQgZjmezi + N1z7XYhY1vH+GdI+TsuYp4MkO8vny2r/dbhNlBk7QMUj6+JMKwIk3YPGV9 + vQtNA4jFAxDlZdlD9gCQCbl2dkGud9h6op6pA/3n3zEU7HfZU/FbfhGpEU + O8N/cPu1y7SxCghlhJfFnZ6YNbC2NKWm3plPSxocMls1aZsGaT49kUKDBN + PWaZDm05N1WkEGDFN6sr30/3pK3q5AhIPlyAUAAA== """ ) METRICS_COUNT = 5 From 8ddfdf24ca11e8814e03049cf2c5503318d78f7c Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Fri, 31 Mar 2023 00:46:47 +0200 Subject: [PATCH 7/9] Fix typo --- .../source-amazon-ads/unit_tests/test_report_streams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py index 26702713b8763..be13de943fbdd 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py @@ -3,7 +3,7 @@ # import re -from base64 import b64decode, b64encode +from base64 import b64decode from datetime import timedelta from functools import partial from unittest import mock From 46c7ec4556621dd2df46e00bc2665e45e149b447 Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Fri, 31 Mar 2023 01:19:54 +0200 Subject: [PATCH 8/9] Fix integrations test tdue migration --- .../source-amazon-ads/acceptance-test-config.yml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml index 213a95432d9a8..1a94dbe998546 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml @@ -42,20 +42,9 @@ acceptance_tests: tests: - config_path: secrets/config.json configured_catalog_path: integration_tests/configured_catalog.json - - config_path: secrets/config_report.json - configured_catalog_path: integration_tests/configured_catalog_report.json timeout_seconds: 3600 incremental: - tests: - - config_path: secrets/config_report.json - configured_catalog_path: integration_tests/configured_catalog_report.json - cursor_paths: - sponsored_products_report_stream: - - "1861552880916640" - - reportDate - future_state: - future_state_path: integration_tests/abnormal_state.json - timeout_seconds: 2400 + bypass_reason: "can't populate stream because it requires real ad campaign" spec: tests: - spec_path: integration_tests/spec.json From 263be8b68e5cf135f965212f840210f532181d9d Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Thu, 30 Mar 2023 23:42:01 +0000 Subject: [PATCH 9/9] auto-bump connector version --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../streams/report_streams/products_report.py | 355 +++++++++--------- .../streams/report_streams/report_streams.py | 15 +- connectors.md | 2 +- 5 files changed, 191 insertions(+), 185 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 88815b9885d40..5c9bd45a48c4e 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -65,7 +65,7 @@ - name: Amazon Ads sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246 dockerRepository: airbyte/source-amazon-ads - dockerImageTag: 1.0.1 + dockerImageTag: 1.0.2 documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-ads icon: amazonads.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 471cd825eb5d9..1d027721d7381 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -822,7 +822,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-amazon-ads:1.0.1" +- dockerImage: "airbyte/source-amazon-ads:1.0.2" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/amazon-ads" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py index a2e268eb0a7f0..036dc67d8258e 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py @@ -4,188 +4,189 @@ from http import HTTPStatus -from .report_streams import ReportStream, ReportInfo from typing import List +from .report_streams import ReportInfo, ReportStream + METRICS_MAP = { "campaigns": [ - "campaignName", - "campaignId", - "campaignStatus", - "campaignBudgetAmount", - "campaignRuleBasedBudgetAmount", - "campaignApplicableBudgetRuleId", - "campaignApplicableBudgetRuleName", - "impressions", - "clicks", - "cost", - "purchases1d", - "purchases7d", - "purchases14d", - "purchases30d", - "purchasesSameSku1d", - "purchasesSameSku7d", - "purchasesSameSku14d", - "purchasesSameSku30d", - "unitsSoldClicks1d", - "unitsSoldClicks7d", - "unitsSoldClicks14d", - "unitsSoldClicks30d", - "sales1d", - "sales7d", - "sales14d", - "sales30d", - "attributedSalesSameSku1d", - "attributedSalesSameSku7d", - "attributedSalesSameSku14d", - "attributedSalesSameSku30d", - "unitsSoldSameSku1d", - "unitsSoldSameSku7d", - "unitsSoldSameSku14d", - "unitsSoldSameSku30d", - ], + "campaignName", + "campaignId", + "campaignStatus", + "campaignBudgetAmount", + "campaignRuleBasedBudgetAmount", + "campaignApplicableBudgetRuleId", + "campaignApplicableBudgetRuleName", + "impressions", + "clicks", + "cost", + "purchases1d", + "purchases7d", + "purchases14d", + "purchases30d", + "purchasesSameSku1d", + "purchasesSameSku7d", + "purchasesSameSku14d", + "purchasesSameSku30d", + "unitsSoldClicks1d", + "unitsSoldClicks7d", + "unitsSoldClicks14d", + "unitsSoldClicks30d", + "sales1d", + "sales7d", + "sales14d", + "sales30d", + "attributedSalesSameSku1d", + "attributedSalesSameSku7d", + "attributedSalesSameSku14d", + "attributedSalesSameSku30d", + "unitsSoldSameSku1d", + "unitsSoldSameSku7d", + "unitsSoldSameSku14d", + "unitsSoldSameSku30d", + ], "adGroups": [ - "campaignName", - "campaignId", - "adGroupName", - "adGroupId", - "impressions", - "clicks", - "cost", - "purchases1d", - "purchases7d", - "purchases14d", - "purchases30d", - "purchasesSameSku1d", - "purchasesSameSku7d", - "purchasesSameSku14d", - "purchasesSameSku30d", - "unitsSoldClicks1d", - "unitsSoldClicks7d", - "unitsSoldClicks14d", - "unitsSoldClicks30d", - "sales1d", - "sales7d", - "sales14d", - "sales30d", - "attributedSalesSameSku1d", - "attributedSalesSameSku7d", - "attributedSalesSameSku14d", - "attributedSalesSameSku30d", - "unitsSoldSameSku1d", - "unitsSoldSameSku7d", - "unitsSoldSameSku14d", - "unitsSoldSameSku30d", - ], + "campaignName", + "campaignId", + "adGroupName", + "adGroupId", + "impressions", + "clicks", + "cost", + "purchases1d", + "purchases7d", + "purchases14d", + "purchases30d", + "purchasesSameSku1d", + "purchasesSameSku7d", + "purchasesSameSku14d", + "purchasesSameSku30d", + "unitsSoldClicks1d", + "unitsSoldClicks7d", + "unitsSoldClicks14d", + "unitsSoldClicks30d", + "sales1d", + "sales7d", + "sales14d", + "sales30d", + "attributedSalesSameSku1d", + "attributedSalesSameSku7d", + "attributedSalesSameSku14d", + "attributedSalesSameSku30d", + "unitsSoldSameSku1d", + "unitsSoldSameSku7d", + "unitsSoldSameSku14d", + "unitsSoldSameSku30d", + ], "keywords": [ - "campaignName", - "campaignId", - "adGroupName", - "adGroupId", - "keywordId", - "keyword", - "matchType", - "impressions", - "clicks", - "cost", - "purchases1d", - "purchases7d", - "purchases14d", - "purchases30d", - "purchasesSameSku1d", - "purchasesSameSku7d", - "purchasesSameSku14d", - "purchasesSameSku30d", - "unitsSoldClicks1d", - "unitsSoldClicks7d", - "unitsSoldClicks14d", - "unitsSoldClicks30d", - "sales1d", - "sales7d", - "sales14d", - "sales30d", - "attributedSalesSameSku1d", - "attributedSalesSameSku7d", - "attributedSalesSameSku14d", - "attributedSalesSameSku30d", - "unitsSoldSameSku1d", - "unitsSoldSameSku7d", - "unitsSoldSameSku14d", - "unitsSoldSameSku30d", - ], + "campaignName", + "campaignId", + "adGroupName", + "adGroupId", + "keywordId", + "keyword", + "matchType", + "impressions", + "clicks", + "cost", + "purchases1d", + "purchases7d", + "purchases14d", + "purchases30d", + "purchasesSameSku1d", + "purchasesSameSku7d", + "purchasesSameSku14d", + "purchasesSameSku30d", + "unitsSoldClicks1d", + "unitsSoldClicks7d", + "unitsSoldClicks14d", + "unitsSoldClicks30d", + "sales1d", + "sales7d", + "sales14d", + "sales30d", + "attributedSalesSameSku1d", + "attributedSalesSameSku7d", + "attributedSalesSameSku14d", + "attributedSalesSameSku30d", + "unitsSoldSameSku1d", + "unitsSoldSameSku7d", + "unitsSoldSameSku14d", + "unitsSoldSameSku30d", + ], "targets": [ - "campaignName", - "campaignId", - "adGroupName", - "adGroupId", - "keywordId", - "keyword", - "targeting", - "keywordType", - "impressions", - "clicks", - "cost", - "purchases1d", - "purchases7d", - "purchases14d", - "purchases30d", - "purchasesSameSku1d", - "purchasesSameSku7d", - "purchasesSameSku14d", - "purchasesSameSku30d", - "unitsSoldClicks1d", - "unitsSoldClicks7d", - "unitsSoldClicks14d", - "unitsSoldClicks30d", - "sales1d", - "sales7d", - "sales14d", - "sales30d", - "attributedSalesSameSku1d", - "attributedSalesSameSku7d", - "attributedSalesSameSku14d", - "attributedSalesSameSku30d", - "unitsSoldSameSku1d", - "unitsSoldSameSku7d", - "unitsSoldSameSku14d", - "unitsSoldSameSku30d", - ], + "campaignName", + "campaignId", + "adGroupName", + "adGroupId", + "keywordId", + "keyword", + "targeting", + "keywordType", + "impressions", + "clicks", + "cost", + "purchases1d", + "purchases7d", + "purchases14d", + "purchases30d", + "purchasesSameSku1d", + "purchasesSameSku7d", + "purchasesSameSku14d", + "purchasesSameSku30d", + "unitsSoldClicks1d", + "unitsSoldClicks7d", + "unitsSoldClicks14d", + "unitsSoldClicks30d", + "sales1d", + "sales7d", + "sales14d", + "sales30d", + "attributedSalesSameSku1d", + "attributedSalesSameSku7d", + "attributedSalesSameSku14d", + "attributedSalesSameSku30d", + "unitsSoldSameSku1d", + "unitsSoldSameSku7d", + "unitsSoldSameSku14d", + "unitsSoldSameSku30d", + ], "productAds": [ - "campaignName", - "campaignId", - "adGroupName", - "adGroupId", - "adId", - "impressions", - "clicks", - "cost", - "campaignBudgetCurrencyCode", - "advertisedAsin", - "purchases1d", - "purchases7d", - "purchases14d", - "purchases30d", - "purchasesSameSku1d", - "purchasesSameSku7d", - "purchasesSameSku14d", - "purchasesSameSku30d", - "unitsSoldClicks1d", - "unitsSoldClicks7d", - "unitsSoldClicks14d", - "unitsSoldClicks30d", - "sales1d", - "sales7d", - "sales14d", - "sales30d", - "attributedSalesSameSku1d", - "attributedSalesSameSku7d", - "attributedSalesSameSku14d", - "attributedSalesSameSku30d", - "unitsSoldSameSku1d", - "unitsSoldSameSku7d", - "unitsSoldSameSku14d", - "unitsSoldSameSku30d", - ], + "campaignName", + "campaignId", + "adGroupName", + "adGroupId", + "adId", + "impressions", + "clicks", + "cost", + "campaignBudgetCurrencyCode", + "advertisedAsin", + "purchases1d", + "purchases7d", + "purchases14d", + "purchases30d", + "purchasesSameSku1d", + "purchasesSameSku7d", + "purchasesSameSku14d", + "purchasesSameSku30d", + "unitsSoldClicks1d", + "unitsSoldClicks7d", + "unitsSoldClicks14d", + "unitsSoldClicks30d", + "sales1d", + "sales7d", + "sales14d", + "sales30d", + "attributedSalesSameSku1d", + "attributedSalesSameSku7d", + "attributedSalesSameSku14d", + "attributedSalesSameSku30d", + "unitsSoldSameSku1d", + "unitsSoldSameSku7d", + "unitsSoldSameSku14d", + "unitsSoldSameSku30d", + ], "asins_keywords": [ "campaignName", "campaignId", @@ -236,7 +237,7 @@ "keywordId", "targeting", "keywordType", - ] + ], } @@ -311,8 +312,8 @@ def _get_init_report_body(self, report_date: str, record_type: str, profile): "reportTypeId": reportTypeId, "filters": filters, "timeUnit": "SUMMARY", - "format": "GZIP_JSON" - } + "format": "GZIP_JSON", + }, } return body diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index a3e9c5980a0b7..3245957196771 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -90,6 +90,7 @@ class ReportStream(BasicAmazonAdsStream, ABC): """ Common base class for report streams """ + API_VERSION = "v2" primary_key = ["profileId", "recordType", "reportDate", "recordId"] # https://advertising.amazon.com/API/docs/en-us/reporting/v2/faq#what-is-the-available-report-history-for-the-version-2-reporting-api @@ -228,11 +229,15 @@ def _generate_model(self): return MetricsReport.generate_metric_model(metrics) def _get_auth_headers(self, profile_id: int): - return { - "Amazon-Advertising-API-ClientId": self._client_id, - "Amazon-Advertising-API-Scope": str(profile_id), - **self._authenticator.get_auth_header(), - } if profile_id else {} + return ( + { + "Amazon-Advertising-API-ClientId": self._client_id, + "Amazon-Advertising-API-Scope": str(profile_id), + **self._authenticator.get_auth_header(), + } + if profile_id + else {} + ) @abstractmethod def report_init_endpoint(self, record_type: str) -> str: diff --git a/connectors.md b/connectors.md index 0291c454b5930..10099dcdcb804 100644 --- a/connectors.md +++ b/connectors.md @@ -11,7 +11,7 @@ | **Airtable** | Airtable icon | Source | airbyte/source-airtable:3.0.0 | generally_available | [link](https://docs.airbyte.com/integrations/sources/airtable) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-airtable) | `14c6e7ea-97ed-4f5e-a7b5-25e9a80b8212` | | **AlloyDB for PostgreSQL** | AlloyDB for PostgreSQL icon | Source | airbyte/source-alloydb:2.0.13 | generally_available | [link](https://docs.airbyte.com/integrations/sources/alloydb) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-alloydb) | `1fa90628-2b9e-11ed-a261-0242ac120002` | | **Alpha Vantage** | Alpha Vantage icon | Source | airbyte/source-alpha-vantage:0.1.1 | alpha | [link](https://docs.airbyte.com/integrations/sources/alpha-vantage) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-alpha-vantage) | `db385323-9333-4fec-bec3-9e0ca9326c90` | -| **Amazon Ads** | Amazon Ads icon | Source | airbyte/source-amazon-ads:1.0.1 | generally_available | [link](https://docs.airbyte.com/integrations/sources/amazon-ads) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-amazon-ads) | `c6b0a29e-1da9-4512-9002-7bfd0cba2246` | +| **Amazon Ads** | Amazon Ads icon | Source | airbyte/source-amazon-ads:1.0.2 | generally_available | [link](https://docs.airbyte.com/integrations/sources/amazon-ads) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-amazon-ads) | `c6b0a29e-1da9-4512-9002-7bfd0cba2246` | | **Amazon SQS** | Amazon SQS icon | Source | airbyte/source-amazon-sqs:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/amazon-sqs) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-amazon-sqs) | `983fd355-6bf3-4709-91b5-37afa391eeb6` | | **Amazon Seller Partner** | Amazon Seller Partner icon | Source | airbyte/source-amazon-seller-partner:1.0.1 | alpha | [link](https://docs.airbyte.com/integrations/sources/amazon-seller-partner) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-amazon-seller-partner) | `e55879a8-0ef8-4557-abcf-ab34c53ec460` | | **Amplitude** | Amplitude icon | Source | airbyte/source-amplitude:0.1.23 | generally_available | [link](https://docs.airbyte.com/integrations/sources/amplitude) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-amplitude) | `fa9f58c6-2d03-4237-aaa4-07d75e0c1396` |