🎉 Amazon Ads - new streams for bids and keyword recommendations

airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml

@@ -31,6 +31,12 @@ acceptance_tests:
           sponsored_product_campaigns:
             - name: dailyBudget
               bypass_reason: "can be updated, also it is sometimes integer, sometimes float"
+          sponsored_product_ad_group_suggested_keywords:
+            - name: suggestedKeywords
+              bypass_reason: "value can be changed because it is real-life recommendation from Amazon"
+          sponsored_product_ad_group_bid_recommendations:
+            - name: suggestedBid
+              bypass_reason: "value can be changed because it is real-life recommendation from Amazon"
         timeout_seconds: 2400
         expect_records:
           path: integration_tests/expected_records.jsonl
@@ -44,7 +50,7 @@ acceptance_tests:
     tests:
       - config_path: secrets/config.json
         backward_compatibility_tests_config:
-          disable_for_version: 1.1.0
+          disable_for_version: 2.1.0
   full_refresh:
     tests:
       - config_path: secrets/config.json

airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog.json

@@ -140,6 +140,26 @@
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"
     },
+    {
+      "stream": {
+        "name": "sponsored_product_ad_group_suggested_keywords",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_primary_key": [["adGroupId"]]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "sponsored_product_ad_group_bid_recommendations",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_primary_key": [["adGroupId"]]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
     {
       "stream": {
         "name": "sponsored_brands_keywords",

airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_records.jsonl

@@ -128,4 +128,6 @@
 {"stream":"sponsored_display_budget_rules","data":{"createdDate":1657024512836,"lastUpdatedDate":1657024512836,"ruleDetails":{"budgetIncreaseBy":{"type":"PERCENT","value":32},"duration":{"dateRangeTypeRuleDuration":null,"eventTypeRuleDuration":{"endDate":"20220713","eventId":"ae0226d3-9f97-5122-a749-2e9ba741a2dc","eventName":"Prime Day","startDate":"20220712"}},"name":"ex","performanceMeasureCondition":null,"recurrence":{"daysOfWeek":null,"intraDaySchedule":null,"type":"DAILY"},"ruleType":"SCHEDULE"},"ruleId":"b5abeec6-7624-49e7-8571-97b8ba61551e","ruleState":"ACTIVE","ruleStatus":"EXPIRED","ruleStatusDetails":null},"emitted_at":1687254964816}
 {"stream":"sponsored_display_budget_rules","data":{"createdDate":1686765545918,"lastUpdatedDate":1686765545918,"ruleDetails":{"budgetIncreaseBy":{"type":"PERCENT","value":1},"duration":{"dateRangeTypeRuleDuration":null,"eventTypeRuleDuration":{"endDate":"20230619","eventId":"553ddee0-8178-544b-a54a-f8918d21ad5f","eventName":"Father's Day","startDate":"20230611"}},"name":"Rule for vadim","performanceMeasureCondition":null,"recurrence":{"daysOfWeek":null,"intraDaySchedule":null,"type":"DAILY"},"ruleType":"SCHEDULE"},"ruleId":"039ff522-f785-4409-8f3a-f6f884ec1750","ruleState":"ACTIVE","ruleStatus":"EXPIRED","ruleStatusDetails":null},"emitted_at":1687254965077}
 {"stream": "portfolios", "data": {"portfolioId": 253945852845204, "name": "Test Portfolio 2", "inBudget": true, "state": "enabled", "creationDate": 1687510907465, "lastUpdatedDate": 1687510907465, "servingStatus": "PORTFOLIO_STATUS_ENABLED"}, "emitted_at": 1688475309870}
-{"stream": "portfolios", "data": {"portfolioId": 270076898441727, "name": "Test Portfolio", "budget": {"amount": 1.0, "currencyCode": "USD", "policy": "dateRange", "startDate": "20230623", "endDate": "20230624"}, "inBudget": true, "state": "enabled", "creationDate": 1687510616329, "lastUpdatedDate": 1687514774484, "servingStatus": "PORTFOLIO_ENDED"}, "emitted_at": 1688475309871}
+{"stream": "portfolios", "data": {"portfolioId": 270076898441727, "name": "Test Portfolio", "budget": {"amount": 1.0, "currencyCode": "USD", "policy": "dateRange", "startDate": "20230623", "endDate": "20230624"}, "inBudget": true, "state": "enabled", "creationDate": 1687510616329, "lastUpdatedDate": 1687514774484, "servingStatus": "PORTFOLIO_ENDED"}, "emitted_at": 1688475309871}
+{"stream":"sponsored_product_ad_group_suggested_keywords","data":{"adGroupId":103188883625219,"suggestedKeywords":[{"keywordText":"disposable hotel slippers","matchType":"broad"},{"keywordText":"hotel slippers women","matchType":"broad"},{"keywordText":"slippers bulk","matchType":"broad"},{"keywordText":"spa slipper","matchType":"broad"},{"keywordText":"disposable guest slippers","matchType":"broad"},{"keywordText":"hotel slipper","matchType":"broad"},{"keywordText":"black bulk slippers","matchType":"broad"},{"keywordText":"disposable black slippers","matchType":"broad"},{"keywordText":"toothbrush oral b medium","matchType":"broad"},{"keywordText":"toothbrush soft oral b","matchType":"broad"},{"keywordText":"diamond cat food wet","matchType":"broad"},{"keywordText":"toothbrush 1 count","matchType":"broad"},{"keywordText":"black slipper pack","matchType":"broad"},{"keywordText":"toothbrush medium","matchType":"broad"},{"keywordText":"peach mango propel water","matchType":"broad"},{"keywordText":"black guest slippers","matchType":"broad"},{"keywordText":"black hotel slippers","matchType":"broad"},{"keywordText":"black house slippers bulk","matchType":"broad"},{"keywordText":"black spa slippers","matchType":"broad"},{"keywordText":"diamond natural wet cat food","matchType":"broad"},{"keywordText":"house slippers 6 pack","matchType":"broad"},{"keywordText":"house slippers guests bulk","matchType":"broad"},{"keywordText":"single toothbrush","matchType":"broad"},{"keywordText":"spa slippers women black","matchType":"broad"},{"keywordText":"toothbrush oral b manual","matchType":"broad"},{"keywordText":"medium toothbrush single","matchType":"broad"},{"keywordText":"toothbrush charcoal","matchType":"broad"},{"keywordText":"toothbrush hard","matchType":"broad"},{"keywordText":"tooth brush medium","matchType":"broad"},{"keywordText":"tooth brush soft","matchType":"broad"},{"keywordText":"house slippers disposable","matchType":"broad"},{"keywordText":"toothbrush oral b","matchType":"broad"},{"keywordText":"toothbrush soft extra","matchType":"broad"},{"keywordText":"black disposable slippers guests","matchType":"broad"},{"keywordText":"bulk slippers guests washable","matchType":"broad"},{"keywordText":"crest toothbrush","matchType":"broad"},{"keywordText":"toothbrush travel","matchType":"broad"},{"keywordText":"black spa slippers bulk","matchType":"broad"},{"keywordText":"house slippers bulk","matchType":"broad"},{"keywordText":"house slippers visitor","matchType":"broad"},{"keywordText":"disposable slipper","matchType":"broad"},{"keywordText":"spa slippers disposable black","matchType":"broad"},{"keywordText":"toothbrush small","matchType":"broad"},{"keywordText":"cepillo de dientes","matchType":"broad"},{"keywordText":"guest slippers bulk","matchType":"broad"},{"keywordText":"soft toothbrush","matchType":"broad"},{"keywordText":"spa house slippers","matchType":"broad"},{"keywordText":"toothbrush firm","matchType":"broad"},{"keywordText":"toothbrush sensitive","matchType":"broad"},{"keywordText":"bulk pack slippers","matchType":"broad"},{"keywordText":"house guest slippers","matchType":"broad"},{"keywordText":"propel peach water","matchType":"broad"},{"keywordText":"teeth brush","matchType":"broad"},{"keywordText":"tooth brush oral b","matchType":"broad"},{"keywordText":"toothbrush amazon fresh","matchType":"broad"},{"keywordText":"toothbrush oralb","matchType":"broad"},{"keywordText":"toothbrush whitening","matchType":"broad"},{"keywordText":"toothbrusj","matchType":"broad"},{"keywordText":"water propel","matchType":"broad"},{"keywordText":"extra soft tooth brush","matchType":"broad"},{"keywordText":"house slippers guests washable","matchType":"broad"},{"keywordText":"propel peach mango","matchType":"broad"},{"keywordText":"tootbrush","matchType":"broad"},{"keywordText":"toothbrush","matchType":"broad"},{"keywordText":"toothbrush bamboo","matchType":"broad"},{"keywordText":"diamond naturals canned cat food","matchType":"broad"},{"keywordText":"organic potato","matchType":"broad"},{"keywordText":"spa slippers bulk","matchType":"broad"},{"keywordText":"toothbrush oral b white","matchType":"broad"},{"keywordText":"toothbrush soft bristle","matchType":"broad"},{"keywordText":"slippers 12 pair","matchType":"broad"},{"keywordText":"black house guest slippers","matchType":"broad"},{"keywordText":"black house slippers pack","matchType":"broad"},{"keywordText":"black slippers set","matchType":"broad"},{"keywordText":"black washable slippers","matchType":"broad"},{"keywordText":"black washable spa slippers","matchType":"broad"},{"keywordText":"bulk house shoes guests","matchType":"broad"},{"keywordText":"diamond naturals cat food can","matchType":"broad"},{"keywordText":"diamond naturals kitten food wet","matchType":"broad"},{"keywordText":"dispisable slippers","matchType":"broad"},{"keywordText":"disposable house slippers black","matchType":"broad"},{"keywordText":"disposable spa slippers bulk","matchType":"broad"},{"keywordText":"disposable washable slippers","matchType":"broad"},{"keywordText":"disposal house slippers","matchType":"broad"},{"keywordText":"fisposable slippers","matchType":"broad"},{"keywordText":"guest slippers washable set","matchType":"broad"},{"keywordText":"hoise slippers","matchType":"broad"},{"keywordText":"home slipper set","matchType":"broad"},{"keywordText":"house alippers guests","matchType":"broad"},{"keywordText":"house shoes guests washable","matchType":"broad"},{"keywordText":"house slipeprs","matchType":"broad"},{"keywordText":"disposable house slippers guest","matchType":"broad"},{"keywordText":"disposable slippers women","matchType":"broad"},{"keywordText":"disposable spa slippers","matchType":"broad"},{"keywordText":"hotel slippers bulk","matchType":"broad"},{"keywordText":"disposable slippers travel","matchType":"broad"},{"keywordText":"one time use slippers","matchType":"broad"},{"keywordText":"pack slippers guest","matchType":"broad"},{"keywordText":"guest slipper","matchType":"broad"},{"keywordText":"guest slippers washable","matchType":"broad"}]},"emitted_at":1688632533382}
+{"stream":"sponsored_product_ad_group_bid_recommendations","data":{"adGroupId":183961953969922,"suggestedBid":{"rangeEnd":1.71,"rangeStart":0.14,"suggested":0.62}},"emitted_at":1688632722904}

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/__init__.py

@@ -6,7 +6,14 @@
 from .profile import Profile
 from .sponsored_brands import BrandsAdGroup, BrandsCampaign
 from .sponsored_display import DisplayAdGroup, DisplayBudgetRules, DisplayCampaign, DisplayProductAds, DisplayTargeting
-from .sponsored_products import ProductAd, ProductAdGroups, ProductCampaign, ProductTargeting
+from .sponsored_products import (
+    ProductAd,
+    ProductAdGroupBidRecommendations,
+    ProductAdGroups,
+    ProductAdGroupSuggestedKeywords,
+    ProductCampaign,
+    ProductTargeting,
+)
 
 __all__ = [
     "BrandsAdGroup",
@@ -23,6 +30,8 @@
     "Portfolio",
     "ProductAd",
     "ProductAdGroups",
+    "ProductAdGroupBidRecommendations",
+    "ProductAdGroupSuggestedKeywords",
     "ProductCampaign",
     "ProductTargeting",
     "Profile",

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_products.py

@@ -3,7 +3,7 @@
 #
 
 from decimal import Decimal
-from typing import Dict, List
+from typing import Dict, List, Optional
 
 from .common import CatalogModel, Targeting
 
@@ -42,6 +42,27 @@ class ProductAdGroups(CatalogModel):
     state: str
 
 
+class SuggestedBid(CatalogModel):
+    suggested: Decimal
+    rangeStart: Decimal
+    rangeEnd: Decimal
+
+
+class ProductAdGroupBidRecommendations(CatalogModel):
+    adGroupId: Decimal
+    suggestedBid: Optional[SuggestedBid] = None
+
+
+class SuggestedKeyword(CatalogModel):
+    keywordText: str
+    matchType: str
+
+
+class ProductAdGroupSuggestedKeywords(CatalogModel):
+    adGroupId: Decimal
+    suggestedKeywords: List[SuggestedKeyword] = None
+
+
 class ProductAd(CatalogModel):
     adId: Decimal
     campaignId: Decimal

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py

@@ -31,7 +31,9 @@
     SponsoredDisplayProductAds,
     SponsoredDisplayReportStream,
     SponsoredDisplayTargetings,
+    SponsoredProductAdGroupBidRecommendations,
     SponsoredProductAdGroups,
+    SponsoredProductAdGroupSuggestedKeywords,
     SponsoredProductAds,
     SponsoredProductCampaignNegativeKeywords,
     SponsoredProductCampaigns,
@@ -105,6 +107,8 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
             SponsoredDisplayBudgetRules,
             SponsoredProductCampaigns,
             SponsoredProductAdGroups,
+            SponsoredProductAdGroupBidRecommendations,
+            SponsoredProductAdGroupSuggestedKeywords,
             SponsoredProductKeywords,
             SponsoredProductNegativeKeywords,
             SponsoredProductCampaignNegativeKeywords,

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/__init__.py

@@ -25,7 +25,9 @@
     SponsoredDisplayTargetings,
 )
 from .sponsored_products import (
+    SponsoredProductAdGroupBidRecommendations,
     SponsoredProductAdGroups,
+    SponsoredProductAdGroupSuggestedKeywords,
     SponsoredProductAds,
     SponsoredProductCampaignNegativeKeywords,
     SponsoredProductCampaigns,
@@ -43,6 +45,8 @@
     "SponsoredDisplayTargetings",
     "SponsoredDisplayBudgetRules",
     "SponsoredProductAdGroups",
+    "SponsoredProductAdGroupBidRecommendations",
+    "SponsoredProductAdGroupSuggestedKeywords",
     "SponsoredProductAds",
     "SponsoredProductCampaigns",
     "SponsoredProductKeywords",

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py

@@ -50,11 +50,11 @@ class to provide explanation why it had been done in this way.
 profile id. Also it stores pydantic model and API url for requests.
 
 AmazonAdsStream is Http based class, it used for making request that could be
-accomlished by single http call (any but report streams).
+accomplished by single http call (any but report streams).
 
 SubProfilesStream is subclass for http streams to perform read_records from
 basic class for EACH profile from self._profiles list. Also provides support
-for Amazon Ads API pagintaion. This is base class for all the sync http streams
+for Amazon Ads API pagination. This is base class for all the sync http streams
 that used by source.
 
 ReportStream (It implemented on report_stream.py file) is subclass for async

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_products.py

@@ -2,8 +2,23 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from source_amazon_ads.schemas import Keywords, NegativeKeywords, ProductAd, ProductAdGroups, ProductCampaign, ProductTargeting
-from source_amazon_ads.streams.common import SubProfilesStream
+from abc import ABC
+from http import HTTPStatus
+from typing import Any, Iterable, List, Mapping, MutableMapping, Optional
+
+import requests as requests
+from airbyte_protocol.models import SyncMode
+from source_amazon_ads.schemas import (
+    Keywords,
+    NegativeKeywords,
+    ProductAd,
+    ProductAdGroupBidRecommendations,
+    ProductAdGroups,
+    ProductAdGroupSuggestedKeywords,
+    ProductCampaign,
+    ProductTargeting,
+)
+from source_amazon_ads.streams.common import AmazonAdsStream, SubProfilesStream
 
 
 class SponsoredProductCampaigns(SubProfilesStream):
@@ -43,6 +58,93 @@ def path(self, **kvargs) -> str:
         return "v2/sp/adGroups"
 
 
+class SponsoredProductAdGroupsWithProfileId(SponsoredProductAdGroups):
+    """Add profileId attr for each records in SponsoredProductAdGroups stream"""
+
+    def parse_response(self, *args, **kwargs) -> Iterable[Mapping]:
+        for record in super().parse_response(*args, **kwargs):
+            record["profileId"] = self._current_profile_id
+            yield record
+
+
+class SponsoredProductAdGroupWithSlicesABC(AmazonAdsStream, ABC):
+    """ABC Class for extraction of additional information for each known sp ad group"""
+
+    primary_key = "adGroupId"
+
+    def __init__(self, *args, **kwargs):
+        self.__args = args
+        self.__kwargs = kwargs
+        super().__init__(*args, **kwargs)
+
+    def request_headers(self, *args, **kvargs) -> MutableMapping[str, Any]:
+        headers = super().request_headers(*args, **kvargs)
+        headers["Amazon-Advertising-API-Scope"] = str(kvargs["stream_slice"]["profileId"])
+        return headers
+
+    def stream_slices(
+        self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None
+    ) -> Iterable[Optional[Mapping[str, Any]]]:
+        yield from SponsoredProductAdGroupsWithProfileId(*self.__args, **self.__kwargs).read_records(
+            sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=None, stream_state=stream_state
+        )
+
+    def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
+
+        resp = response.json()
+        if response.status_code == HTTPStatus.OK:
+            yield resp
+
+        if response.status_code == HTTPStatus.BAD_REQUEST:
+            # 400 error message for bids recommendation:
+            #   Bid recommendation for AD group in Manual Targeted Campaign is not supported.
+            # 400 error message for keywords recommendation:
+            #   Getting keyword recommendations for AD Group in Auto Targeted Campaign is not supported
+            self.logger.warning(
+                f"Skip current AdGroup because it does not support request {response.request.url} for "
+                f"{response.request.headers['Amazon-Advertising-API-Scope']} profile: {response.text}"
+            )
+
+        else:
+            response.raise_for_status()
+
+
+class SponsoredProductAdGroupBidRecommendations(SponsoredProductAdGroupWithSlicesABC):
+    """Docs:
+    Latest API:
+        https://advertising.amazon.com/API/docs/en-us/sponsored-display/3-0/openapi#/Bid%20Recommendations/getTargetBidRecommendations
+        POST /sd/targets/bid/recommendations
+        Note: does not work, always get "403 Forbidden"
+
+    V2 API:
+        https://advertising.amazon.com/API/docs/en-us/sponsored-products/2-0/openapi#/Bid%20recommendations/getAdGroupBidRecommendations
+        GET /v2/sp/adGroups/{adGroupId}/bidRecommendations
+    """
+
+    model = ProductAdGroupBidRecommendations
+
+    def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
+        return f"v2/sp/adGroups/{stream_slice['adGroupId']}/bidRecommendations"
+
+
+class SponsoredProductAdGroupSuggestedKeywords(SponsoredProductAdGroupWithSlicesABC):
+    """Docs:
+    Latest API:
+        https://advertising.amazon.com/API/docs/en-us/sponsored-products/3-0/openapi/prod#/Keyword%20Targets/getRankedKeywordRecommendation
+        POST /sp/targets/keywords/recommendations
+        Note: does not work, always get "403 Forbidden"
+
+    V2 API:
+        https://advertising.amazon.com/API/docs/en-us/sponsored-products/2-0/openapi#/Suggested%20keywords
+        GET /v2/sp/adGroups/{{adGroupId}}>/suggested/keywords
+    """
+
+    model = ProductAdGroupSuggestedKeywords
+
+    def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
+        return f"v2/sp/adGroups/{stream_slice['adGroupId']}/suggested/keywords"
+
+
 class SponsoredProductKeywords(SubProfilesStream):
     """
     This stream corresponds to Amazon Advertising API - Sponsored Products Keywords

airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py

@@ -90,7 +90,7 @@ def test_source_streams(config):
     setup_responses()
     source = SourceAmazonAds()
     streams = source.streams(config)
-    assert len(streams) == 26
+    assert len(streams) == 28
     actual_stream_names = {stream.name for stream in streams}
     expected_stream_names = set(
         [
@@ -99,6 +99,8 @@ def test_source_streams(config):
             "sponsored_display_campaigns",
             "sponsored_product_campaigns",
             "sponsored_product_ad_groups",
+            "sponsored_product_ad_group_suggested_keywords",
+            "sponsored_product_ad_group_bid_recommendations",
             "sponsored_product_keywords",
             "sponsored_product_negative_keywords",
             "sponsored_product_campaign_negative_keywords",