✨Source Amazon Ads: Add filter for Marketplace ID (airbytehq#29233)

Co-authored-by: marcosmarxm <marcosmarxm@gmail.com>
Co-authored-by: Serhii Lazebnyi <53845333+lazebnyi@users.noreply.github.com>
Co-authored-by: Marcos Marx <marcosmarxm@users.noreply.github.com>

airbyte-integrations/connectors/source-amazon-ads/Dockerfile

@@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
 
-LABEL io.airbyte.version=3.1.1
+LABEL io.airbyte.version=3.1.2
 LABEL io.airbyte.name=airbyte/source-amazon-ads

airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json

@@ -47,13 +47,22 @@
       },
       "profiles": {
         "title": "Profile IDs",
-        "description": "Profile IDs you want to fetch data for. See <a href=\"https://advertising.amazon.com/API/docs/en-us/concepts/authorization/profiles\">docs</a> for more details.",
+        "description": "Profile IDs you want to fetch data for. See <a href=\"https://advertising.amazon.com/API/docs/en-us/concepts/authorization/profiles\">docs</a> for more details. Note: If Marketplace IDs are also selected, profiles will be selected if they match the Profile ID OR the Marketplace ID.",
         "order": 6,
         "type": "array",
         "items": {
           "type": "integer"
         }
       },
+      "marketplace_ids": {
+        "title": "Marketplace IDs",
+        "description": "Marketplace IDs you want to fetch data for. Note: If Profile IDs are also selected, profiles will be selected if they match the Profile ID OR the Marketplace ID.",
+        "order": 7,
+        "type": "array",
+        "items": {
+          "type": "string"
+        }
+      },
       "state_filter": {
         "title": "State Filter",
         "description": "Reflects the state of the Display, Product, and Brand Campaign streams as enabled, paused, or archived. If you do not populate this field, it will be ignored completely.",
@@ -63,14 +72,14 @@
         },
         "type": "array",
         "uniqueItems": true,
-        "order": 7
+        "order": 8
       },
       "look_back_window": {
         "title": "Look Back Window",
         "description": "The amount of days to go back in time to get the updated data from Amazon Ads",
         "default": 3,
         "examples": [3, 10],
-        "order": 8,
+        "order": 9,
         "type": "integer"
       },
       "report_record_types": {
@@ -91,7 +100,7 @@
         },
         "type": "array",
         "uniqueItems": true,
-        "order": 9
+        "order": 10
       }
     },
     "required": ["client_id", "client_secret", "refresh_token"],

airbyte-integrations/connectors/source-amazon-ads/metadata.yaml

@@ -8,7 +8,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246
-  dockerImageTag: 3.1.1
+  dockerImageTag: 3.1.2
   dockerRepository: airbyte/source-amazon-ads
   githubIssueLabel: source-amazon-ads
   icon: amazonads.svg

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py

@@ -79,7 +79,10 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) ->
         # in response body.
         # It doesnt support pagination so there is no sense of reading single
         # record, it would fetch all the data anyway.
-        Profiles(config, authenticator=self._make_authenticator(config)).get_all_profiles()
+        profiles_list = Profiles(config, authenticator=self._make_authenticator(config)).get_all_profiles()
+        filtered_profiles = self._choose_profiles(config, profiles_list)
+        if not filtered_profiles:
+            return False, "No profiles found after filtering by Profile ID and Marketplace ID"
         return True, None
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
@@ -139,7 +142,13 @@ def _make_authenticator(config: Mapping[str, Any]):
         )
 
     @staticmethod
-    def _choose_profiles(config: Mapping[str, Any], profiles: List[Profile]):
-        if not config.get("profiles"):
-            return profiles
-        return list(filter(lambda profile: profile.profileId in config["profiles"], profiles))
+    def _choose_profiles(config: Mapping[str, Any], available_profiles: List[Profile]):
+        requested_profiles = config.get("profiles", [])
+        requested_marketplace_ids = config.get("marketplace_ids", [])
+        if requested_profiles or requested_marketplace_ids:
+            return [
+                profile
+                for profile in available_profiles
+                if profile.profileId in requested_profiles or profile.accountInfo.marketplaceStringId in requested_marketplace_ids
+            ]
+        return available_profiles

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml

@@ -58,13 +58,18 @@ connectionSpecification:
       type: string
     profiles:
       title: Profile IDs
-      description:
-        Profile IDs you want to fetch data for. See <a href="https://advertising.amazon.com/API/docs/en-us/concepts/authorization/profiles">docs</a>
-        for more details.
+      description: 'Profile IDs you want to fetch data for. See <a href="https://advertising.amazon.com/API/docs/en-us/concepts/authorization/profiles">docs</a> for more details. Note: If Marketplace IDs are also selected, profiles will be selected if they match the Profile ID OR the Marketplace ID.'
       order: 6
       type: array
       items:
         type: integer
+    marketplace_ids:
+      title: Marketplace IDs
+      description: "Marketplace IDs you want to fetch data for. Note: If Profile IDs are also selected, profiles will be selected if they match the Profile ID OR the Marketplace ID."
+      order: 7
+      type: array
+      items:
+        type: string
     state_filter:
       title: State Filter
       description: Reflects the state of the Display, Product, and Brand Campaign streams as enabled, paused, or archived. If you do not populate this field, it will be ignored completely.
@@ -76,7 +81,7 @@ connectionSpecification:
           - archived
       type: array
       uniqueItems: true
-      order: 7
+      order: 8
     look_back_window:
       title: "Look Back Window"
       description: "The amount of days to go back in time to get the updated data from Amazon Ads"
@@ -85,7 +90,7 @@ connectionSpecification:
         - 10
       type: "integer"
       default: 3
-      order: 8
+      order: 9
     report_record_types:
       title: Report Record Types
       description:
@@ -107,7 +112,7 @@ connectionSpecification:
           - targets
       type: array
       uniqueItems: true
-      order: 9
+      order: 10
   required:
     - client_id
     - client_secret

airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py

@@ -2,10 +2,12 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
+
 import responses
 from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConnectorSpecification, Status, Type
 from jsonschema import Draft4Validator
 from source_amazon_ads import SourceAmazonAds
+from source_amazon_ads.schemas import Profile
 
 from .utils import command_check, url_strip_query
 
@@ -19,7 +21,7 @@ def setup_responses():
     responses.add(
         responses.GET,
         "https://advertising-api.amazon.com/v2/profiles",
-        json=[],
+        json=[{"profileId": 111, "timezone": "gtm", "accountInfo": {"marketplaceStringId": "mkt_id_1", "id": "111", "type": "vendor"}}],
     )
 
 
@@ -119,3 +121,68 @@ def test_source_streams(config):
         ]
     )
     assert not expected_stream_names - actual_stream_names
+
+
+def test_filter_profiles_exist():
+    source = SourceAmazonAds()
+    mock_objs = [
+        {
+            "profileId": 111,
+            "timezone": "gtm",
+            "accountInfo": {
+                "marketplaceStringId": "mkt_id_1",
+                "id": "111",
+                "type": "vendor"
+            }
+        },
+        {
+            "profileId": 222,
+            "timezone": "gtm",
+            "accountInfo": {
+                "marketplaceStringId": "mkt_id_2",
+                "id": "222",
+                "type": "vendor"
+            }
+        },
+        {
+            "profileId": 333,
+            "timezone": "gtm",
+            "accountInfo": {
+                "marketplaceStringId": "mkt_id_3",
+                "id": "333",
+                "type": "vendor"
+            }
+        }
+    ]
+
+    mock_profiles = [Profile.parse_obj(profile) for profile in mock_objs]
+
+    filtered_profiles = source._choose_profiles({}, mock_profiles)
+    assert len(filtered_profiles) == 3
+
+    filtered_profiles = source._choose_profiles({"profiles": [111]}, mock_profiles)
+    assert len(filtered_profiles) == 1
+    assert filtered_profiles[0].profileId == 111
+
+    filtered_profiles = source._choose_profiles({"profiles": [111, 333]}, mock_profiles)
+    assert len(filtered_profiles) == 2
+
+    filtered_profiles = source._choose_profiles({"profiles": [444]}, mock_profiles)
+    assert len(filtered_profiles) == 0
+
+    filtered_profiles = source._choose_profiles({"marketplace_ids": ["mkt_id_4"]}, mock_profiles)
+    assert len(filtered_profiles) == 0
+
+    filtered_profiles = source._choose_profiles({"marketplace_ids": ["mkt_id_1"]}, mock_profiles)
+    assert len(filtered_profiles) == 1
+    assert filtered_profiles[0].accountInfo.marketplaceStringId == "mkt_id_1"
+
+    filtered_profiles = source._choose_profiles({"marketplace_ids": ["mkt_id_1", "mkt_id_3"]}, mock_profiles)
+    assert len(filtered_profiles) == 2
+
+    filtered_profiles = source._choose_profiles({"profiles": [111], "marketplace_ids": ["mkt_id_2"]}, mock_profiles)
+    assert len(filtered_profiles) == 2
+
+    filtered_profiles = source._choose_profiles({"profiles": [111], "marketplace_ids": ["mkt_id_1"]}, mock_profiles)
+    assert len(filtered_profiles) == 1
+    assert filtered_profiles[0].profileId == 111

docs/integrations/sources/amazon-ads.md

@@ -9,6 +9,7 @@ This page contains the setup guide and reference information for the Amazon Ads
 * Region
 * Start Date (Optional)
 * Profile IDs (Optional)
+* Marketplace IDs (Optional)
 
 ## Setup guide
 ### Step 1: Set up Amazon Ads
@@ -32,7 +33,8 @@ To use the [Amazon Ads API](https://advertising.amazon.com/API/docs/en-us), you
 6. Select **Region** to pull data from **North America (NA)**, **Europe (EU)**, **Far East (FE)**. See [docs](https://advertising.amazon.com/API/docs/en-us/info/api-overview#api-endpoints) for more details.
 7. **Start Date (Optional)** is used for generating reports starting from the specified start date. Should be in YYYY-MM-DD format and not more than 60 days in the past. If not specified today's date is used. The date is treated in the timezone of the processed profile.
 8. **Profile IDs (Optional)** you want to fetch data for. See [docs](https://advertising.amazon.com/API/docs/en-us/concepts/authorization/profiles) for more details.
-9. Click `Set up source`.
+9. **Marketplace IDs (Optional)** you want to fetch data for. _Note: If Profile IDs are also selected, profiles will be selected if they match the Profile ID **OR** the Marketplace ID._
+10. Click `Set up source`.
 <!-- /env:cloud -->
 
 <!-- env:oss -->
@@ -41,6 +43,10 @@ To use the [Amazon Ads API](https://advertising.amazon.com/API/docs/en-us), you
 1. **Client ID** of your Amazon Ads developer application. See [onboarding process](https://advertising.amazon.com/API/docs/en-us/setting-up/overview) for more details.
 2. **Client Secret** of your Amazon Ads developer application. See [onboarding process](https://advertising.amazon.com/API/docs/en-us/setting-up/overview) for more details.
 3. **Refresh Token**. See [onboarding process](https://advertising.amazon.com/API/docs/en-us/setting-up/overview) for more details.
+4. Select **Region** to pull data from **North America (NA)**, **Europe (EU)**, **Far East (FE)**. See [docs](https://advertising.amazon.com/API/docs/en-us/info/api-overview#api-endpoints) for more details.
+5. **Start Date (Optional)** is used for generating reports starting from the specified start date. Should be in YYYY-MM-DD format and not more than 60 days in the past. If not specified today's date is used. The date is treated in the timezone of the processed profile.
+6. **Profile IDs (Optional)** you want to fetch data for. See [docs](https://advertising.amazon.com/API/docs/en-us/concepts/authorization/profiles) for more details.
+7. **Marketplace IDs (Optional)** you want to fetch data for. _Note: If Profile IDs are also selected, profiles will be selected if they match the Profile ID **OR** the Marketplace ID._
 <!-- /env:oss -->
 
 ## Supported sync modes
@@ -103,6 +109,7 @@ Information about expected report generation waiting time you may find [here](ht
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                         |
 |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
+| 3.1.2   | 2023-08-16 | [29233](https://github.com/airbytehq/airbyte/pull/29233) | Add filter for Marketplace IDs                                                                                  |
 | 3.1.1   | 2023-08-28 | [29900](https://github.com/airbytehq/airbyte/pull/29900) | Add 404 handling for no assotiated with bid ad groups                                                           |
 | 3.1.0   | 2023-08-08 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Add `T00030` tactic support for `sponsored_display_report_stream`                                               |
 | 3.0.0   | 2023-07-24 | [27868](https://github.com/airbytehq/airbyte/pull/27868) | Fix attribution report stream schemas                                                                           |