✨ Source Bing Ads: add foreign keys (#32131)

Co-authored-by: darynaishchenko <darynaishchenko@users.noreply.github.com>
airbytehq · Nov 6, 2023 · 1a2fd7b · 1a2fd7b
1 parent 1afab61
commit 1a2fd7b
Show file tree

Hide file tree

Showing 8 changed files with 67 additions and 5 deletions.
diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl
@@ -1,6 +1,6 @@
-{"stream":"ad_groups","data":{"AdRotation":null,"AudienceAdsBidAdjustment":null,"BiddingScheme":{"Type":"InheritFromParent","InheritedBidStrategyType":"EnhancedCpc"},"CpcBid":{"Amount":0.05},"EndDate":null,"FinalUrlSuffix":null,"ForwardCompatibilityMap":null,"Id":1357897480389129,"Language":null,"Name":"Data Integration Tool","Network":"OwnedAndOperatedAndSyndicatedSearch","PrivacyStatus":null,"Settings":null,"StartDate":{"Day":4,"Month":12,"Year":2020},"Status":"Paused","TrackingUrlTemplate":null,"UrlCustomParameters":null,"AdScheduleUseSearcherTimeZone":false,"AdGroupType":"SearchStandard","CpvBid":{"Amount":null},"CpmBid":{"Amount":null}},"emitted_at":1675189566179}
-{"stream":"ads","data":{"AdFormatPreference":"All","DevicePreference":0,"EditorialStatus":"ActiveLimited","FinalAppUrls":null,"FinalMobileUrls":{"string":["https://airbyte.io"]},"FinalUrlSuffix":null,"FinalUrls":{"string":["https://airbyte.io"]},"ForwardCompatibilityMap":null,"Id":84525295496190,"Status":"Active","TrackingUrlTemplate":null,"Type":"ResponsiveSearch","UrlCustomParameters":null,"Descriptions":{"AssetLink":[{"Asset":{"Id":10239221964468,"Name":null,"Type":"TextAsset","Text":"Open data integration for modern data teams"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239221964466,"Name":null,"Type":"TextAsset","Text":"Get your data pipelines running in minutes. With pre-built or custom connectors"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null}]},"Domain":"airbyte.io","Headlines":{"AssetLink":[{"Asset":{"Id":10239221964471,"Name":null,"Type":"TextAsset","Text":"Data Integration Tool"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239221964469,"Name":null,"Type":"TextAsset","Text":"1,000+ Members"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239221964467,"Name":null,"Type":"TextAsset","Text":"Data Management Software"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null}]},"Path1":null,"Path2":null},"emitted_at":1675189577521}
-{"stream":"campaigns","data":{"AudienceAdsBidAdjustment":0,"BiddingScheme":{"Type":"EnhancedCpc"},"BudgetType":"DailyBudgetStandard","DailyBudget":0.1,"ExperimentId":null,"FinalUrlSuffix":null,"ForwardCompatibilityMap":null,"Id":407519039,"MultimediaAdsBidAdjustment":40,"Name":"integration-test-campaign","Status":"Paused","SubType":null,"TimeZone":"Arizona","TrackingUrlTemplate":null,"UrlCustomParameters":null,"CampaignType":"Search","Settings":{"Setting":[{"Type":"TargetSetting","Details":{"TargetSettingDetail":[{"CriterionTypeGroup":"Audience","TargetAndBid":false}]}}]},"BudgetId":null,"Languages":{"string":["English"]},"AdScheduleUseSearcherTimeZone":false},"emitted_at":1675189585409}
+{"stream": "ad_groups", "data": {"AdRotation": null, "AudienceAdsBidAdjustment": null, "BiddingScheme": {"Type": "InheritFromParent", "InheritedBidStrategyType": "EnhancedCpc"}, "CpcBid": {"Amount": 0.05}, "EndDate": null, "FinalUrlSuffix": null, "ForwardCompatibilityMap": null, "Id": 1357897480389129, "Language": null, "Name": "Data Integration Tool", "Network": "OwnedAndOperatedAndSyndicatedSearch", "PrivacyStatus": null, "Settings": null, "StartDate": {"Day": 4, "Month": 12, "Year": 2020}, "Status": "Paused", "TrackingUrlTemplate": null, "UrlCustomParameters": null, "AdScheduleUseSearcherTimeZone": false, "AdGroupType": "SearchStandard", "CpvBid": {"Amount": null}, "CpmBid": {"Amount": null}, "CampaignId": 407519039, "AccountId": 180278106, "CustomerId": 251186883}, "emitted_at": 1698928139401}
+{"stream": "ads", "data": {"AdFormatPreference": "All", "DevicePreference": 0, "EditorialStatus": "ActiveLimited", "FinalAppUrls": null, "FinalMobileUrls": {"string": ["https://airbyte.io"]}, "FinalUrlSuffix": null, "FinalUrls": {"string": ["https://airbyte.io"]}, "ForwardCompatibilityMap": null, "Id": 84525295496190, "Status": "Active", "TrackingUrlTemplate": null, "Type": "ResponsiveSearch", "UrlCustomParameters": null, "Descriptions": {"AssetLink": [{"Asset": {"Id": 10239221964468, "Name": null, "Type": "TextAsset", "Text": "Open data integration for modern data teams"}, "AssetPerformanceLabel": "Learning", "EditorialStatus": "Active", "PinnedField": null}, {"Asset": {"Id": 10239221964466, "Name": null, "Type": "TextAsset", "Text": "Get your data pipelines running in minutes. With pre-built or custom connectors"}, "AssetPerformanceLabel": "Learning", "EditorialStatus": "Active", "PinnedField": null}]}, "Domain": "airbyte.io", "Headlines": {"AssetLink": [{"Asset": {"Id": 10239221964471, "Name": null, "Type": "TextAsset", "Text": "Data Integration Tool"}, "AssetPerformanceLabel": "Learning", "EditorialStatus": "Active", "PinnedField": null}, {"Asset": {"Id": 10239221964469, "Name": null, "Type": "TextAsset", "Text": "1,000+ Members"}, "AssetPerformanceLabel": "Learning", "EditorialStatus": "Active", "PinnedField": null}, {"Asset": {"Id": 10239221964467, "Name": null, "Type": "TextAsset", "Text": "Data Management Software"}, "AssetPerformanceLabel": "Learning", "EditorialStatus": "Active", "PinnedField": null}]}, "Path1": null, "Path2": null, "AdGroupId": 1352400325389092, "AccountId": 180278106, "CustomerId": 251186883}, "emitted_at": 1698927837499}
+{"stream": "campaigns", "data": {"AudienceAdsBidAdjustment": 0, "BiddingScheme": {"Type": "EnhancedCpc"}, "BudgetType": "DailyBudgetStandard", "DailyBudget": 0.1, "ExperimentId": null, "FinalUrlSuffix": null, "ForwardCompatibilityMap": null, "Id": 407519039, "MultimediaAdsBidAdjustment": 40, "Name": "integration-test-campaign", "Status": "Paused", "SubType": null, "TimeZone": "Arizona", "TrackingUrlTemplate": null, "UrlCustomParameters": null, "CampaignType": "Search", "Settings": {"Setting": [{"Type": "TargetSetting", "Details": {"TargetSettingDetail": [{"CriterionTypeGroup": "Audience", "TargetAndBid": false}]}}]}, "BudgetId": null, "Languages": {"string": ["English"]}, "AdScheduleUseSearcherTimeZone": false, "AccountId": 180278106, "CustomerId": 251186883}, "emitted_at": 1698928296177}
 {"stream":"accounts","data":{"BillToCustomerId":251186883,"CurrencyCode":"USD","AccountFinancialStatus":"ClearFinancialStatus","Id":180278106,"Language":"English","LastModifiedByUserId":0,"LastModifiedTime":"2023-08-11T03:26:10.277000","Name":"Daxtarity Inc.","Number":"F149GKV5","ParentCustomerId":251186883,"PaymentMethodId":138188746,"PaymentMethodType":"CreditCard","PrimaryUserId":138225488,"AccountLifeCycleStatus":"Active","TimeStamp":"AAAAAH1yyMo=","TimeZone":"Arizona","PauseReason":null,"ForwardCompatibilityMap":null,"LinkedAgencies":null,"SalesHouseCustomerId":null,"TaxInformation":null,"BackUpPaymentInstrumentId":null,"BillingThresholdAmount":null,"BusinessAddress":{"City":"San Francisco","CountryCode":"US","Id":149004358,"Line1":"350 29th avenue","Line2":null,"Line3":null,"Line4":null,"PostalCode":"94121","StateOrProvince":"CA","TimeStamp":null,"BusinessName":"Daxtarity Inc."},"AutoTagType":"Inactive","SoldToPaymentInstrumentId":null,"AccountMode":"Expert"},"emitted_at":1698149762546}
 {"stream":"account_performance_report_daily","data":{"AccountId":180278106,"TimePeriod":"2021-06-09","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Smartphone","Network":"Bing and Yahoo! search","DeliveredMatchType":"Broad","DeviceOS":"Android","TopVsOther":"Bing and Yahoo! search - Top","BidMatchType":"Broad","AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","PhoneImpressions":0,"PhoneCalls":0,"Clicks":0,"Ctr":0.0,"Spend":0.0,"Impressions":3,"CostPerConversion":null,"Ptr":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"LowQualityClicks":0,"LowQualityClicksPercent":null,"LowQualityImpressions":0,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1698149775393}
 {"stream":"account_performance_report_weekly","data":{"AccountId":180278106,"TimePeriod":"2021-06-06","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Bing and Yahoo! search","DeliveredMatchType":"Exact","DeviceOS":"Windows","TopVsOther":"Bing and Yahoo! search - Other","BidMatchType":"Broad","AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","PhoneImpressions":0,"PhoneCalls":0,"Clicks":0,"Ctr":null,"Spend":0.0,"Impressions":0,"CostPerConversion":null,"Ptr":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"LowQualityClicks":0,"LowQualityClicksPercent":null,"LowQualityImpressions":7,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1698149785273}

diff --git a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml
@@ -16,7 +16,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 47f25999-dd5e-4636-8c39-e7cea2453331
-  dockerImageTag: 1.8.0
+  dockerImageTag: 1.9.0
   dockerRepository: airbyte/source-bing-ads
   documentationUrl: https://docs.airbyte.com/integrations/sources/bing-ads
   githubIssueLabel: source-bing-ads

diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_groups.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_groups.json
@@ -2,6 +2,15 @@
   "$schema": "http://json-schema.org/draft-07/schema#",
   "type": "object",
   "properties": {
+    "CampaignId": {
+      "type": ["null", "integer"]
+    },
+    "AccountId": {
+      "type": ["null", "integer"]
+    },
+    "CustomerId": {
+      "type": ["null", "integer"]
+    },
     "AdRotation": {
       "type": ["null", "object"],
       "properties": {

diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ads.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ads.json
@@ -2,6 +2,15 @@
   "$schema": "http://json-schema.org/draft-07/schema#",
   "type": "object",
   "properties": {
+    "AdGroupId": {
+      "type": ["null", "integer"]
+    },
+    "AccountId": {
+      "type": ["null", "integer"]
+    },
+    "CustomerId": {
+      "type": ["null", "integer"]
+    },
     "AdFormatPreference": {
       "type": ["null", "string"]
     },

diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaigns.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaigns.json
@@ -2,6 +2,12 @@
   "$schema": "http://json-schema.org/draft-07/schema#",
   "type": "object",
   "properties": {
+    "AccountId": {
+      "type": ["null", "integer"]
+    },
+    "CustomerId": {
+      "type": ["null", "integer"]
+    },
     "AudienceAdsBidAdjustment": {
       "type": ["null", "number"]
     },

diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/streams.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/streams.py
@@ -60,6 +60,17 @@ def service_name(self) -> str:
         """
         pass
 
+    @property
+    def parent_key_to_foreign_key_map(self) -> MutableMapping[str, str]:
+        """
+        Specifies dict with field in record as kay and slice key as value to be inserted in record in transform method.
+        """
+        return {}
+
+    def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]:
+        foreign_keys = {key: stream_slice.get(value) for key, value in self.parent_key_to_foreign_key_map.items()}
+        return record | foreign_keys
+
     @property
     def _service(self) -> Union[ServiceClient, ReportingServiceManager]:
         return self.client.get_service(service_name=self.service_name)
@@ -120,7 +131,7 @@ def read_records(
             )
             response = self.send_request(params, customer_id=customer_id, account_id=account_id)
             for record in self.parse_response(response):
-                yield record
+                yield self.transform(record, stream_slice)
 
             next_page_token = self.next_page_token(response, current_page_token=next_page_token)
             if not next_page_token:
@@ -448,6 +459,11 @@ class Campaigns(BingAdsCampaignManagementStream):
     ]
     campaign_types: Iterable[str] = ["Audience", "DynamicSearchAds", "Search", "Shopping"]
 
+    parent_key_to_foreign_key_map = {
+        "AccountId": "account_id",
+        "CustomerId": "customer_id",
+    }
+
     def request_params(
         self,
         stream_slice: Mapping[str, Any] = None,
@@ -484,6 +500,8 @@ class AdGroups(BingAdsCampaignManagementStream):
     operation_name: str = "GetAdGroupsByCampaignId"
     additional_fields: str = "AdGroupType AdScheduleUseSearcherTimeZone CpmBid CpvBid MultimediaAdsBidAdjustment"
 
+    parent_key_to_foreign_key_map = {"CampaignId": "campaign_id", "AccountId": "account_id", "CustomerId": "customer_id"}
+
     def request_params(
         self,
         stream_slice: Mapping[str, Any] = None,
@@ -527,6 +545,8 @@ class Ads(BingAdsCampaignManagementStream):
         "ResponsiveSearch",
     ]
 
+    parent_key_to_foreign_key_map = {"AdGroupId": "ad_group_id", "AccountId": "account_id", "CustomerId": "customer_id"}
+
     def request_params(
         self,
         stream_slice: Mapping[str, Any] = None,

diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py
@@ -230,3 +230,20 @@ def test_bulk_stream_read_with_chunks(mocked_client, config):
         "Tracking Template": None,
         "Type": "App Install Ad",
     }
+
+
+@patch.object(source_bing_ads.source, "Client")
+def test_transform(mocked_client, config):
+    record = {"AdFormatPreference": "All", "DevicePreference": 0, "EditorialStatus": "ActiveLimited", "FinalAppUrls": None}
+    transformed_record = Ads(mocked_client, config).transform(
+        record=record, stream_slice={"ad_group_id": 90909090, "account_id": 909090, "customer_id": 9090909}
+    )
+    assert transformed_record == {
+        "AccountId": 909090,
+        "AdFormatPreference": "All",
+        "AdGroupId": 90909090,
+        "CustomerId": 9090909,
+        "DevicePreference": 0,
+        "EditorialStatus": "ActiveLimited",
+        "FinalAppUrls": None,
+    }
diff --git a/docs/integrations/sources/bing-ads.md b/docs/integrations/sources/bing-ads.md
@@ -150,6 +150,7 @@ The Bing Ads API limits the number of requests for all Microsoft Advertising cli
 
 | Version | Date       | Pull Request                                                                                                                     | Subject                                                                                                                                      |
 |:--------|:-----------|:---------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------|
+| 1.9.0   | 2023-11-03 | [32131](https://github.com/airbytehq/airbyte/pull/32131)                                                                         | Add  "CampaignId", "AccountId", "CustomerId" fields to Ad Groups, Ads and Campaigns streams.                                                 |
 | 1.8.0   | 2023-11-02 | [32059](https://github.com/airbytehq/airbyte/pull/32059)                                                                         | Add new streams `CampaignImpressionPerformanceReport` (daily, hourly, weekly, monthly)                                                       |
 | 1.7.1   | 2023-11-02 | [32088](https://github.com/airbytehq/airbyte/pull/32088)                                                                         | Raise config error when user does not have accounts                                                                                          |
 | 1.7.0   | 2023-11-01 | [32027](https://github.com/airbytehq/airbyte/pull/32027)                                                                         | Add new streams `AdGroupImpressionPerformanceReport`                                                                                         |