✨ Source Bing Ads: add foreign keys

airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl

@@ -1,6 +1,6 @@
-{"stream":"ad_groups","data":{"AdRotation":null,"AudienceAdsBidAdjustment":null,"BiddingScheme":{"Type":"InheritFromParent","InheritedBidStrategyType":"EnhancedCpc"},"CpcBid":{"Amount":0.05},"EndDate":null,"FinalUrlSuffix":null,"ForwardCompatibilityMap":null,"Id":1357897480389129,"Language":null,"Name":"Data Integration Tool","Network":"OwnedAndOperatedAndSyndicatedSearch","PrivacyStatus":null,"Settings":null,"StartDate":{"Day":4,"Month":12,"Year":2020},"Status":"Paused","TrackingUrlTemplate":null,"UrlCustomParameters":null,"AdScheduleUseSearcherTimeZone":false,"AdGroupType":"SearchStandard","CpvBid":{"Amount":null},"CpmBid":{"Amount":null}},"emitted_at":1675189566179}
-{"stream":"ads","data":{"AdFormatPreference":"All","DevicePreference":0,"EditorialStatus":"ActiveLimited","FinalAppUrls":null,"FinalMobileUrls":{"string":["https://airbyte.io"]},"FinalUrlSuffix":null,"FinalUrls":{"string":["https://airbyte.io"]},"ForwardCompatibilityMap":null,"Id":84525295496190,"Status":"Active","TrackingUrlTemplate":null,"Type":"ResponsiveSearch","UrlCustomParameters":null,"Descriptions":{"AssetLink":[{"Asset":{"Id":10239221964468,"Name":null,"Type":"TextAsset","Text":"Open data integration for modern data teams"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239221964466,"Name":null,"Type":"TextAsset","Text":"Get your data pipelines running in minutes. With pre-built or custom connectors"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null}]},"Domain":"airbyte.io","Headlines":{"AssetLink":[{"Asset":{"Id":10239221964471,"Name":null,"Type":"TextAsset","Text":"Data Integration Tool"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239221964469,"Name":null,"Type":"TextAsset","Text":"1,000+ Members"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239221964467,"Name":null,"Type":"TextAsset","Text":"Data Management Software"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null}]},"Path1":null,"Path2":null},"emitted_at":1675189577521}
-{"stream":"campaigns","data":{"AudienceAdsBidAdjustment":0,"BiddingScheme":{"Type":"EnhancedCpc"},"BudgetType":"DailyBudgetStandard","DailyBudget":0.1,"ExperimentId":null,"FinalUrlSuffix":null,"ForwardCompatibilityMap":null,"Id":407519039,"MultimediaAdsBidAdjustment":40,"Name":"integration-test-campaign","Status":"Paused","SubType":null,"TimeZone":"Arizona","TrackingUrlTemplate":null,"UrlCustomParameters":null,"CampaignType":"Search","Settings":{"Setting":[{"Type":"TargetSetting","Details":{"TargetSettingDetail":[{"CriterionTypeGroup":"Audience","TargetAndBid":false}]}}]},"BudgetId":null,"Languages":{"string":["English"]},"AdScheduleUseSearcherTimeZone":false},"emitted_at":1675189585409}
+{"stream": "ad_groups", "data": {"AdRotation": null, "AudienceAdsBidAdjustment": null, "BiddingScheme": {"Type": "InheritFromParent", "InheritedBidStrategyType": "EnhancedCpc"}, "CpcBid": {"Amount": 0.05}, "EndDate": null, "FinalUrlSuffix": null, "ForwardCompatibilityMap": null, "Id": 1357897480389129, "Language": null, "Name": "Data Integration Tool", "Network": "OwnedAndOperatedAndSyndicatedSearch", "PrivacyStatus": null, "Settings": null, "StartDate": {"Day": 4, "Month": 12, "Year": 2020}, "Status": "Paused", "TrackingUrlTemplate": null, "UrlCustomParameters": null, "AdScheduleUseSearcherTimeZone": false, "AdGroupType": "SearchStandard", "CpvBid": {"Amount": null}, "CpmBid": {"Amount": null}, "CampaignId": 407519039, "AccountId": 180278106, "CustomerId": 251186883}, "emitted_at": 1698928139401}
+{"stream": "ads", "data": {"AdFormatPreference": "All", "DevicePreference": 0, "EditorialStatus": "ActiveLimited", "FinalAppUrls": null, "FinalMobileUrls": {"string": ["https://airbyte.io"]}, "FinalUrlSuffix": null, "FinalUrls": {"string": ["https://airbyte.io"]}, "ForwardCompatibilityMap": null, "Id": 84525295496190, "Status": "Active", "TrackingUrlTemplate": null, "Type": "ResponsiveSearch", "UrlCustomParameters": null, "Descriptions": {"AssetLink": [{"Asset": {"Id": 10239221964468, "Name": null, "Type": "TextAsset", "Text": "Open data integration for modern data teams"}, "AssetPerformanceLabel": "Learning", "EditorialStatus": "Active", "PinnedField": null}, {"Asset": {"Id": 10239221964466, "Name": null, "Type": "TextAsset", "Text": "Get your data pipelines running in minutes. With pre-built or custom connectors"}, "AssetPerformanceLabel": "Learning", "EditorialStatus": "Active", "PinnedField": null}]}, "Domain": "airbyte.io", "Headlines": {"AssetLink": [{"Asset": {"Id": 10239221964471, "Name": null, "Type": "TextAsset", "Text": "Data Integration Tool"}, "AssetPerformanceLabel": "Learning", "EditorialStatus": "Active", "PinnedField": null}, {"Asset": {"Id": 10239221964469, "Name": null, "Type": "TextAsset", "Text": "1,000+ Members"}, "AssetPerformanceLabel": "Learning", "EditorialStatus": "Active", "PinnedField": null}, {"Asset": {"Id": 10239221964467, "Name": null, "Type": "TextAsset", "Text": "Data Management Software"}, "AssetPerformanceLabel": "Learning", "EditorialStatus": "Active", "PinnedField": null}]}, "Path1": null, "Path2": null, "AdGroupId": 1352400325389092, "AccountId": 180278106, "CustomerId": 251186883}, "emitted_at": 1698927837499}
+{"stream": "campaigns", "data": {"AudienceAdsBidAdjustment": 0, "BiddingScheme": {"Type": "EnhancedCpc"}, "BudgetType": "DailyBudgetStandard", "DailyBudget": 0.1, "ExperimentId": null, "FinalUrlSuffix": null, "ForwardCompatibilityMap": null, "Id": 407519039, "MultimediaAdsBidAdjustment": 40, "Name": "integration-test-campaign", "Status": "Paused", "SubType": null, "TimeZone": "Arizona", "TrackingUrlTemplate": null, "UrlCustomParameters": null, "CampaignType": "Search", "Settings": {"Setting": [{"Type": "TargetSetting", "Details": {"TargetSettingDetail": [{"CriterionTypeGroup": "Audience", "TargetAndBid": false}]}}]}, "BudgetId": null, "Languages": {"string": ["English"]}, "AdScheduleUseSearcherTimeZone": false, "AccountId": 180278106, "CustomerId": 251186883}, "emitted_at": 1698928296177}
 {"stream":"accounts","data":{"BillToCustomerId":251186883,"CurrencyCode":"USD","AccountFinancialStatus":"ClearFinancialStatus","Id":180278106,"Language":"English","LastModifiedByUserId":0,"LastModifiedTime":"2023-08-11T03:26:10.277000","Name":"Daxtarity Inc.","Number":"F149GKV5","ParentCustomerId":251186883,"PaymentMethodId":138188746,"PaymentMethodType":"CreditCard","PrimaryUserId":138225488,"AccountLifeCycleStatus":"Active","TimeStamp":"AAAAAH1yyMo=","TimeZone":"Arizona","PauseReason":null,"ForwardCompatibilityMap":null,"LinkedAgencies":null,"SalesHouseCustomerId":null,"TaxInformation":null,"BackUpPaymentInstrumentId":null,"BillingThresholdAmount":null,"BusinessAddress":{"City":"San Francisco","CountryCode":"US","Id":149004358,"Line1":"350 29th avenue","Line2":null,"Line3":null,"Line4":null,"PostalCode":"94121","StateOrProvince":"CA","TimeStamp":null,"BusinessName":"Daxtarity Inc."},"AutoTagType":"Inactive","SoldToPaymentInstrumentId":null,"AccountMode":"Expert"},"emitted_at":1698149762546}
 {"stream":"account_performance_report_daily","data":{"AccountId":180278106,"TimePeriod":"2021-06-09","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Smartphone","Network":"Bing and Yahoo! search","DeliveredMatchType":"Broad","DeviceOS":"Android","TopVsOther":"Bing and Yahoo! search - Top","BidMatchType":"Broad","AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","PhoneImpressions":0,"PhoneCalls":0,"Clicks":0,"Ctr":0.0,"Spend":0.0,"Impressions":3,"CostPerConversion":null,"Ptr":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"LowQualityClicks":0,"LowQualityClicksPercent":null,"LowQualityImpressions":0,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1698149775393}
 {"stream":"account_performance_report_weekly","data":{"AccountId":180278106,"TimePeriod":"2021-06-06","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Bing and Yahoo! search","DeliveredMatchType":"Exact","DeviceOS":"Windows","TopVsOther":"Bing and Yahoo! search - Other","BidMatchType":"Broad","AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","PhoneImpressions":0,"PhoneCalls":0,"Clicks":0,"Ctr":null,"Spend":0.0,"Impressions":0,"CostPerConversion":null,"Ptr":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"LowQualityClicks":0,"LowQualityClicksPercent":null,"LowQualityImpressions":7,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1698149785273}

airbyte-integrations/connectors/source-bing-ads/metadata.yaml

@@ -16,7 +16,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 47f25999-dd5e-4636-8c39-e7cea2453331
-  dockerImageTag: 1.7.1
+  dockerImageTag: 1.8.0
   dockerRepository: airbyte/source-bing-ads
   documentationUrl: https://docs.airbyte.com/integrations/sources/bing-ads
   githubIssueLabel: source-bing-ads

airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_groups.json

@@ -2,6 +2,15 @@
   "$schema": "http://json-schema.org/draft-07/schema#",
   "type": "object",
   "properties": {
+    "CampaignId": {
+      "type" : ["null", "integer"]
+    },
+    "AccountId": {
+      "type" : ["null", "integer"]
+    },
+    "CustomerId": {
+      "type" : ["null", "integer"]
+    },
     "AdRotation": {
       "type": ["null", "object"],
       "properties": {

airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ads.json

@@ -2,6 +2,15 @@
   "$schema": "http://json-schema.org/draft-07/schema#",
   "type": "object",
   "properties": {
+    "AdGroupId": {
+      "type" : ["null", "integer"]
+    },
+    "AccountId": {
+      "type" : ["null", "integer"]
+    },
+    "CustomerId": {
+      "type" : ["null", "integer"]
+    },
     "AdFormatPreference": {
       "type": ["null", "string"]
     },

airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaigns.json

@@ -2,6 +2,12 @@
   "$schema": "http://json-schema.org/draft-07/schema#",
   "type": "object",
   "properties": {
+    "AccountId": {
+      "type" : ["null", "integer"]
+    },
+    "CustomerId": {
+      "type" : ["null", "integer"]
+    },
     "AudienceAdsBidAdjustment": {
       "type": ["null", "number"]
     },

airbyte-integrations/connectors/source-bing-ads/source_bing_ads/streams.py

@@ -76,6 +76,12 @@ def additional_fields(self) -> Optional[str]:
         """
         pass
 
+    def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]:
+        """
+        Specifies additional transformation for the record. Can be overwritten if stream requires to transform a record.
+        """
+        return record
+
     @property
     def _service(self) -> Union[ServiceClient, ReportingServiceManager]:
         return self.client.get_service(service_name=self.service_name)
@@ -142,7 +148,7 @@ def read_records(
             )
             response = self.send_request(params, customer_id=customer_id, account_id=account_id)
             for record in self.parse_response(response):
-                yield record
+                yield self.transform(record, stream_slice)
 
             next_page_token = self.next_page_token(response, current_page_token=next_page_token)
             if not next_page_token:
@@ -429,6 +435,11 @@ def stream_slices(
 
         yield from []
 
+    def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]:
+        record["AccountId"] = stream_slice.get("account_id")
+        record["CustomerId"] = stream_slice.get("customer_id")
+        return record
+
 
 class AdGroups(BingAdsStream):
     """
@@ -466,6 +477,12 @@ def stream_slices(
 
         yield from []
 
+    def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]:
+        record["CampaignId"] = stream_slice.get("campaign_id")
+        record["AccountId"] = stream_slice.get("account_id")
+        record["CustomerId"] = stream_slice.get("customer_id")
+        return record
+
 
 class Ads(BingAdsStream):
     """
@@ -511,6 +528,12 @@ def stream_slices(
                 yield {"ad_group_id": ad_group["Id"], "account_id": slice["account_id"], "customer_id": slice["customer_id"]}
         yield from []
 
+    def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]:
+        record["AdGroupId"] = stream_slice.get("ad_group_id")
+        record["AccountId"] = stream_slice.get("account_id")
+        record["CustomerId"] = stream_slice.get("customer_id")
+        return record
+
 
 class BudgetSummaryReport(ReportsMixin, BingAdsStream):
     data_field: str = ""

docs/integrations/sources/bing-ads.md

@@ -146,6 +146,7 @@ The Bing Ads API limits the number of requests for all Microsoft Advertising cli
 
 | Version | Date       | Pull Request                                                                                                                     | Subject                                                                                                                                      |
 |:--------|:-----------|:---------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------|
+| 1.8.0   | 2023-11-03 | [32131](https://github.com/airbytehq/airbyte/pull/32131)                                                                         | Add  "CampaignId", "AccountId", "CustomerId" fields to Ad Groups, Ads and Campaigns streams.                                                 |
 | 1.7.1   | 2023-11-02 | [32088](https://github.com/airbytehq/airbyte/pull/32088)                                                                         | Raise config error when user does not have accounts                                                                                          |
 | 1.7.0   | 2023-11-01 | [32027](https://github.com/airbytehq/airbyte/pull/32027)                                                                         | Add new streams `AdGroupImpressionPerformanceReport`                                                                                         |
 | 1.6.0   | 2023-10-31 | [32008](https://github.com/airbytehq/airbyte/pull/32008)                                                                         | Add new streams `Keywords`                                                                                                                   |