✨ Source Bing Ads: add new stream SearchQueryPerformanceReport

airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml

@@ -34,12 +34,14 @@ acceptance_tests:
             bypass_reason: "Hourly reports are disabled, because sync is too long"
           - name: age_gender_audience_report_hourly
             bypass_reason: "Empty report; hourly data fetched is limited to 180 days"
-        timeout_seconds: 900
+          - name: search_query_performance_report_hourly
+            bypass_reason: "Empty report; hourly data fetched is limited to 180 days"
+        timeout_seconds: 1200
   full_refresh:
     tests:
       - config_path: secrets/config.json
         configured_catalog_path: integration_tests/configured_catalog.json
-        timeout_seconds: 900
+        timeout_seconds: 1200
   incremental:
     tests:
     bypass_reason: "SAT doesn't support complex nested states used in incremental report streams"

airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog.json

@@ -305,6 +305,46 @@
       "sync_mode": "incremental",
       "cursor_field": ["TimePeriod"],
       "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "search_query_performance_report_hourly",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental", "full_refresh"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["TimePeriod"],
+      "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "search_query_performance_report_daily",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental", "full_refresh"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["TimePeriod"],
+      "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "search_query_performance_report_weekly",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental", "full_refresh"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["TimePeriod"],
+      "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "search_query_performance_report_monthly",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental", "full_refresh"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["TimePeriod"],
+      "destination_sync_mode": "append"
     }
   ]
 }

airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl

@@ -25,3 +25,6 @@
 {"stream":"age_gender_audience_report_weekly","data":{"AccountId":180278106,"AgeGroup":"Unknown","Gender":"Female","TimePeriod":"2021-07-18","AllConversions":0,"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","CampaignName":"Test 2","CampaignId":413444833,"AdGroupName":"Airbyte","AdGroupId":1352400325389092,"AdDistribution":"Search","Impressions":1,"Clicks":0,"Conversions":0.0,"Spend":0.0,"Revenue":0.0,"ExtendedCost":0.0,"Assists":0,"Language":"English","AccountStatus":"Active","CampaignStatus":"Paused","AdGroupStatus":"Active","BaseCampaignId":"413444833","AllRevenue":0.0,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":0.00,"TopImpressionRatePercent":0.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0},"emitted_at":1698144728984}
 {"stream":"age_gender_audience_report_weekly","data":{"AccountId":180278106,"AgeGroup":"18-24","Gender":"Unknown","TimePeriod":"2021-06-06","AllConversions":0,"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","CampaignName":"Test 2","CampaignId":413444833,"AdGroupName":"Airbyte","AdGroupId":1352400325389092,"AdDistribution":"Search","Impressions":5,"Clicks":0,"Conversions":0.0,"Spend":0.0,"Revenue":0.0,"ExtendedCost":0.0,"Assists":0,"Language":"English","AccountStatus":"Active","CampaignStatus":"Paused","AdGroupStatus":"Active","BaseCampaignId":"413444833","AllRevenue":0.0,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":20.00,"TopImpressionRatePercent":80.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0},"emitted_at":1698144728986}
 {"stream":"age_gender_audience_report_monthly","data":{"AccountId":180278106,"AgeGroup":"50-64","Gender":"Female","TimePeriod":"2021-07-01","AllConversions":0,"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","CampaignName":"Test 2","CampaignId":413444833,"AdGroupName":"Airbyte","AdGroupId":1352400325389092,"AdDistribution":"Search","Impressions":3,"Clicks":0,"Conversions":0.0,"Spend":0.0,"Revenue":0.0,"ExtendedCost":0.0,"Assists":0,"Language":"English","AccountStatus":"Active","CampaignStatus":"Paused","AdGroupStatus":"Active","BaseCampaignId":"413444833","AllRevenue":0.0,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":33.33,"TopImpressionRatePercent":33.33,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0},"emitted_at":1698144746216}
+{"stream":"search_query_performance_report_daily","data":{"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","AccountId":180278106,"TimePeriod":"2021-07-27","CampaignName":"Test 2","CampaignId":413444833,"AdGroupName":"Airbyte","AdGroupId":1352400325389092,"AdId":84525295496190,"AdType":"Responsive search ad","DestinationUrl":null,"BidMatchType":"Broad","DeliveredMatchType":"Exact (close variant)","CampaignStatus":"Paused","AdStatus":"Active","Impressions":112,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"SearchQuery":"data integration tool","Keyword":"data integration tools","AdGroupCriterionId":null,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"Language":"English","KeywordId":84525593559629,"Network":"Syndicated search partners","TopVsOther":"Syndicated search partners - Top","DeviceType":"Smartphone","DeviceOS":"Android","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"AccountStatus":"Active","AdGroupStatus":"Active","KeywordStatus":"Active","CampaignType":"Search & content","CustomerId":251186883,"CustomerName":"Daxtarity Inc.","AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":17.86,"TopImpressionRatePercent":100.0,"AverageCpm":0.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0},"emitted_at":1698172081512}
+{"stream":"search_query_performance_report_weekly","data":{"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","AccountId":180278106,"TimePeriod":"2021-07-11","CampaignName":"Test 2","CampaignId":413444833,"AdGroupName":"Airbyte","AdGroupId":1352400325389092,"AdId":84525295496190,"AdType":"Responsive search ad","DestinationUrl":null,"BidMatchType":"Broad","DeliveredMatchType":"Exact (close variant)","CampaignStatus":"Paused","AdStatus":"Active","Impressions":1,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"SearchQuery":"data integration tool","Keyword":"data integration tools","AdGroupCriterionId":null,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"Language":"English","KeywordId":84525593559629,"Network":"Syndicated search partners","TopVsOther":"Syndicated search partners - Top","DeviceType":"Tablet","DeviceOS":"Android","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"AccountStatus":"Active","AdGroupStatus":"Active","KeywordStatus":"Active","CampaignType":"Search & content","CustomerId":251186883,"CustomerName":"Daxtarity Inc.","AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":0.0,"TopImpressionRatePercent":100.0,"AverageCpm":0.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0},"emitted_at":1698172090980}
+{"stream":"search_query_performance_report_monthly","data":{"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","AccountId":180278106,"TimePeriod":"2021-07-01","CampaignName":"Test 2","CampaignId":413444833,"AdGroupName":"Airbyte","AdGroupId":1352400325389092,"AdId":84525295496190,"AdType":"Responsive search ad","DestinationUrl":null,"BidMatchType":"Broad","DeliveredMatchType":"Broad","CampaignStatus":"Paused","AdStatus":"Active","Impressions":551,"Clicks":11,"Ctr":2.0,"AverageCpc":0.03,"Spend":0.33,"AveragePosition":0.0,"SearchQuery":"data management platform","Keyword":"data integration tools","AdGroupCriterionId":null,"Conversions":0,"ConversionRate":0.0,"CostPerConversion":null,"Language":"English","KeywordId":84525593559629,"Network":"Syndicated search partners","TopVsOther":"Syndicated search partners - Top","DeviceType":"Smartphone","DeviceOS":"Android","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"AccountStatus":"Active","AdGroupStatus":"Active","KeywordStatus":"Active","CampaignType":"Search & content","CustomerId":251186883,"CustomerName":"Daxtarity Inc.","AllConversions":0,"AllRevenue":0.0,"AllConversionRate":0.0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":5.99,"TopImpressionRatePercent":100.0,"AverageCpm":0.6,"ConversionsQualified":0.0,"AllConversionsQualified":0.0},"emitted_at":1698172100880}

airbyte-integrations/connectors/source-bing-ads/metadata.yaml

@@ -16,7 +16,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 47f25999-dd5e-4636-8c39-e7cea2453331
-  dockerImageTag: 1.1.0
+  dockerImageTag: 1.2.0
   dockerRepository: airbyte/source-bing-ads
   documentationUrl: https://docs.airbyte.com/integrations/sources/bing-ads
   githubIssueLabel: source-bing-ads

airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py

@@ -166,6 +166,7 @@ def report_name(self) -> str:
     def report_columns(self) -> Iterable[str]:
         """
         Specifies bing ads report naming
+        TODO: refactor to use list(self.get_json_schema().get("properties", {}).keys()), see AgeGenderAudienceReport
         """
         pass
 

airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/search_query_performance_report.json

@@ -0,0 +1,180 @@
+{
+  "$schema": "https://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "AccountName": {
+      "type": ["null", "string"]
+    },
+    "AccountNumber": {
+      "type": ["null", "string"]
+    },
+    "AccountId": {
+      "type": ["null", "integer"]
+    },
+    "TimePeriod": {
+      "type": ["null", "string"]
+    },
+    "CampaignName": {
+      "type": ["null", "string"]
+    },
+    "CampaignId": {
+      "type": ["null", "integer"]
+    },
+    "AdGroupName": {
+      "type": ["null", "string"]
+    },
+    "AdGroupId": {
+      "type": ["null", "integer"]
+    },
+    "AdId": {
+      "type": ["null", "integer"]
+    },
+    "AdType": {
+      "type": ["null", "string"]
+    },
+    "DestinationUrl": {
+      "type": ["null", "string"]
+    },
+    "BidMatchType": {
+      "type": ["null", "string"]
+    },
+    "DeliveredMatchType": {
+      "type": ["null", "string"]
+    },
+    "CampaignStatus": {
+      "type": ["null", "string"]
+    },
+    "AdStatus": {
+      "type": ["null", "string"]
+    },
+    "Impressions": {
+      "type": ["null", "integer"]
+    },
+    "Clicks": {
+      "type": ["null", "integer"]
+    },
+    "Ctr": {
+      "type": ["null", "number"]
+    },
+    "AverageCpc": {
+      "type": ["null", "number"]
+    },
+    "Spend": {
+      "type": ["null", "number"]
+    },
+    "AveragePosition": {
+      "type": ["null", "number"]
+    },
+    "SearchQuery": {
+      "type": ["null", "string"]
+    },
+    "Keyword": {
+      "type": ["null", "string"]
+    },
+    "AdGroupCriterionId": {
+      "type": ["null", "string"]
+    },
+    "Conversions": {
+      "type": ["null", "integer"]
+    },
+    "ConversionRate": {
+      "type": ["null", "number"]
+    },
+    "CostPerConversion": {
+      "type": ["null", "integer"]
+    },
+    "Language": {
+      "type": ["null", "string"]
+    },
+    "KeywordId": {
+      "type": ["null", "integer"]
+    },
+    "Network": {
+      "type": ["null", "string"]
+    },
+    "TopVsOther": {
+      "type": ["null", "string"]
+    },
+    "DeviceType": {
+      "type": ["null", "string"]
+    },
+    "DeviceOS": {
+      "type": ["null", "string"]
+    },
+    "Assists": {
+      "type": ["null", "integer"]
+    },
+    "Revenue": {
+      "type": ["null", "number"]
+    },
+    "ReturnOnAdSpend": {
+      "type": ["null", "number"]
+    },
+    "CostPerAssist": {
+      "type": ["null", "number"]
+    },
+    "RevenuePerConversion": {
+      "type": ["null", "number"]
+    },
+    "RevenuePerAssist": {
+      "type": ["null", "number"]
+    },
+    "AccountStatus": {
+      "type": ["null", "string"]
+    },
+    "AdGroupStatus": {
+      "type": ["null", "string"]
+    },
+    "KeywordStatus": {
+      "type": ["null", "string"]
+    },
+    "CampaignType": {
+      "type": ["null", "string"]
+    },
+    "CustomerId": {
+      "type": ["null", "integer"]
+    },
+    "CustomerName": {
+      "type": ["null", "string"]
+    },
+    "AllConversions": {
+      "type": ["null", "integer"]
+    },
+    "AllRevenue": {
+      "type": ["null", "number"]
+    },
+    "AllConversionRate": {
+      "type": ["null", "number"]
+    },
+    "AllCostPerConversion": {
+      "type": ["null", "number"]
+    },
+    "AllReturnOnAdSpend": {
+      "type": ["null", "number"]
+    },
+    "AllRevenuePerConversion": {
+      "type": ["null", "number"]
+    },
+    "Goal": {
+      "type": ["null", "string"]
+    },
+    "GoalType": {
+      "type": ["null", "string"]
+    },
+    "AbsoluteTopImpressionRatePercent": {
+      "type": ["null", "number"]
+    },
+    "TopImpressionRatePercent": {
+      "type": ["null", "number"]
+    },
+    "AverageCpm": {
+      "type": ["null", "number"]
+    },
+    "ConversionsQualified": {
+      "type": ["null", "number"]
+    },
+    "AllConversionsQualified": {
+      "type": ["null", "number"]
+    }
+  }
+}

airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py

@@ -43,6 +43,10 @@
     KeywordPerformanceReportHourly,
     KeywordPerformanceReportMonthly,
     KeywordPerformanceReportWeekly,
+    SearchQueryPerformanceReportDaily,
+    SearchQueryPerformanceReportHourly,
+    SearchQueryPerformanceReportMonthly,
+    SearchQueryPerformanceReportWeekly,
 )
 
 
@@ -80,6 +84,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
             "AdPerformanceReport",
             "CampaignPerformanceReport",
             "GeographicPerformanceReport",
+            "SearchQueryPerformanceReport",
         )
         report_aggregation = ("Hourly", "Daily", "Weekly", "Monthly")
         streams.extend([eval(f"{report}{aggregation}")(client, config) for (report, aggregation) in product(reports, report_aggregation)])

airbyte-integrations/connectors/source-bing-ads/source_bing_ads/streams.py

@@ -860,3 +860,45 @@ class AgeGenderAudienceReportWeekly(AgeGenderAudienceReport):
 
 class AgeGenderAudienceReportMonthly(AgeGenderAudienceReport):
     report_aggregation = "Monthly"
+
+
+class SearchQueryPerformanceReport(PerformanceReportsMixin, BingAdsStream, ABC):
+    data_field: str = ""
+    service_name: str = "ReportingService"
+    report_name: str = "SearchQueryPerformanceReport"
+    operation_name: str = "download_report"
+    additional_fields: str = ""
+    cursor_field = "TimePeriod"
+    report_schema_name = "search_query_performance_report"
+    primary_key = [
+        "SearchQuery",
+        "Keyword",
+        "TimePeriod",
+        "AccountId",
+        "CampaignId",
+        "Language",
+        "DeliveredMatchType",
+        "DeviceType",
+        "DeviceOS",
+        "TopVsOther",
+    ]
+
+    @property
+    def report_columns(self) -> List[str]:
+        return list(self.get_json_schema().get("properties", {}).keys())
+
+
+class SearchQueryPerformanceReportHourly(SearchQueryPerformanceReport):
+    report_aggregation = "Hourly"
+
+
+class SearchQueryPerformanceReportDaily(SearchQueryPerformanceReport):
+    report_aggregation = "Daily"
+
+
+class SearchQueryPerformanceReportWeekly(SearchQueryPerformanceReport):
+    report_aggregation = "Weekly"
+
+
+class SearchQueryPerformanceReportMonthly(SearchQueryPerformanceReport):
+    report_aggregation = "Monthly"

airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py

@@ -28,7 +28,7 @@ def logger_mock_fixture():
 @patch.object(source_bing_ads.source, "Client")
 def test_streams_config_based(mocked_client, config):
     streams = SourceBingAds().streams(config)
-    assert len(streams) == 33
+    assert len(streams) == 37
 
 
 @patch.object(source_bing_ads.source, "Client")

docs/integrations/sources/bing-ads.md

@@ -127,7 +127,8 @@ The Bing Ads API limits the number of requests for all Microsoft Advertising cli
 
 | Version | Date       | Pull Request                                                                                                                     | Subject                                                                                                                                      |
 |:--------|:-----------|:---------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------|
-| 1.1.0   | 2023-10-24 | [31712](https://github.com/airbytehq/airbyte/pull/31712)                                                                         | Add new stream: AgeGenderAudienceReport (daily, hourly, weekly, monthly)                                                                     |
+| 1.2.0   | 2023-10-24 | [31783](https://github.com/airbytehq/airbyte/pull/31783)                                                                         | Add new stream: `SearchQueryPerformanceReport` (daily, hourly, weekly, monthly)                                                              |
+| 1.1.0   | 2023-10-24 | [31712](https://github.com/airbytehq/airbyte/pull/31712)                                                                         | Add new stream: `AgeGenderAudienceReport` (daily, hourly, weekly, monthly)                                                                   |
 | 1.0.2   | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599)                                                                         | Base image migration: remove Dockerfile and use the python-connector-base image                                                              |
 | 1.0.1   | 2023-10-16 | [31432](https://github.com/airbytehq/airbyte/pull/31432)                                                                         | Remove primary keys from the geographic performance reports - complete what was missed in version 1.0.0                                      |
 | 1.0.0   | 2023-10-11 | [31277](https://github.com/airbytehq/airbyte/pull/31277)                                                                         | Remove primary keys from the geographic performance reports.                                                                                 |