✨ Source Bing Ads: add new stream AgeGenderAudienceReport

airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml

@@ -9,8 +9,6 @@ acceptance_tests:
       - config_path: secrets/config.json
   connection:
     tests:
-      - config_path: secrets/config_old.json
-        status: succeed
       - config_path: secrets/config.json
         status: succeed
       - config_path: integration_tests/invalid_config.json
@@ -19,7 +17,7 @@ acceptance_tests:
     tests:
       - config_path: secrets/config.json
         expect_records:
-          path: "integration_tests/expected_records.txt"
+          path: "integration_tests/expected_records.jsonl"
           extra_records: yes
         empty_streams:
           - name: account_performance_report_hourly
@@ -34,6 +32,8 @@ acceptance_tests:
             bypass_reason: "Hourly reports are disabled, because sync is too long"
           - name: geographic_performance_report_hourly
             bypass_reason: "Hourly reports are disabled, because sync is too long"
+          - name: age_gender_audience_report_hourly
+            bypass_reason: "Empty report; hourly data fetched is limited to 180 days"
         timeout_seconds: 900
   full_refresh:
     tests:

airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog.json

@@ -265,6 +265,46 @@
       "sync_mode": "incremental",
       "cursor_field": ["TimePeriod"],
       "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "age_gender_audience_report_daily",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental", "full_refresh"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["TimePeriod"],
+      "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "age_gender_audience_report_weekly",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental", "full_refresh"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["TimePeriod"],
+      "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "age_gender_audience_report_hourly",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental", "full_refresh"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["TimePeriod"],
+      "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "age_gender_audience_report_monthly",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental", "full_refresh"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["TimePeriod"],
+      "destination_sync_mode": "append"
     }
   ]
 }

airbyte-integrations/connectors/source-bing-ads/metadata.yaml

@@ -16,7 +16,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 47f25999-dd5e-4636-8c39-e7cea2453331
-  dockerImageTag: 1.0.2
+  dockerImageTag: 1.1.0
   dockerRepository: airbyte/source-bing-ads
   documentationUrl: https://docs.airbyte.com/integrations/sources/bing-ads
   githubIssueLabel: source-bing-ads

airbyte-integrations/connectors/source-bing-ads/setup.py

@@ -5,7 +5,7 @@
 
 from setuptools import find_packages, setup
 
-MAIN_REQUIREMENTS = ["airbyte-cdk", "bingads~=13.0.13", "vcrpy==4.1.1", "backoff==1.10.0", "pendulum==2.1.2", "urllib3<2.0"]
+MAIN_REQUIREMENTS = ["airbyte-cdk", "bingads~=13.0.13", "urllib3<2.0"]
 
 TEST_REQUIREMENTS = [
     "requests-mock~=1.9.3",

airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py

@@ -11,6 +11,7 @@
 from airbyte_cdk.models import SyncMode
 from airbyte_cdk.sources.streams.core import package_name_from_class
 from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader
+from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
 from bingads.service_client import ServiceClient
 from bingads.v13.internal.reporting.row_report import _RowReport
 from bingads.v13.internal.reporting.row_report_iterator import _RowReportRecord
@@ -149,6 +150,7 @@ class ReportsMixin(ABC):
     timeout: int = 300000
     report_file_format: str = "Csv"
 
+    transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization)
     primary_key: List[str] = ["TimePeriod", "Network", "DeviceType"]
 
     @property
@@ -296,25 +298,18 @@ def parse_response(self, response: sudsobject.Object, **kwargs: Mapping[str, Any
 
         yield from []
 
-    def get_column_value(self, row: _RowReportRecord, column: str) -> Union[str, None, int, float]:
+    @staticmethod
+    def get_column_value(row: _RowReportRecord, column: str) -> Union[str, None, int, float]:
         """
-        Reads field value from row and transforms string type field to numeric if possible
+        Reads field value from row and transforms:
+        1. empty values to logical None
+        2. Percent values to numeric string e.g. "12.25%" -> "12.25"
         """
         value = row.value(column)
-        if value == "":
+        if not value or value == "--":
             return None
-
-        if value is not None and column in REPORT_FIELD_TYPES:
-            if REPORT_FIELD_TYPES[column] == "integer":
-                value = 0 if value == "--" else int(value.replace(",", ""))
-            elif REPORT_FIELD_TYPES[column] == "number":
-                if value == "--":
-                    value = 0.0
-                else:
-                    if "%" in value:
-                        value = float(value.replace("%", "").replace(",", "")) / 100
-                    else:
-                        value = float(value.replace(",", ""))
+        if "%" in value:
+            value = value.replace("%", "")
 
         return value
 
@@ -348,7 +343,7 @@ def get_start_date(self, stream_state: Mapping[str, Any] = None, account_id: str
 
         if self.config.get("lookback_window"):
             # Datetime subtract won't work with days = 0
-            # it'll output an AirbuteError
+            # it'll output an AirbyteError
             return start_date.subtract(days=self.config["lookback_window"])
         else:
             return start_date

airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/age_gender_audience_report.json

@@ -0,0 +1,108 @@
+{
+  "$schema": "https://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "AccountId": {
+      "type": ["null", "integer"]
+    },
+    "AgeGroup": {
+      "type": ["null", "string"]
+    },
+    "Gender": {
+      "type": ["null", "string"]
+    },
+    "TimePeriod": {
+      "type": ["null", "string"]
+    },
+    "AllConversions": {
+      "type": ["null", "integer"]
+    },
+    "AccountName": {
+      "type": ["null", "string"]
+    },
+    "AccountNumber": {
+      "type": ["null", "string"]
+    },
+    "CampaignName": {
+      "type": ["null", "string"]
+    },
+    "CampaignId": {
+      "type": ["null", "integer"]
+    },
+    "AdGroupName": {
+      "type": ["null", "string"]
+    },
+    "AdGroupId": {
+      "type": ["null", "integer"]
+    },
+    "AdDistribution": {
+      "type": ["null", "string"]
+    },
+    "Impressions": {
+      "type": ["null", "integer"]
+    },
+    "Clicks": {
+      "type": ["null", "integer"]
+    },
+    "Conversions": {
+      "type": ["null", "number"]
+    },
+    "Spend": {
+      "type": ["null", "number"]
+    },
+    "Revenue": {
+      "type": ["null", "number"]
+    },
+    "ExtendedCost": {
+      "type": ["null", "number"]
+    },
+    "Assists": {
+      "type": ["null", "integer"]
+    },
+    "Language": {
+      "type": ["null", "string"]
+    },
+    "AccountStatus": {
+      "type": ["null", "string"]
+    },
+    "CampaignStatus": {
+      "type": ["null", "string"]
+    },
+    "AdGroupStatus": {
+      "type": ["null", "string"]
+    },
+    "BaseCampaignId": {
+      "type": ["null", "string"]
+    },
+    "AllRevenue": {
+      "type": ["null", "number"]
+    },
+    "ViewThroughConversions": {
+      "type": ["null", "integer"]
+    },
+    "Goal": {
+      "type": ["null", "string"]
+    },
+    "GoalType": {
+      "type": ["null", "string"]
+    },
+    "AbsoluteTopImpressionRatePercent": {
+      "type": ["null", "number"]
+    },
+    "TopImpressionRatePercent": {
+      "type": ["null", "number"]
+    },
+    "ConversionsQualified": {
+      "type": ["null", "number"]
+    },
+    "AllConversionsQualified": {
+      "type": ["null", "number"]
+    },
+    "ViewThroughConversionsQualified": {
+      "type": ["null", "number"]
+    },
+    "ViewThroughRevenue": {
+      "type": ["null", "number"]
+    }
+  }
+}