✨ Source Google-Ads: add incremental streams based on events

airbyte-integrations/connectors/source-google-ads/Dockerfile

@@ -13,5 +13,5 @@ COPY main.py ./
 
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.8.1
+LABEL io.airbyte.version=0.9.0
 LABEL io.airbyte.name=airbyte/source-google-ads

airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml

@@ -20,18 +20,35 @@ acceptance_tests:
       - config_path: "secrets/config.json"
         expect_records:
           path: "integration_tests/expected_records.jsonl"
-        timeout_seconds: 600
+        timeout_seconds: 3600
         empty_streams:
-          - name: "display_topics_performance_report"
-            bypass_reason: "Stream not filled yet."
           - name: "account_labels"
-            bypass_reason: "Unable to seed the stream"
+            bypass_reason: "Data is present in UI, but not in API: supposedly insufficient permissions"
+          - name: "shopping_performance_report"
+            bypass_reason: "No shopping campaign, need item for sale"
+          - name: "display_topics_performance_report"
+            bypass_reason: "No data for this date range, tested in next config"
           - name: "click_view"
-            bypass_reason: "Stream not filled yet."
-          - name: "unhappytable"
-            bypass_reason: "Stream not filled yet."
+            bypass_reason: "Stream has data only for last 90 days, next config is used for testing it"
+        ignored_fields:
+          accounts:
+            - name: customer.optimization_score_weight
+              bypass_reason: "Value can be updated by Google Ads"
+            - name: customer.optimization_score
+              bypass_reason: "Value can be updated by Google Ads"
+      - config_path: "secrets/config_click_view.json"
+        expect_records:
+          path: "integration_tests/expected_records_click.jsonl"
+        timeout_seconds: 3600
+        empty_streams:
+          - name: "account_labels"
+            bypass_reason: "Data is present in UI, but not in API: supposedly insufficient permissions"
           - name: "shopping_performance_report"
-            bypass_reason: "Stream not filled yet."
+            bypass_reason: "No shopping campaign, need item for sale"
+          - name: "display_keyword_performance_report"
+            bypass_reason: "No data for this date range, tested in previous config"
+          - name: "keyword_report"
+            bypass_reason: "No data for this date range, tested in previous config"
         ignored_fields:
           accounts:
             - name: customer.optimization_score_weight
@@ -42,17 +59,10 @@ acceptance_tests:
     tests:
       - config_path: "secrets/config.json"
         configured_catalog_path: "integration_tests/configured_catalog.json"
-      - config_path: "secrets/config_with_gaql.json"
-        configured_catalog_path: "integration_tests/configured_catalog_with_gaql_only.json"
-        # This config allows to read from the click_view stream which is empty in other configs.
-        # It should be tested anyway because it has different date range compared to other streams.
-      - config_path: "secrets/config_click_view.json"
-        configured_catalog_path: "integration_tests/configured_catalog_with_click_view.json"
   incremental:
     tests:
       - config_path: "secrets/incremental_config.json"
         timeout_seconds: 3600
         configured_catalog_path: "integration_tests/incremental_catalog.json"
-        skip_comprehensive_incremental_tests: true
         future_state:
           future_state_path: "integration_tests/abnormal_state.json"

airbyte-integrations/connectors/source-google-ads/integration_tests/abnormal_state.json

@@ -110,5 +110,31 @@
       "stream_state": { "4651612872": { "segments.date": "2222-01-01" } },
       "stream_descriptor": { "name": "ad_group_bidding_strategies" }
     }
+  },
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "change_status": {
+          "4651612872": {
+            "change_status.last_change_date_time": "2024-08-16 13:20:01.003295"
+          }
+        }
+      },
+      "stream_descriptor": { "name": "ad_group_criterions" }
+    }
+  },
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "change_status": {
+          "4651612872": {
+            "change_status.last_change_date_time": "2024-08-16 13:20:01.003295"
+          }
+        }
+      },
+      "stream_descriptor": { "name": "campaign_criterion" }
+    }
   }
 ]

airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json

@@ -12,6 +12,23 @@
       "destination_sync_mode": "overwrite",
       "cursor_field": ["segments.date"]
     },
+    {
+      "stream": {
+        "name": "campaign_budget",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["segments.date"],
+        "source_defined_primary_key": [
+          ["campaign_budget.id"],
+          ["segments.date"]
+        ]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite",
+      "cursor_field": ["segments.date"],
+      "primary_key": [["campaign_budget.id"], ["segments.date"]]
+    },
     {
       "stream": {
         "name": "ad_group_custom",
@@ -323,29 +340,23 @@
       "stream": {
         "name": "ad_group_criterions",
         "json_schema": {},
-        "supported_sync_modes": ["full_refresh"],
-        "source_defined_primary_key": [
-          ["ad_group.id"],
-          ["ad_group_criterion.criterion_id"]
-        ]
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_primary_key": [["ad_group_criterion.resource_name"]]
       },
-      "sync_mode": "full_refresh",
+      "sync_mode": "incremental",
       "destination_sync_mode": "overwrite",
-      "primary_key": [["ad_group.id"], ["ad_group_criterion.criterion_id"]]
+      "primary_key": [["ad_group_criterion.resource_name"]]
     },
     {
       "stream": {
         "name": "ad_listing_group_criterions",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh"],
-        "source_defined_primary_key": [
-          ["ad_group.id"],
-          ["ad_group_criterion.criterion_id"]
-        ]
+        "source_defined_primary_key": [["ad_group_criterion.resource_name"]]
       },
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite",
-      "primary_key": [["ad_group.id"], ["ad_group_criterion.criterion_id"]]
+      "primary_key": [["ad_group_criterion.resource_name"]]
     },
     {
       "stream": {
@@ -362,20 +373,14 @@
     },
     {
       "stream": {
-        "name": "campaign_budget",
+        "name": "campaign_criterion",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["segments.date"],
-        "source_defined_primary_key": [
-          ["campaign_budget.id"],
-          ["segments.date"]
-        ]
+        "source_defined_primary_key": [["campaign_criterion.resource_name"]]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "overwrite",
-      "cursor_field": ["segments.date"],
-      "primary_key": [["campaign_budget.id"], ["segments.date"]]
+      "primary_key": [["campaign_criterion.resource_name"]]
     }
   ]
 }