Fix expected records and primary keys

airbytehq · Aug 15, 2023 · d255428 · d255428
1 parent 0e7fe43
commit d255428
Show file tree

Hide file tree

Showing 7 changed files with 120 additions and 145 deletions.
diff --git a/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml
@@ -26,63 +26,18 @@ acceptance_tests:
             bypass_reason: "Stream not filled yet."
           - name: "account_labels"
             bypass_reason: "Unable to seed the stream"
-          - name: "click_view"
-            bypass_reason: "Stream not filled yet."
           - name: "unhappytable"
             bypass_reason: "Stream not filled yet."
           - name: "shopping_performance_report"
             bypass_reason: "Stream not filled yet."
-          - name: "campaign_budget"
-            bypass_reason: "Stream not filled yet."
-      - config_path: "secrets/config_click_view.json"
-        expect_records:
-          path: "integration_tests/expected_records.jsonl"
-        timeout_seconds: 600
-        empty_streams:
-          - name: "accounts"
-            bypass_reason: "Floating data"
-          - name: "display_topics_performance_report"
-            bypass_reason: "Stream not filled yet."
-          - name: "account_labels"
-            bypass_reason: "Unable to seed the stream"
-          - name: "click_view"
-            bypass_reason: "Stream not filled yet."
-          - name: "unhappytable"
-            bypass_reason: "Stream not filled yet."
-          - name: "shopping_performance_report"
-            bypass_reason: "Stream not filled yet."
-          - name: "display_keyword_performance_report"
-            bypass_reason: "Stream not filled yet."
-          - name: "keyword_report"
-            bypass_reason: "Stream not filled yet."
-          - name: "ad_group_bidding_strategies"
-            bypass_reason: "Stream not filled yet."
-          - name: "labels"
-            bypass_reason: "Stream not filled yet."
-          - name: "user_interest"
-            bypass_reason: "Stream not filled yet."
-          - name: "ad_group_criterion_labels"
-            bypass_reason: "Stream not filled yet."
-          - name: "ad_group_criterions"
-            bypass_reason: "Stream not filled yet."
-          - name: "campaign_bidding_strategies"
-            bypass_reason: "Stream not filled yet."
-          - name: "ad_listing_group_criterions"
-            bypass_reason: "Stream not filled yet."
-          - name: "campaign_budget"
+          - name: "ad_group_ad_report"
             bypass_reason: "Stream not filled yet."
-          - name: "ad_group_criterions"
+          - name: "ad_group_ad_label_custom"
             bypass_reason: "Stream not filled yet."
   full_refresh:
     tests:
       - config_path: "secrets/config.json"
         configured_catalog_path: "integration_tests/configured_catalog.json"
-      - config_path: "secrets/config_with_gaql.json"
-        configured_catalog_path: "integration_tests/configured_catalog_with_gaql_only.json"
-        # This config allows to read from the click_view stream which is empty in other configs.
-        # It should be tested anyway because it has different date range compared to other streams.
-      - config_path: "secrets/config_click_view.json"
-        configured_catalog_path: "integration_tests/configured_catalog_with_click_view.json"
   incremental:
     tests:
       - config_path: "secrets/incremental_config.json"

diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json
@@ -14,6 +14,20 @@
       "primary_key": [["customer.id"], ["ad_group_ad.ad.id"], ["segments.date"], ["segments.ad_network_type"]],
       "cursor_field": ["segments.date"]
     },
+    {
+      "stream": {
+        "name": "campaign_budget",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["segments.date"],
+        "source_defined_primary_key": [["customer.id"], ["campaign_budget.id"], ["segments.date"]]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite",
+      "cursor_field": ["segments.date"],
+      "primary_key": [["customer.id"], ["campaign_budget.id"], ["segments.date"]]
+    },
     {
       "stream": {
         "name": "ad_group_custom",
@@ -48,26 +62,26 @@
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "source_defined_primary_key": [["click_view.gclid"], ["segments.date"], ["segments.ad_network_type"]],
+        "source_defined_primary_key": [["customer.id"], ["click_view.gclid"], ["segments.date"], ["segments.ad_network_type"]],
         "default_cursor_field": ["segments.date"]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "overwrite",
       "cursor_field": ["segments.date"],
-      "primary_key": [["click_view.gclid"], ["segments.date"], ["segments.ad_network_type"]]
+      "primary_key": [["customer.id"], ["click_view.gclid"], ["segments.date"], ["segments.ad_network_type"]]
     },
     {
       "stream": {
         "name": "geographic_report",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_primary_key": [["geographic_view.country_criterion_id"], ["ad_group.id"], ["segments.date"]],
+        "source_defined_primary_key": [["customer.id"], ["geographic_view.country_criterion_id"], ["ad_group.id"], ["segments.date"]],
         "source_defined_cursor": true,
         "default_cursor_field": ["segments.date"]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "overwrite",
-      "primary_key": [["geographic_view.country_criterion_id"], ["ad_group.id"], ["segments.date"]],
+      "primary_key": [["customer.id"], ["geographic_view.country_criterion_id"], ["ad_group.id"], ["segments.date"]],
       "cursor_field": ["segments.date"]
     },
     {
@@ -128,16 +142,13 @@
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "source_defined_primary_key": [
-          ["ad_group_ad.ad.id"],
-          ["segments.date"]
-        ],
+        "source_defined_primary_key": [["customer.id"], ["ad_group_ad.ad.id"], ["segments.date"]],
         "default_cursor_field": ["segments.date"]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "overwrite",
       "cursor_field": ["segments.date"],
-      "primary_key": [["ad_group_ad.ad.id"], ["segments.date"]]
+      "primary_key": [["customer.id"], ["ad_group_ad.ad.id"], ["segments.date"]]
     },
     {
       "stream": {
@@ -174,23 +185,23 @@
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
         "default_cursor_field": ["segments.date"],
-        "source_defined_primary_key": [["customer.id"], ["campaign.id"], ["segments.date"]]
+        "source_defined_primary_key": [["customer.id"], ["campaign.id"], ["segments.date"], ["segments.hour"], ["segments.ad_network_type"]]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "overwrite",
       "cursor_field": ["segments.date"],
-      "primary_key": [["customer.id"], ["campaign.id"], ["segments.date"]]
+      "primary_key": [["customer.id"], ["campaign.id"], ["segments.date"], ["segments.hour"], ["segments.ad_network_type"]]
     },
     {
       "stream": {
         "name": "campaign_labels",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh"],
-        "source_defined_primary_key": [["campaign.id"], ["labels.id"]]
+        "source_defined_primary_key": [["customer.id"], ["campaign.id"], ["label.id"]]
       },
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite",
-      "primary_key": [["campaign.id"], ["labels.id"]]
+      "primary_key": [["customer.id"], ["campaign.id"], ["label.id"]]
     },
     {
       "stream": {
@@ -293,38 +304,38 @@
         "name": "account_labels",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh"],
-        "source_defined_primary_key": [["customer.id"], ["label.id"]]
+        "source_defined_primary_key": [["customer_label.resource_name"]]
       },
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite",
-      "primary_key": [["customer.id"], ["label.id"]]
+      "primary_key": [["customer_label.resource_name"]]
     },
     {
       "stream": {
         "name": "campaign_bidding_strategies",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_primary_key": [["campaign.id"], ["bidding_strategy.id"], ["segments.date"]],
+        "source_defined_primary_key": [["customer.id"], ["campaign.id"], ["bidding_strategy.id"], ["segments.date"]],
         "source_defined_cursor": true,
         "default_cursor_field": ["segments.date"]
       },
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite",
-      "primary_key": [["campaign.id"], ["bidding_strategy.id"], ["segments.date"]],
+      "primary_key": [["customer.id"], ["campaign.id"], ["bidding_strategy.id"], ["segments.date"]],
       "cursor_field": ["segments.date"]
     },
     {
       "stream": {
         "name": "ad_group_bidding_strategies",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_primary_key": [["ad_group.id"], ["bidding_strategy.id"], ["segments.date"]],
+        "source_defined_primary_key": [["customer.id"], ["ad_group.id"], ["bidding_strategy.id"], ["segments.date"]],
         "source_defined_cursor": true,
         "default_cursor_field": ["segments.date"]
       },
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite",
-      "primary_key": [["ad_group.id"], ["bidding_strategy.id"], ["segments.date"]],
+      "primary_key": [["customer.id"], ["ad_group.id"], ["bidding_strategy.id"], ["segments.date"]],
       "cursor_field": ["segments.date"]
     },
     {