Source Chargebee: Updates schemas for validation and missing fields e…

…rrors, updates test bypass, adds expected records, adds custom error handling, adds incremental support for three streams (#34053)
airbytehq · Feb 13, 2024 · fa66dc8 · fa66dc8
1 parent 180335c
commit fa66dc8
Show file tree

Hide file tree

Showing 38 changed files with 1,254 additions and 1,929 deletions.
diff --git a/airbyte-integrations/connectors/source-chargebee/acceptance-test-config.yml b/airbyte-integrations/connectors/source-chargebee/acceptance-test-config.yml
@@ -28,24 +28,22 @@ acceptance_tests:
             bypass_reason: "Not available for Product Catalog 2.0 sites."
           - name: "virtual_bank_account"
             bypass_reason: "Cannot populate with test data"
-          - name: "unbilled_charge"
-            bypass_reason: "Empty stream. Unstable data"
-          - name: "hosted_page"
-            bypass_reason: "Empty stream. Unstable data"
           - name: "event"
-            bypass_reason: "Unstable data. Updated daily."
+            bypass_reason: "Unstable data. Test data is not persistent."
           - name: "site_migration_detail"
             bypass_reason: "Cannnot populate with test data."
-        ignored_fields:
-          subscription:
-            - name: "current_term_start"
-              bypass_reason: "Field updated daily."
+          - name: "customer"
+            bypass_reason: "To be Tested with integration tests."
+          - name: "subscription"
+            bypass_reason: "To be Tested with integration tests."
+          - name: "coupon"
+            bypass_reason: "To be Tested with integration tests."
         expect_records:
           path: "integration_tests/expected_records.jsonl"
           extra_fields: no
           exact_order: no
           extra_records: yes
-        fail_on_extra_columns: false
+        fail_on_extra_columns: true
   incremental:
     tests:
       - config_path: "secrets/config.json"
@@ -54,12 +52,12 @@ acceptance_tests:
         future_state:
           future_state_path: "integration_tests/future_state.json"
           missing_streams:
-            - name: attached_item
-              bypass_reason: "This stream is Full-Refresh only"
             - name: contact
               bypass_reason: "This stream is Full-Refresh only"
             - name: quote_line_group
               bypass_reason: "This stream is Full-Refresh only"
+            - name: attached_item
+              bypass_reason: "This stream is Full-Refresh only"
   full_refresh:
     tests:
       - config_path: "secrets/config.json"

diff --git a/airbyte-integrations/connectors/source-chargebee/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-chargebee/integration_tests/configured_catalog.json
@@ -13,44 +13,6 @@
       "destination_sync_mode": "append",
       "cursor_field": ["updated_at"]
     },
-    {
-      "stream": {
-        "name": "event",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["occurred_at"],
-        "source_defined_primary_key": [["id"]]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append",
-      "cursor_field": ["occurred_at"]
-    },
-    {
-      "stream": {
-        "name": "customer",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["updated_at"],
-        "source_defined_primary_key": [["id"]]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append",
-      "cursor_field": ["updated_at"]
-    },
-    {
-      "stream": {
-        "name": "contact",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["updated_at"],
-        "source_defined_primary_key": [["id"]]
-      },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "append"
-    },
     {
       "stream": {
         "name": "invoice",
@@ -107,13 +69,11 @@
       "stream": {
         "name": "attached_item",
         "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["updated_at"],
+        "supported_sync_modes": ["full_refresh"],
         "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "full_refresh",
-      "destination_sync_mode": "append"
+      "destination_sync_mode": "overwrite"
     },
     {
       "stream": {
@@ -147,7 +107,7 @@
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["updated_at"],
+        "default_cursor_field": ["created_at"],
         "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "incremental",
@@ -160,25 +120,12 @@
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["updated_at"],
-        "source_defined_primary_key": [["id"]]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append",
-      "cursor_field": ["updated_at"]
-    },
-    {
-      "stream": {
-        "name": "coupon",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["updated_at"],
-        "source_defined_primary_key": [["id"]]
+        "source_defined_primary_key": [["id"]],
+        "default_cursor_field": ["created_at"]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append",
-      "cursor_field": ["updated_at"]
+      "cursor_field": ["created_at"]
     },
     {
       "stream": {
@@ -212,8 +159,8 @@
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["updated_at"],
-        "source_defined_primary_key": [["id"]]
+        "source_defined_primary_key": [["id"]],
+        "default_cursor_field": ["updated_at"]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append",
@@ -236,26 +183,11 @@
       "stream": {
         "name": "quote_line_group",
         "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["updated_at"],
+        "supported_sync_modes": ["full_refresh"],
         "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "full_refresh",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "site_migration_detail",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["migrated_at"],
-        "source_defined_primary_key": [["entity_id"]]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append",
-      "cursor_field": ["migrated_at"]
+      "destination_sync_mode": "overwrite"
     },
     {
       "stream": {