🎉 Recurly Schema Revamp (#9866)

* Cleanup Recurly connector schemas * Add more Recurly schemas to the connector - `billing_infos` - `shipping_addresses` - `shipping_methods` - `subscription_changes` * Add Recurly `add-on` resouce * Add Recurly's account notes resource schema * Add unique coupons to Recurly source * Add credit payments to Recurly connector * Add Recurly resources to integration tests configurations * Bump Recurly source version to `0.4.0` * Add `line_items` Recurly resource * Add `line_items` to Recurly documentation * Add missing `line_items` JSON schema * Replace Subscription Change Recurly API call with Subscription `pending_changes` field * Replace Recurly unique coupon codes API call with coupons `unique_coupon` field To avoid the extra API call to import unique coupon calls * Revert "Replace Recurly unique coupon codes API call with coupons `unique_coupon` field" This reverts commit 1c4592d. * Add `end_time` parameter to Recurly connector * Order Recurly specs * Set the Recurly `begin_time` and `end_time` to be optional * Add `order` to Recurly `source_spec.yaml` * Add `maxLength` to Recurly source schemas * Set `maxLength` for Recurly Subscription and Transaction `uuid` * Fix Recurly `export_dates` acceptance tests
airbytehq · Mar 8, 2022 · 63af98e · 63af98e
1 parent 9dede53
commit 63af98e
Show file tree

Hide file tree

Showing 29 changed files with 2,115 additions and 2,511 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -634,7 +634,7 @@
 - name: Recurly
   sourceDefinitionId: cd42861b-01fc-4658-a8ab-5d11d0510f01
   dockerRepository: airbyte/source-recurly
-  dockerImageTag: 0.3.2
+  dockerImageTag: 0.4.0
   documentationUrl: https://docs.airbyte.io/integrations/sources/recurly
   icon: recurly.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -6617,7 +6617,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-recurly:0.3.2"
+- dockerImage: "airbyte/source-recurly:0.4.0"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/recurly"
     connectionSpecification:
@@ -6628,19 +6628,29 @@
       - "api_key"
       additionalProperties: false
       properties:
+        api_key:
+          type: "string"
+          title: "API Key"
+          airbyte_secret: true
+          description: "Recurly API Key. See the  <a href=\"https://docs.airbyte.com/integrations/sources/recurly\"\
+            >docs</a> for more information on how to generate this key."
+          order: 1
         begin_time:
           type: "string"
           description: "ISO8601 timestamp from which the replication from Recurly\
             \ API will start from."
           examples:
           - "2021-12-01T00:00:00"
-          pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$"
-        api_key:
+          pattern: "^$|^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$"
+          order: 2
+        end_time:
           type: "string"
-          title: "API Key"
-          airbyte_secret: true
-          description: "Recurly API Key. See the  <a href=\"https://docs.airbyte.com/integrations/sources/recurly\"\
-            >docs</a> for more information on how to generate this key."
+          description: "ISO8601 timestamp to which the replication from Recurly API\
+            \ will stop. Records after that date won't be imported."
+          examples:
+          - "2021-12-01T00:00:00"
+          pattern: "^$|^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$"
+          order: 3
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []

diff --git a/airbyte-integrations/connectors/source-recurly/Dockerfile b/airbyte-integrations/connectors/source-recurly/Dockerfile
@@ -34,5 +34,5 @@ COPY source_recurly ./source_recurly
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.3.2
+LABEL io.airbyte.version=0.4.0
 LABEL io.airbyte.name=airbyte/source-recurly
diff --git a/airbyte-integrations/connectors/source-recurly/acceptance-test-config.yml b/airbyte-integrations/connectors/source-recurly/acceptance-test-config.yml
@@ -14,7 +14,8 @@ tests:
   basic_read:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
-      empty_streams: ["account_coupon_redemptions"]
+      empty_streams: ["account_coupon_redemptions", "account_notes", "add_ons", "billing_infos", "credit_payments",
+                      "line_items", "shipping_methods", "unique_coupons", "export_dates"]
   full_refresh:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"

diff --git a/airbyte-integrations/connectors/source-recurly/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-recurly/integration_tests/configured_catalog.json
@@ -12,6 +12,54 @@
       "sync_mode": "incremental",
       "destination_sync_mode": "overwrite"
     },
+    {
+      "stream": {
+        "name": "account_coupon_redemptions",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["updated_at"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "account_notes",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["created_at"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "add_ons",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["updated_at"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "billing_infos",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["updated_at"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite"
+    },
     {
       "stream": {
         "name": "coupons",
@@ -26,7 +74,7 @@
     },
     {
       "stream": {
-        "name": "account_coupon_redemptions",
+        "name": "credit_payments",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
@@ -35,6 +83,15 @@
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "overwrite"
+    },
+      {
+      "stream": {
+        "name": "export_dates",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
     },
     {
       "stream": {
@@ -50,7 +107,7 @@
     },
     {
       "stream": {
-        "name": "plans",
+        "name": "line_items",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
@@ -72,6 +129,42 @@
       "sync_mode": "incremental",
       "destination_sync_mode": "overwrite"
     },
+    {
+      "stream": {
+        "name": "plans",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["updated_at"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "shipping_addresses",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["updated_at"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "shipping_methods",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["updated_at"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite"
+    },
     {
       "stream": {
         "name": "subscriptions",
@@ -95,6 +188,18 @@
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "unique_coupons",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["updated_at"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite"
     }
   ]
 }
diff --git a/airbyte-integrations/connectors/source-recurly/integration_tests/future_state.json b/airbyte-integrations/connectors/source-recurly/integration_tests/future_state.json
@@ -1,26 +1,50 @@
 {
+  "accounts": {
+    "updated_at": "2022-07-19T22:21:37Z"
+  },
   "account_coupon_redemptions": {
     "updated_at": "2022-07-19T22:21:37Z"
   },
-  "accounts": {
+  "account_notes": {
+    "created_at": "2022-07-19T22:21:37Z"
+  },
+  "add_ons": {
+    "updated_at": "2022-07-19T22:21:37Z"
+  },
+  "billing_infos": {
     "updated_at": "2022-07-19T22:21:37Z"
   },
   "coupons": {
     "updated_at": "2022-07-19T22:21:37Z"
   },
+  "credit_payments": {
+    "updated_at": "2022-07-19T22:21:37Z"
+  },
   "invoices": {
     "updated_at": "2022-07-19T22:21:37Z"
   },
+  "line_items": {
+    "updated_at": "2022-07-19T22:21:37Z"
+  },
   "measured_units": {
     "updated_at": "2022-07-19T22:21:37Z"
   },
   "plans": {
     "updated_at": "2022-07-19T22:21:37Z"
   },
+  "shipping_addresses": {
+    "updated_at": "2022-07-19T22:21:37Z"
+  },
+  "shipping_methods": {
+    "updated_at": "2022-07-19T22:21:37Z"
+  },
   "subscriptions": {
     "updated_at": "2022-07-19T22:21:37Z"
   },
   "transactions": {
     "updated_at": "2022-07-19T22:21:37Z"
+  },
+  "unique_coupons": {
+    "updated_at": "2022-07-19T22:21:37Z"
   }
 }
diff --git a/airbyte-integrations/connectors/source-recurly/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-recurly/sample_files/configured_catalog.json
@@ -2,107 +2,12 @@
   "streams": [
     {
       "stream": {
-        "name": "accounts",
+        "name": "unique_coupons",
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
         "default_cursor_field": ["updated_at"],
         "source_defined_primary_key": [["id"]],
-        "json_schema": {
-          "$schema": "http://json-schema.org/draft-07/schema#",
-          "type": "object",
-          "properties": {
-            "id": {
-              "type": "string"
-            },
-            "object": {
-              "type": "string"
-            },
-            "code": {
-              "type": "string"
-            },
-            "parent_account_id": {
-              "type": "string"
-            },
-            "bill_to": {
-              "type": "string"
-            },
-            "state": {
-              "type": "string"
-            },
-            "username": {
-              "type": "string"
-            },
-            "email": {
-              "type": "string"
-            },
-            "cc_emails": {
-              "type": "string"
-            },
-            "preferred_locale": {
-              "type": "string"
-            },
-            "first_name": {
-              "type": "string"
-            },
-            "last_name": {
-              "type": "string"
-            },
-            "company": {
-              "type": "string"
-            },
-            "vat_number": {
-              "type": "string"
-            },
-            "tax_exempt": {
-              "type": "boolean"
-            },
-            "exemption_certificate": {
-              "type": "string"
-            },
-            "address": {
-              "type": "object"
-            },
-            "billing_info": {
-              "type": "object"
-            },
-            "shipping_addresses": {
-              "type": "array"
-            },
-            "custom_fields": {
-              "type": "array"
-            },
-            "has_live_subscription": {
-              "type": "boolean"
-            },
-            "has_active_subscription": {
-              "type": "boolean"
-            },
-            "has_future_subscription": {
-              "type": "boolean"
-            },
-            "has_canceled_subscription": {
-              "type": "boolean"
-            },
-            "has_paused_subscription": {
-              "type": "boolean"
-            },
-            "has_past_due_invoice": {
-              "type": "boolean"
-            },
-            "dunning_campaign_id": {
-              "type": ["null", "string"]
-            },
-            "created_at": {
-              "type": "string"
-            },
-            "updated_at": {
-              "type": "string"
-            },
-            "deleted_at": {
-              "type": "string"
-            }
-          }
-        }
+        "json_schema": {}
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "overwrite"