🐛 Source Recharge: change cursor_field from created_at on updated_at (#…

…6149) * Source Recharge: change cursor_field from created_at on updated_at
airbytehq · Sep 17, 2021 · 76140b5 · 76140b5
1 parent 58b6a1b
commit 76140b5
Show file tree

Hide file tree

Showing 11 changed files with 55 additions and 46 deletions.
diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/45d2e135-2ede-49e1-939f-3e3ec357a65e.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/45d2e135-2ede-49e1-939f-3e3ec357a65e.json
@@ -2,6 +2,6 @@
   "sourceDefinitionId": "45d2e135-2ede-49e1-939f-3e3ec357a65e",
   "name": "Recharge",
   "dockerRepository": "airbyte/source-recharge",
-  "dockerImageTag": "0.1.1",
+  "dockerImageTag": "0.1.2",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/recharge"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -324,7 +324,7 @@
 - sourceDefinitionId: 45d2e135-2ede-49e1-939f-3e3ec357a65e
   name: Recharge
   dockerRepository: airbyte/source-recharge
-  dockerImageTag: 0.1.1
+  dockerImageTag: 0.1.2
   documentationUrl: https://docs.airbyte.io/integrations/sources/recharge
 - sourceDefinitionId: fe2b4084-3386-4d3b-9ad6-308f61a6f1e6
   name: Harvest

diff --git a/airbyte-integrations/connectors/source-recharge/Dockerfile b/airbyte-integrations/connectors/source-recharge/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.1
+LABEL io.airbyte.version=0.1.2
 LABEL io.airbyte.name=airbyte/source-recharge
diff --git a/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml b/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml
@@ -12,12 +12,13 @@ tests:
   basic_read:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/streams_with_output_records_catalog.json"
+      timeout_seconds: 1200
   incremental:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/streams_with_output_records_catalog.json"
       future_state_path: "integration_tests/abnormal_state.json"
-      cursor_paths:
-        addresses: [ "created_at" ]
+      timeout_seconds: 900
   full_refresh:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
+      timeout_seconds: 1200
diff --git a/airbyte-integrations/connectors/source-recharge/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-recharge/integration_tests/abnormal_state.json
@@ -1,23 +1,23 @@
 {
   "addresses": {
-    "created_at": "2050-05-18T00:00:00"
+    "updated_at": "2050-05-18T00:00:00"
   },
   "charges": {
-    "created_at": "2050-05-18T00:00:00"
+    "updated_at": "2050-05-18T00:00:00"
   },
   "customers": {
-    "created_at": "2050-05-18T00:00:00"
+    "updated_at": "2050-05-18T00:00:00"
   },
   "discounts": {
-    "created_at": "2050-05-18T00:00:00"
+    "updated_at": "2050-05-18T00:00:00"
   },
   "onetimes": {
-    "created_at": "2050-05-18T00:00:00"
+    "updated_at": "2050-05-18T00:00:00"
   },
   "orders": {
-    "created_at": "2050-05-18T00:00:00"
+    "updated_at": "2050-05-18T00:00:00"
   },
   "subscriptions": {
-    "created_at": "2050-05-18T00:00:00"
+    "updated_at": "2050-05-18T00:00:00"
   }
 }
diff --git a/airbyte-integrations/connectors/source-recharge/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-recharge/integration_tests/configured_catalog.json
@@ -6,25 +6,25 @@
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["created_at"],
+        "default_cursor_field": ["updated_at"],
         "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append",
-      "cursor_field": ["created_at"]
+      "cursor_field": ["updated_at"]
     },
     {
       "stream": {
         "name": "charges",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["created_at"],
+        "default_cursor_field": ["updated_at"],
         "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append",
-      "cursor_field": ["created_at"]
+      "cursor_field": ["updated_at"]
     },
     {
       "stream": {
@@ -42,25 +42,25 @@
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["created_at"],
+        "default_cursor_field": ["updated_at"],
         "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append",
-      "cursor_field": ["created_at"]
+      "cursor_field": ["updated_at"]
     },
     {
       "stream": {
         "name": "discounts",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["created_at"],
+        "default_cursor_field": ["updated_at"],
         "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append",
-      "cursor_field": ["created_at"]
+      "cursor_field": ["updated_at"]
     },
     {
       "stream": {
@@ -78,25 +78,25 @@
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["created_at"],
+        "default_cursor_field": ["updated_at"],
         "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append",
-      "cursor_field": ["created_at"]
+      "cursor_field": ["updated_at"]
     },
     {
       "stream": {
         "name": "orders",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["created_at"],
+        "default_cursor_field": ["updated_at"],
         "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append",
-      "cursor_field": ["created_at"]
+      "cursor_field": ["updated_at"]
     },
     {
       "stream": {
@@ -124,12 +124,12 @@
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["created_at"],
+        "default_cursor_field": ["updated_at"],
         "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append",
-      "cursor_field": ["created_at"]
+      "cursor_field": ["updated_at"]
     }
   ]
 }
diff --git a/...ons/connectors/source-recharge/integration_tests/streams_with_output_records_catalog.json b/...ons/connectors/source-recharge/integration_tests/streams_with_output_records_catalog.json
@@ -6,25 +6,25 @@
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["created_at"],
+        "default_cursor_field": ["updated_at"],
         "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append",
-      "cursor_field": ["created_at"]
+      "cursor_field": ["updated_at"]
     },
     {
       "stream": {
         "name": "charges",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["created_at"],
+        "default_cursor_field": ["updated_at"],
         "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append",
-      "cursor_field": ["created_at"]
+      "cursor_field": ["updated_at"]
     },
     {
       "stream": {
@@ -42,38 +42,38 @@
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["created_at"],
+        "default_cursor_field": ["updated_at"],
         "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append",
-      "cursor_field": ["created_at"]
+      "cursor_field": ["updated_at"]
     },
     {
       "stream": {
         "name": "discounts",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["created_at"],
+        "default_cursor_field": ["updated_at"],
         "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append",
-      "cursor_field": ["created_at"]
+      "cursor_field": ["updated_at"]
     },
     {
       "stream": {
         "name": "orders",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["created_at"],
+        "default_cursor_field": ["updated_at"],
         "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append",
-      "cursor_field": ["created_at"]
+      "cursor_field": ["updated_at"]
     },
     {
       "stream": {
@@ -101,12 +101,12 @@
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["created_at"],
+        "default_cursor_field": ["updated_at"],
         "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append",
-      "cursor_field": ["created_at"]
+      "cursor_field": ["updated_at"]
     }
   ]
 }
diff --git a/airbyte-integrations/connectors/source-recharge/requirements.txt b/airbyte-integrations/connectors/source-recharge/requirements.txt
@@ -0,0 +1,2 @@
+-e ../../bases/source-acceptance-test
+-e .
diff --git a/airbyte-integrations/connectors/source-recharge/sample_files/sample_state.json b/airbyte-integrations/connectors/source-recharge/sample_files/sample_state.json
@@ -1,23 +1,23 @@
 {
   "addresses": {
-    "created_at": "2021-04-02T00:00:00"
+    "updated_at": "2021-04-02T00:00:00"
   },
   "charges": {
-    "created_at": "2021-04-02T00:00:00"
+    "updated_at": "2021-04-02T00:00:00"
   },
   "customers": {
-    "created_at": "2021-04-02T00:00:00"
+    "updated_at": "2021-04-02T00:00:00"
   },
   "discounts": {
-    "created_at": "2021-04-02T00:00:00"
+    "updated_at": "2021-04-02T00:00:00"
   },
   "onetimes": {
-    "created_at": "2021-04-02T00:00:00"
+    "updated_at": "2021-04-02T00:00:00"
   },
   "orders": {
-    "created_at": "2021-04-02T00:00:00"
+    "updated_at": "2021-04-02T00:00:00"
   },
   "subscriptions": {
-    "created_at": "2021-04-02T00:00:00"
+    "updated_at": "2021-04-02T00:00:00"
   }
 }
diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py
@@ -79,7 +79,7 @@ def get_stream_data(self, response_data: Any) -> List[dict]:
 
 class IncrementalRechargeStream(RechargeStream, ABC):
 
-    cursor_field = "created_at"
+    cursor_field = "updated_at"
 
     def __init__(self, start_date, **kwargs):
         super().__init__(**kwargs)

diff --git a/docs/integrations/sources/recharge.md b/docs/integrations/sources/recharge.md
@@ -46,3 +46,9 @@ The Recharge connector should gracefully handle Recharge API limitations under n
 
 Please read [How to generate your API token](https://support.rechargepayments.com/hc/en-us/articles/360008829993-ReCharge-API-).
 
+## Changelog
+
+| Version | Date       | Pull Request | Subject |
+| :------ | :--------  | :-----       | :------ |
+| 0.1.2  | 2021-09-17 | [6149](https://github.com/airbytehq/airbyte/pull/6149) | Change `cursor_field` for Incremental streams |
+|