🐛 Source Zendesk Support: fix incremental logic for ticket_comments stream

airbyte-integrations/connectors/source-zendesk-support/integration_tests/integration_test.py

@@ -46,16 +46,19 @@ def prepare_stream_args():
 
     def _test_export_stream(self, stream_cls: type):
         stream = stream_cls(**self.prepare_stream_args())
+        stream.page_size = 1
         record_timestamps = {}
         for record in stream.read_records(sync_mode=None):
             # save the first 5 records
             if len(record_timestamps) > 5:
                 break
-            record_timestamps[record["id"]] = record[stream.cursor_field]
+            if stream._last_end_time not in record_timestamps.values():
+                record_timestamps[record["id"]] = stream._last_end_time
+
+        stream.page_size = 10
         for record_id, timestamp in record_timestamps.items():
-            state = {stream.cursor_field: timestamp}
+            state = {"_last_end_time": timestamp}
             for record in stream.read_records(sync_mode=None, stream_state=state):
-                print(f"First record: {record}!={record_id} ({timestamp}), state:{state}")
                 assert record["id"] != record_id
                 break
 

airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_metrics.json

@@ -131,7 +131,8 @@
       "type": ["null", "string"]
     },
     "initially_assigned_at": {
-      "type": ["null", "string"]
+      "type": ["null", "string"],
+      "format": "datetime"
     },
     "assigned_at": {
       "type": ["null", "string"],

airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py

@@ -80,13 +80,13 @@ def backoff_time(self, response: requests.Response) -> Union[int, float]:
         retry_after = int(response.headers.get("Retry-After", 0))
         if retry_after and retry_after > 0:
             return int(retry_after)
+
         # the header X-Rate-Limit returns a amount of requests per minute
         # we try to wait twice as long
-
-        rate_limit = float(response.headers.get("X-Rate-Limit", 2))
+        rate_limit = float(response.headers.get("X-Rate-Limit", 0))
         if rate_limit and rate_limit > 0:
             return (60.0 / rate_limit) * 2
-        return 60
+        return super().backoff_time(response)
 
     @staticmethod
     def str2datetime(str_dt: str) -> datetime:
@@ -218,7 +218,6 @@ def request_params(
                 # try to search all reconds with generated_timestamp > start_time
                 current_state = stream_state.get(self.cursor_field)
                 if current_state and isinstance(current_state, str) and not current_state.isdigit():
-                    # try to save a stage with UnixTime format
                     current_state = self.str2unixtime(current_state)
 
             start_time = int(current_state or time.mktime(self._start_date.timetuple()))

airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py

@@ -49,22 +49,22 @@ def test_backoff_cases(self):
         """Zendesk sends the header different value for backoff logic"""
 
         stream = Tags(**self.prepare_stream_args())
-        default_timeout = 60
+        default_timeout = None
         with requests_mock.Mocker() as m:
             url = stream.url_base + stream.path()
 
             # with the Retry-After header > 0
-            m.get(url, headers={"Retry-After": str(123)})
+            m.get(url, headers={"Retry-After": str(123)}, status_code=429)
             assert stream.backoff_time(requests.get(url)) == 123
             # with the Retry-After header < 0,  must return a default value
-            m.get(url, headers={"Retry-After": str(-123)})
+            m.get(url, headers={"Retry-After": str(-123)}, status_code=429)
             assert stream.backoff_time(requests.get(url)) == default_timeout
 
             # with the Retry-After header > 0
-            m.get(url, headers={"X-Rate-Limit": str(100)})
+            m.get(url, headers={"X-Rate-Limit": str(100)}, status_code=429)
             assert (stream.backoff_time(requests.get(url)) - 1.2) < 0.0005
             # with the Retry-After header < 0,  must return a default value
-            m.get(url, headers={"X-Rate-Limit": str(-100)})
+            m.get(url, headers={"X-Rate-Limit": str(-100)}, status_code=429)
             assert stream.backoff_time(requests.get(url)) == default_timeout
 
             # without rate headers