updated after review

airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py

@@ -115,6 +115,12 @@ def str2unixtime(str_dt: str) -> Optional[int]:
         dt = datetime.strptime(str_dt, DATETIME_FORMAT)
         return calendar.timegm(dt.utctimetuple())
 
+    @staticmethod
+    def _parse_next_page_number(response: requests.Response) -> Optional[int]:
+        """Parses a response and tries to find next page number"""
+        next_page = response.json().get("next_page")
+        return dict(parse_qsl(urlparse(next_page).query)).get("page") if next_page else None
+
     def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]:
         """try to select relevant data only"""
 
@@ -150,6 +156,16 @@ def __init__(self, authenticator: Union[AuthBase, HttpAuthenticator] = None, **k
         self._session.auth = authenticator
         self.future_requests = deque()
 
+    @property
+    def url_base(self) -> str:
+        return f"https://{self._subdomain}.zendesk.com/api/v2/"
+
+    def path(self, **kwargs):
+        return self.name
+
+    def next_page_token(self, *args, **kwargs):
+        return None
+
     def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]:
         latest_benchmark = latest_record[self.cursor_field]
         if current_stream_state.get(self.cursor_field):
@@ -271,24 +287,6 @@ def read_records(
             else:
                 yield from self.parse_response(response, stream_state=stream_state, stream_slice=stream_slice)
 
-    @property
-    def url_base(self) -> str:
-        return f"https://{self._subdomain}.zendesk.com/api/v2/"
-
-    @staticmethod
-    def _parse_next_page_number(response: requests.Response) -> Optional[int]:
-        """Parses a response and tries to find next page number"""
-        next_page = response.json().get("next_page")
-        if next_page:
-            return dict(parse_qsl(urlparse(next_page).query)).get("page")
-        return None
-
-    def path(self, **kwargs):
-        return self.name
-
-    def next_page_token(self, *args, **kwargs):
-        return None
-
 
 class SourceZendeskSupportFullRefreshStream(BaseSourceZendeskSupportStream):
     """
@@ -306,14 +304,6 @@ def url_base(self) -> str:
     def path(self, **kwargs):
         return self.name
 
-    @staticmethod
-    def _parse_next_page_number(response: requests.Response) -> Optional[int]:
-        """Parses a response and tries to find next page number"""
-        next_page = response.json().get("next_page")
-        if next_page:
-            return dict(parse_qsl(urlparse(next_page).query)).get("page")
-        return None
-
     def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
         next_page = self._parse_next_page_number(response)
         if not next_page:
@@ -369,54 +359,55 @@ def request_params(
         return params
 
 
-class ZendeskSupportTicektEventsExportStream(SourceZendeskSupportCursorPaginationStream):
+class ZendeskSupportTicketEventsExportStream(SourceZendeskSupportCursorPaginationStream):
     """Incremental Export from TicketEvents stream:
     https://developer.zendesk.com/api-reference/ticketing/ticket-management/incremental_exports/#incremental-ticket-event-export
-    
+
     @ param response_list_name: the main nested entity to look at inside of response, defualt = "ticket_events"
-    @ param responce_target_entity: nested property inside of `response_list_name`, default = "child_events"
+    @ param response_target_entity: nested property inside of `response_list_name`, default = "child_events"
     @ param list_entities_from_event : the list of nested child_events entities to include from parent record
     @ param sideload_param : parameter variable to include various information to child_events property
         more info: https://developer.zendesk.com/documentation/ticketing/using-the-zendesk-api/side_loading/#supported-endpoints
     @ param event_type : specific event_type to check ["Audit", "Change", "Comment", etc]
     """
-    
+
     response_list_name: str = "ticket_events"
-    responce_target_entity: str = "child_events"
+    response_target_entity: str = "child_events"
     list_entities_from_event: List[str] = None
     sideload_param: str = None
     event_type: str = None
-
+
+    @property
+    def update_event_from_record(self) -> bool:
+        """Returns True/False based on list_entities_from_event property"""
+        return True if len(self.list_entities_from_event) > 0 else False
+
+    def path(self, **kwargs) -> str:
+        return "incremental/ticket_events"
+
     def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
         """
         Returns next_page_token based on `end_of_stream` parameter inside of response
         """
         next_page_token = super().next_page_token(response)
-        end_of_stream = response.json().get(END_OF_STREAM_KEY, False)
-        return None if end_of_stream else next_page_token
-
+        return None if response.json().get(END_OF_STREAM_KEY, False) else next_page_token
+
     def request_params(
         self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs
     ) -> MutableMapping[str, Any]:
         params = super().request_params(stream_state, next_page_token, **kwargs)
         if self.sideload_param:
             params["include"] = self.sideload_param
         return params
-
-    def update_event_props(self, record: dict = None, event: dict = None, props: list = None) -> MutableMapping[str, Any]:
-        """Update the event mapping with the specified fields from record entity"""
-        if self.list_entities_from_event and len(self.list_entities_from_event) > 0:
-            for prop in props:
-                target_prop = record.get(prop)
-                event[prop] = target_prop if target_prop else None
-        return event
-
-    def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]:
-        records = response.json().get(self.response_list_name) or []
-        for record in records:
-            for event in record.get(self.responce_target_entity):
+
+    def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
+        for record in response.json().get(self.response_list_name, []):
+            for event in record.get(self.response_target_entity, []):
                 if event.get("event_type") == self.event_type:
-                    yield self.update_event_props(record, event, self.list_entities_from_event)
+                    if self.update_event_from_record:
+                        for prop in self.list_entities_from_event:
+                            event[prop] = record.get(prop)
+                    yield event
 
 
 class Users(SourceZendeskSupportStream):
@@ -442,7 +433,7 @@ def request_params(self, **kwargs) -> MutableMapping[str, Any]:
         return params
 
 
-class TicketComments(ZendeskSupportTicektEventsExportStream):
+class TicketComments(ZendeskSupportTicketEventsExportStream):
     """
     Fetch the TicketComments incrementaly from TicketEvents Export stream
     """
@@ -452,9 +443,6 @@ class TicketComments(ZendeskSupportTicektEventsExportStream):
     sideload_param = "comment_events"
     event_type = "Comment"
 
-    def path(self, **kwargs) -> str:
-        return "incremental/ticket_events"
-
 
 class Groups(SourceZendeskSupportStream):
     """Groups stream: https://developer.zendesk.com/api-reference/ticketing/groups/groups/"""

airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_other.py

@@ -0,0 +1,48 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+
+import calendar
+from datetime import datetime
+from urllib.parse import parse_qsl, urlparse
+
+import pytz
+import requests
+from source_zendesk_support.streams import DATETIME_FORMAT, BaseSourceZendeskSupportStream
+
+DATETIME_STR = "2021-07-22T06:55:55Z"
+DATETIME_FROM_STR = datetime.strptime(DATETIME_STR, DATETIME_FORMAT)
+STREAM_URL = "https://subdomain.zendesk.com/api/v2/stream.json?&start_time=1647532987&page=1"
+STREAM_RESPONSE: dict = {
+    "data": [],
+    "next_page": "https://subdomain.zendesk.com/api/v2/stream.json?&start_time=1647532987&page=2",
+    "count": 215,
+    "end_of_stream": True,
+    "end_time": 1647532987,
+}
+
+
+def test_str2datetime():
+    expected = datetime.strptime(DATETIME_STR, DATETIME_FORMAT)
+    output = BaseSourceZendeskSupportStream.str2datetime(DATETIME_STR)
+    assert output == expected
+
+
+def test_datetime2str():
+    expected = datetime.strftime(DATETIME_FROM_STR.replace(tzinfo=pytz.UTC), DATETIME_FORMAT)
+    output = BaseSourceZendeskSupportStream.datetime2str(DATETIME_FROM_STR)
+    assert output == expected
+
+
+def test_str2unixtime():
+    expected = calendar.timegm(DATETIME_FROM_STR.utctimetuple())
+    output = BaseSourceZendeskSupportStream.str2unixtime(DATETIME_STR)
+    assert output == expected
+
+
+def test_parse_next_page_number(requests_mock):
+    expected = dict(parse_qsl(urlparse(STREAM_RESPONSE.get("next_page")).query)).get("page")
+    requests_mock.get(STREAM_URL, json=STREAM_RESPONSE)
+    test_response = requests.get(STREAM_URL)
+    output = BaseSourceZendeskSupportStream._parse_next_page_number(test_response)
+    assert output == expected

docs/integrations/sources/zendesk-support.md

@@ -10,27 +10,26 @@ This source can sync data for the [Zendesk Support API](https://developer.zendes
 
 This Source is capable of syncing the following core Streams:
 
-* [Tickets](https://developer.zendesk.com/rest_api/docs/support/tickets)
+* [Brands](https://developer.zendesk.com/api-reference/ticketing/account-configuration/brands/#list-brands)
+* [Custom Roles](https://developer.zendesk.com/api-reference/ticketing/account-configuration/custom_roles/#list-custom-roles)
 * [Groups](https://developer.zendesk.com/rest_api/docs/support/groups)
-* [Users](https://developer.zendesk.com/rest_api/docs/support/users)
+* [Group Memberships](https://developer.zendesk.com/rest_api/docs/support/group_memberships)
+* [Macros](https://developer.zendesk.com/rest_api/docs/support/macros)
 * [Organizations](https://developer.zendesk.com/rest_api/docs/support/organizations)
+* [Satisfaction Ratings](https://developer.zendesk.com/rest_api/docs/support/satisfaction_ratings)
+* [Schedules](https://developer.zendesk.com/api-reference/ticketing/ticket-management/schedules/#list-schedules)
+* [SLA Policies](https://developer.zendesk.com/rest_api/docs/support/sla_policies)
+* [Tags](https://developer.zendesk.com/rest_api/docs/support/tags)
+* [Tickets](https://developer.zendesk.com/rest_api/docs/support/tickets)
 * [Ticket Audits](https://developer.zendesk.com/rest_api/docs/support/ticket_audits)
 * [Ticket Comments](https://developer.zendesk.com/api-reference/ticketing/ticket-management/incremental_exports/#incremental-ticket-event-export)
 * [Ticket Fields](https://developer.zendesk.com/rest_api/docs/support/ticket_fields)
 * [Ticket Forms](https://developer.zendesk.com/rest_api/docs/support/ticket_forms)
 * [Ticket Metrics](https://developer.zendesk.com/rest_api/docs/support/ticket_metrics)
 * [Ticket Metric Events](https://developer.zendesk.com/api-reference/ticketing/tickets/ticket_metric_events/)
-* [Group Memberships](https://developer.zendesk.com/rest_api/docs/support/group_memberships)
-* [Macros](https://developer.zendesk.com/rest_api/docs/support/macros)
-* [Satisfaction Ratings](https://developer.zendesk.com/rest_api/docs/support/satisfaction_ratings)
-* [Tags](https://developer.zendesk.com/rest_api/docs/support/tags)
-* [SLA Policies](https://developer.zendesk.com/rest_api/docs/support/sla_policies)
-* [Brands](https://developer.zendesk.com/api-reference/ticketing/account-configuration/brands/#list-brands)
-* [Custom Roles](https://developer.zendesk.com/api-reference/ticketing/account-configuration/custom_roles/#list-custom-roles)
-* [Schedules](https://developer.zendesk.com/api-reference/ticketing/ticket-management/schedules/#list-schedules)
-
+* [Users](https://developer.zendesk.com/rest_api/docs/support/users)
 
-There are a lot of space for future work, the next streams could be added in the future:
+The streams below are not implemented. Please open a Github issue or request it through Airbyte Cloud's support box if you are interested in them.
 
 **Tickets**
 
@@ -67,10 +66,10 @@ There are a lot of space for future work, the next streams could be added in the
 
 | Feature | Supported?\(Yes/No\) | Notes |
 | :--- | :--- | :--- |
-| Full Refresh Sync | Yes | ... |
-| Incremental - Append Sync | Yes | ... |
+| Full Refresh Sync | Yes |  |
+| Incremental - Append Sync | Yes |  |
 | Incremental - Debuped + History Sync | Yes | Enabled according to type of destination |
-| Namespaces | No | ... |
+| Namespaces | No |  |
 
 ### Performance considerations