From b3bf7851c342f9450889329375fa5928f7766dad Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Sat, 19 Mar 2022 03:25:17 +0200 Subject: [PATCH] updated after review --- .../source_zendesk_support/streams.py | 94 ++++++++----------- .../unit_tests/test_other.py | 48 ++++++++++ docs/integrations/sources/zendesk-support.md | 29 +++--- 3 files changed, 103 insertions(+), 68 deletions(-) create mode 100644 airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_other.py diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py index ab261958717e7..c35f75f72425d 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py @@ -115,6 +115,12 @@ def str2unixtime(str_dt: str) -> Optional[int]: dt = datetime.strptime(str_dt, DATETIME_FORMAT) return calendar.timegm(dt.utctimetuple()) + @staticmethod + def _parse_next_page_number(response: requests.Response) -> Optional[int]: + """Parses a response and tries to find next page number""" + next_page = response.json().get("next_page") + return dict(parse_qsl(urlparse(next_page).query)).get("page") if next_page else None + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: """try to select relevant data only""" @@ -150,6 +156,16 @@ def __init__(self, authenticator: Union[AuthBase, HttpAuthenticator] = None, **k self._session.auth = authenticator self.future_requests = deque() + @property + def url_base(self) -> str: + return f"https://{self._subdomain}.zendesk.com/api/v2/" + + def path(self, **kwargs): + return self.name + + def next_page_token(self, *args, **kwargs): + return None + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: latest_benchmark = latest_record[self.cursor_field] if current_stream_state.get(self.cursor_field): @@ -271,24 +287,6 @@ def read_records( else: yield from self.parse_response(response, stream_state=stream_state, stream_slice=stream_slice) - @property - def url_base(self) -> str: - return f"https://{self._subdomain}.zendesk.com/api/v2/" - - @staticmethod - def _parse_next_page_number(response: requests.Response) -> Optional[int]: - """Parses a response and tries to find next page number""" - next_page = response.json().get("next_page") - if next_page: - return dict(parse_qsl(urlparse(next_page).query)).get("page") - return None - - def path(self, **kwargs): - return self.name - - def next_page_token(self, *args, **kwargs): - return None - class SourceZendeskSupportFullRefreshStream(BaseSourceZendeskSupportStream): """ @@ -306,14 +304,6 @@ def url_base(self) -> str: def path(self, **kwargs): return self.name - @staticmethod - def _parse_next_page_number(response: requests.Response) -> Optional[int]: - """Parses a response and tries to find next page number""" - next_page = response.json().get("next_page") - if next_page: - return dict(parse_qsl(urlparse(next_page).query)).get("page") - return None - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: next_page = self._parse_next_page_number(response) if not next_page: @@ -369,32 +359,39 @@ def request_params( return params -class ZendeskSupportTicektEventsExportStream(SourceZendeskSupportCursorPaginationStream): +class ZendeskSupportTicketEventsExportStream(SourceZendeskSupportCursorPaginationStream): """Incremental Export from TicketEvents stream: https://developer.zendesk.com/api-reference/ticketing/ticket-management/incremental_exports/#incremental-ticket-event-export - + @ param response_list_name: the main nested entity to look at inside of response, defualt = "ticket_events" - @ param responce_target_entity: nested property inside of `response_list_name`, default = "child_events" + @ param response_target_entity: nested property inside of `response_list_name`, default = "child_events" @ param list_entities_from_event : the list of nested child_events entities to include from parent record @ param sideload_param : parameter variable to include various information to child_events property more info: https://developer.zendesk.com/documentation/ticketing/using-the-zendesk-api/side_loading/#supported-endpoints @ param event_type : specific event_type to check ["Audit", "Change", "Comment", etc] """ - + response_list_name: str = "ticket_events" - responce_target_entity: str = "child_events" + response_target_entity: str = "child_events" list_entities_from_event: List[str] = None sideload_param: str = None event_type: str = None - + + @property + def update_event_from_record(self) -> bool: + """Returns True/False based on list_entities_from_event property""" + return True if len(self.list_entities_from_event) > 0 else False + + def path(self, **kwargs) -> str: + return "incremental/ticket_events" + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: """ Returns next_page_token based on `end_of_stream` parameter inside of response """ next_page_token = super().next_page_token(response) - end_of_stream = response.json().get(END_OF_STREAM_KEY, False) - return None if end_of_stream else next_page_token - + return None if response.json().get(END_OF_STREAM_KEY, False) else next_page_token + def request_params( self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs ) -> MutableMapping[str, Any]: @@ -402,21 +399,15 @@ def request_params( if self.sideload_param: params["include"] = self.sideload_param return params - - def update_event_props(self, record: dict = None, event: dict = None, props: list = None) -> MutableMapping[str, Any]: - """Update the event mapping with the specified fields from record entity""" - if self.list_entities_from_event and len(self.list_entities_from_event) > 0: - for prop in props: - target_prop = record.get(prop) - event[prop] = target_prop if target_prop else None - return event - - def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: - records = response.json().get(self.response_list_name) or [] - for record in records: - for event in record.get(self.responce_target_entity): + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + for record in response.json().get(self.response_list_name, []): + for event in record.get(self.response_target_entity, []): if event.get("event_type") == self.event_type: - yield self.update_event_props(record, event, self.list_entities_from_event) + if self.update_event_from_record: + for prop in self.list_entities_from_event: + event[prop] = record.get(prop) + yield event class Users(SourceZendeskSupportStream): @@ -442,7 +433,7 @@ def request_params(self, **kwargs) -> MutableMapping[str, Any]: return params -class TicketComments(ZendeskSupportTicektEventsExportStream): +class TicketComments(ZendeskSupportTicketEventsExportStream): """ Fetch the TicketComments incrementaly from TicketEvents Export stream """ @@ -452,9 +443,6 @@ class TicketComments(ZendeskSupportTicektEventsExportStream): sideload_param = "comment_events" event_type = "Comment" - def path(self, **kwargs) -> str: - return "incremental/ticket_events" - class Groups(SourceZendeskSupportStream): """Groups stream: https://developer.zendesk.com/api-reference/ticketing/groups/groups/""" diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_other.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_other.py new file mode 100644 index 0000000000000..367973ff79b87 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_other.py @@ -0,0 +1,48 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import calendar +from datetime import datetime +from urllib.parse import parse_qsl, urlparse + +import pytz +import requests +from source_zendesk_support.streams import DATETIME_FORMAT, BaseSourceZendeskSupportStream + +DATETIME_STR = "2021-07-22T06:55:55Z" +DATETIME_FROM_STR = datetime.strptime(DATETIME_STR, DATETIME_FORMAT) +STREAM_URL = "https://subdomain.zendesk.com/api/v2/stream.json?&start_time=1647532987&page=1" +STREAM_RESPONSE: dict = { + "data": [], + "next_page": "https://subdomain.zendesk.com/api/v2/stream.json?&start_time=1647532987&page=2", + "count": 215, + "end_of_stream": True, + "end_time": 1647532987, +} + + +def test_str2datetime(): + expected = datetime.strptime(DATETIME_STR, DATETIME_FORMAT) + output = BaseSourceZendeskSupportStream.str2datetime(DATETIME_STR) + assert output == expected + + +def test_datetime2str(): + expected = datetime.strftime(DATETIME_FROM_STR.replace(tzinfo=pytz.UTC), DATETIME_FORMAT) + output = BaseSourceZendeskSupportStream.datetime2str(DATETIME_FROM_STR) + assert output == expected + + +def test_str2unixtime(): + expected = calendar.timegm(DATETIME_FROM_STR.utctimetuple()) + output = BaseSourceZendeskSupportStream.str2unixtime(DATETIME_STR) + assert output == expected + + +def test_parse_next_page_number(requests_mock): + expected = dict(parse_qsl(urlparse(STREAM_RESPONSE.get("next_page")).query)).get("page") + requests_mock.get(STREAM_URL, json=STREAM_RESPONSE) + test_response = requests.get(STREAM_URL) + output = BaseSourceZendeskSupportStream._parse_next_page_number(test_response) + assert output == expected diff --git a/docs/integrations/sources/zendesk-support.md b/docs/integrations/sources/zendesk-support.md index 2d89157539dba..0cd4228a0754b 100644 --- a/docs/integrations/sources/zendesk-support.md +++ b/docs/integrations/sources/zendesk-support.md @@ -10,27 +10,26 @@ This source can sync data for the [Zendesk Support API](https://developer.zendes This Source is capable of syncing the following core Streams: -* [Tickets](https://developer.zendesk.com/rest_api/docs/support/tickets) +* [Brands](https://developer.zendesk.com/api-reference/ticketing/account-configuration/brands/#list-brands) +* [Custom Roles](https://developer.zendesk.com/api-reference/ticketing/account-configuration/custom_roles/#list-custom-roles) * [Groups](https://developer.zendesk.com/rest_api/docs/support/groups) -* [Users](https://developer.zendesk.com/rest_api/docs/support/users) +* [Group Memberships](https://developer.zendesk.com/rest_api/docs/support/group_memberships) +* [Macros](https://developer.zendesk.com/rest_api/docs/support/macros) * [Organizations](https://developer.zendesk.com/rest_api/docs/support/organizations) +* [Satisfaction Ratings](https://developer.zendesk.com/rest_api/docs/support/satisfaction_ratings) +* [Schedules](https://developer.zendesk.com/api-reference/ticketing/ticket-management/schedules/#list-schedules) +* [SLA Policies](https://developer.zendesk.com/rest_api/docs/support/sla_policies) +* [Tags](https://developer.zendesk.com/rest_api/docs/support/tags) +* [Tickets](https://developer.zendesk.com/rest_api/docs/support/tickets) * [Ticket Audits](https://developer.zendesk.com/rest_api/docs/support/ticket_audits) * [Ticket Comments](https://developer.zendesk.com/api-reference/ticketing/ticket-management/incremental_exports/#incremental-ticket-event-export) * [Ticket Fields](https://developer.zendesk.com/rest_api/docs/support/ticket_fields) * [Ticket Forms](https://developer.zendesk.com/rest_api/docs/support/ticket_forms) * [Ticket Metrics](https://developer.zendesk.com/rest_api/docs/support/ticket_metrics) * [Ticket Metric Events](https://developer.zendesk.com/api-reference/ticketing/tickets/ticket_metric_events/) -* [Group Memberships](https://developer.zendesk.com/rest_api/docs/support/group_memberships) -* [Macros](https://developer.zendesk.com/rest_api/docs/support/macros) -* [Satisfaction Ratings](https://developer.zendesk.com/rest_api/docs/support/satisfaction_ratings) -* [Tags](https://developer.zendesk.com/rest_api/docs/support/tags) -* [SLA Policies](https://developer.zendesk.com/rest_api/docs/support/sla_policies) -* [Brands](https://developer.zendesk.com/api-reference/ticketing/account-configuration/brands/#list-brands) -* [Custom Roles](https://developer.zendesk.com/api-reference/ticketing/account-configuration/custom_roles/#list-custom-roles) -* [Schedules](https://developer.zendesk.com/api-reference/ticketing/ticket-management/schedules/#list-schedules) - +* [Users](https://developer.zendesk.com/rest_api/docs/support/users) -There are a lot of space for future work, the next streams could be added in the future: +The streams below are not implemented. Please open a Github issue or request it through Airbyte Cloud's support box if you are interested in them. **Tickets** @@ -67,10 +66,10 @@ There are a lot of space for future work, the next streams could be added in the | Feature | Supported?\(Yes/No\) | Notes | | :--- | :--- | :--- | -| Full Refresh Sync | Yes | ... | -| Incremental - Append Sync | Yes | ... | +| Full Refresh Sync | Yes | | +| Incremental - Append Sync | Yes | | | Incremental - Debuped + History Sync | Yes | Enabled according to type of destination | -| Namespaces | No | ... | +| Namespaces | No | | ### Performance considerations