Source Zendesk Support: add stream Deleted Tickets

airbyte-integrations/connectors/source-zendesk-support/Dockerfile

@@ -25,5 +25,5 @@ COPY source_zendesk_support ./source_zendesk_support
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=1.6.0
+LABEL io.airbyte.version=1.7.0
 LABEL io.airbyte.name=airbyte/source-zendesk-support

airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl

@@ -13,6 +13,7 @@
 {"stream": "audit_logs", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/audit_logs/7502393054223.json", "id": 7502393054223, "action_label": "Signed in", "actor_id": 360786799676, "source_id": 360786799676, "source_type": "user", "source_label": "Team member: Team Airbyte", "action": "login", "change_description": "Successful sign-in using Zendesk password from https://d3v-airbyte.zendesk.com/access/login", "ip_address": "109.86.166.58", "created_at": "2023-07-24T10:56:28Z", "actor_name": "Team Airbyte"}, "emitted_at": 1690888150345}
 {"stream": "audit_logs", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/audit_logs/7465455408271.json", "id": 7465455408271, "action_label": "Signed in", "actor_id": 360786799676, "source_id": 360786799676, "source_type": "user", "source_label": "Team member: Team Airbyte", "action": "login", "change_description": "Successful sign-in using Zendesk password from https://d3v-airbyte.zendesk.com/access/login", "ip_address": "109.86.166.58", "created_at": "2023-07-21T08:03:28Z", "actor_name": "Team Airbyte"}, "emitted_at": 1690888150346}
 {"stream": "audit_logs", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/audit_logs/7453133196303.json", "id": 7453133196303, "action_label": "Signed in", "actor_id": 360786799676, "source_id": 360786799676, "source_type": "user", "source_label": "Team member: Team Airbyte", "action": "login", "change_description": "Successful sign-in using Zendesk password from https://d3v-airbyte.zendesk.com/access/login", "ip_address": "136.24.229.166", "created_at": "2023-07-19T19:09:32Z", "actor_name": "Team Airbyte"}, "emitted_at": 1690888150346}
+{"stream":"deleted_tickets","data":{"id":123,"subject":"Voicemail from: Caller +1 (607) 210-9549","description":"Call from: +1 (607) 210-9549\\nTime of call: July 13, 2022 at 2:33:27 PM","actor":{"id":360786799676,"name":"Team Airbyte"},"deleted_at":"2023-09-07T16:46:02Z","previous_state":"new"},"emitted_at":1694110102400}
 {"stream": "group_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/group_memberships/360007820916.json", "id": 360007820916, "user_id": 360786799676, "group_id": 360003074836, "default": true, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2020-12-11T18:34:05Z"}, "emitted_at": 1690888151470}
 {"stream": "group_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/group_memberships/360011727976.json", "id": 360011727976, "user_id": 361084605116, "group_id": 360003074836, "default": true, "created_at": "2021-04-23T14:33:11Z", "updated_at": "2021-04-23T14:33:11Z"}, "emitted_at": 1690888151471}
 {"stream": "group_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/group_memberships/360011812655.json", "id": 360011812655, "user_id": 361089721035, "group_id": 360003074836, "default": true, "created_at": "2021-04-23T14:34:20Z", "updated_at": "2021-04-23T14:34:20Z"}, "emitted_at": 1690888151471}
@@ -54,7 +55,6 @@
 {"stream": "ticket_skips", "data": {"id": 7290088475023, "ticket_id": 125, "user_id": 360786799676, "reason": "Another test skip.", "created_at": "2023-06-27T08:30:01Z", "updated_at": "2023-06-27T08:30:01Z", "ticket": {"url": "https://d3v-airbyte.zendesk.com/api/v2/tickets/125.json", "id": 125, "external_id": null, "via": {"channel": "web", "source": {"from": {}, "to": {}, "rel": null}}, "created_at": "2022-07-18T10:16:53Z", "updated_at": "2022-07-18T10:36:02Z", "type": "question", "subject": "Ticket Test 2", "raw_subject": "Ticket Test 2", "description": "238473846", "priority": "urgent", "status": "open", "recipient": null, "requester_id": 360786799676, "submitter_id": 360786799676, "assignee_id": 361089721035, "organization_id": 360033549136, "group_id": 5059439464079, "collaborator_ids": [360786799676], "follower_ids": [360786799676], "email_cc_ids": [], "forum_topic_id": null, "problem_id": null, "has_incidents": false, "is_public": false, "due_at": null, "tags": [], "custom_fields": [], "satisfaction_rating": {"score": "unoffered"}, "sharing_agreement_ids": [], "custom_status_id": 4044376, "fields": [], "followup_ids": [], "ticket_form_id": 360000084116, "deleted_ticket_form_id": null, "brand_id": 360000358316, "allow_channelback": false, "allow_attachments": true, "from_messaging_channel": false}}, "emitted_at": 1690888182192}
 {"stream":"tickets","data":{"url":"https://d3v-airbyte.zendesk.com/api/v2/tickets/121.json","id":121,"external_id":null,"via":{"channel":"voice","source":{"rel":"voicemail","from":{"formatted_phone":"+1 (689) 689-8023","phone":"+16896898023","name":"Caller +1 (689) 689-8023"},"to":{"formatted_phone":"+1 (205) 953-1462","phone":"+12059531462","name":"Airbyte","brand_id":360000358316}}},"created_at":"2022-06-17T14:49:20Z","updated_at":"2022-06-17T16:01:42Z","type":null,"subject":"Voicemail from: Caller +1 (689) 689-8023","raw_subject":"Voicemail from: Caller +1 (689) 689-8023","description":"Call from: +1 (689) 689-8023\\nTime of call: June 17, 2022 at 2:48:27 PM","priority":null,"status":"new","recipient":null,"requester_id":4992781783439,"submitter_id":4992781783439,"assignee_id":null,"organization_id":null,"group_id":null,"collaborator_ids":[],"follower_ids":[],"email_cc_ids":[],"forum_topic_id":null,"problem_id":null,"has_incidents":false,"is_public":false,"due_at":null,"tags":[],"custom_fields":[],"satisfaction_rating":{"score":"offered"},"sharing_agreement_ids":[],"custom_status_id":4044356,"fields":[],"followup_ids":[],"ticket_form_id":360000084116,"brand_id":360000358316,"allow_channelback":false,"allow_attachments":true,"from_messaging_channel":false,"generated_timestamp":1655481702},"emitted_at":1694176872771}
 {"stream":"tickets","data":{"url":"https://d3v-airbyte.zendesk.com/api/v2/tickets/122.json","id":122,"external_id":null,"via":{"channel":"voice","source":{"rel":"voicemail","from":{"formatted_phone":"+1 (912) 420-0314","phone":"+19124200314","name":"Caller +1 (912) 420-0314"},"to":{"formatted_phone":"+1 (205) 953-1462","phone":"+12059531462","name":"Airbyte","brand_id":360000358316}}},"created_at":"2022-06-17T19:52:39Z","updated_at":"2022-06-17T21:01:41Z","type":null,"subject":"Voicemail from: Caller +1 (912) 420-0314","raw_subject":"Voicemail from: Caller +1 (912) 420-0314","description":"Call from: +1 (912) 420-0314\\nTime of call: June 17, 2022 at 7:52:02 PM","priority":null,"status":"new","recipient":null,"requester_id":4993467856015,"submitter_id":4993467856015,"assignee_id":null,"organization_id":null,"group_id":null,"collaborator_ids":[],"follower_ids":[],"email_cc_ids":[],"forum_topic_id":null,"problem_id":null,"has_incidents":false,"is_public":false,"due_at":null,"tags":[],"custom_fields":[],"satisfaction_rating":{"score":"offered"},"sharing_agreement_ids":[],"custom_status_id":4044356,"fields":[],"followup_ids":[],"ticket_form_id":360000084116,"brand_id":360000358316,"allow_channelback":false,"allow_attachments":true,"from_messaging_channel":false,"generated_timestamp":1655499701},"emitted_at":1694176872772}
-{"stream":"tickets","data":{"url":"https://d3v-airbyte.zendesk.com/api/v2/tickets/123.json","id":123,"external_id":null,"via":{"channel":"voice","source":{"rel":"voicemail","from":{"formatted_phone":"+1 (607) 210-9549","phone":"+16072109549","name":"Caller +1 (607) 210-9549"},"to":{"formatted_phone":"+1 (205) 953-1462","phone":"+12059531462","name":"Airbyte","brand_id":360000358316}}},"created_at":"2022-07-13T14:34:05Z","updated_at":"2023-09-07T16:46:02Z","type":null,"subject":"Voicemail from: Caller +1 (607) 210-9549","raw_subject":"Voicemail from: Caller +1 (607) 210-9549","description":"Call from: +1 (607) 210-9549\\nTime of call: July 13, 2022 at 2:33:27 PM","priority":null,"status":"deleted","recipient":null,"requester_id":5137812260495,"submitter_id":5137812260495,"assignee_id":null,"organization_id":null,"group_id":null,"collaborator_ids":[],"follower_ids":[],"email_cc_ids":[],"forum_topic_id":null,"problem_id":null,"has_incidents":false,"is_public":false,"due_at":null,"tags":[],"custom_fields":[],"satisfaction_rating":{"score":"offered"},"sharing_agreement_ids":[],"custom_status_id":4044356,"fields":[],"followup_ids":[],"ticket_form_id":360000084116,"brand_id":360000358316,"allow_channelback":false,"allow_attachments":true,"from_messaging_channel":false,"generated_timestamp":1694105162},"emitted_at":1694176872784}
 {"stream":"users","data":{"id":4992781783439,"url":"https://d3v-airbyte.zendesk.com/api/v2/users/4992781783439.json","name":"Caller +1 (689) 689-8023","email":null,"created_at":"2022-06-17T14:49:19Z","updated_at":"2022-06-17T14:49:19Z","time_zone":"Pacific/Noumea","iana_time_zone":"Pacific/Noumea","phone":"+16896898023","shared_phone_number":false,"photo":null,"locale_id":1,"locale":"en-US","organization_id":null,"role":"end-user","verified":true,"external_id":null,"tags":[],"alias":null,"active":true,"shared":false,"shared_agent":false,"last_login_at":null,"two_factor_auth_enabled":false,"signature":null,"details":null,"notes":null,"role_type":null,"custom_role_id":null,"moderator":false,"ticket_restriction":"requested","only_private_comments":false,"restricted_agent":true,"suspended":false,"default_group_id":null,"report_csv":false,"user_fields":{"test_display_name_checkbox_field":false,"test_display_name_decimal_field":null,"test_display_name_text_field":null}},"emitted_at":1693221422922}
 {"stream":"users","data":{"id":4993467856015,"url":"https://d3v-airbyte.zendesk.com/api/v2/users/4993467856015.json","name":"Caller +1 (912) 420-0314","email":null,"created_at":"2022-06-17T19:52:38Z","updated_at":"2022-06-17T19:52:38Z","time_zone":"Pacific/Noumea","iana_time_zone":"Pacific/Noumea","phone":"+19124200314","shared_phone_number":false,"photo":null,"locale_id":1,"locale":"en-US","organization_id":null,"role":"end-user","verified":true,"external_id":null,"tags":[],"alias":null,"active":true,"shared":false,"shared_agent":false,"last_login_at":null,"two_factor_auth_enabled":false,"signature":null,"details":null,"notes":null,"role_type":null,"custom_role_id":null,"moderator":false,"ticket_restriction":"requested","only_private_comments":false,"restricted_agent":true,"suspended":false,"default_group_id":null,"report_csv":false,"user_fields":{"test_display_name_checkbox_field":false,"test_display_name_decimal_field":null,"test_display_name_text_field":null}},"emitted_at":1693221422922}
 {"stream":"users","data":{"id":5137812260495,"url":"https://d3v-airbyte.zendesk.com/api/v2/users/5137812260495.json","name":"Caller +1 (607) 210-9549","email":null,"created_at":"2022-07-13T14:34:04Z","updated_at":"2022-07-13T14:34:04Z","time_zone":"Pacific/Noumea","iana_time_zone":"Pacific/Noumea","phone":"+16072109549","shared_phone_number":false,"photo":null,"locale_id":1,"locale":"en-US","organization_id":null,"role":"end-user","verified":true,"external_id":null,"tags":[],"alias":null,"active":true,"shared":false,"shared_agent":false,"last_login_at":null,"two_factor_auth_enabled":false,"signature":null,"details":null,"notes":null,"role_type":null,"custom_role_id":null,"moderator":false,"ticket_restriction":"requested","only_private_comments":false,"restricted_agent":true,"suspended":false,"default_group_id":null,"report_csv":false,"user_fields":{"test_display_name_checkbox_field":false,"test_display_name_decimal_field":null,"test_display_name_text_field":null}},"emitted_at":1693221422922}

airbyte-integrations/connectors/source-zendesk-support/metadata.yaml

@@ -7,7 +7,7 @@ data:
   connectorType: source
   maxSecondsBetweenMessages: 10800
   definitionId: 79c1aa37-dae3-42ae-b333-d1c105477715
-  dockerImageTag: 1.6.0
+  dockerImageTag: 1.7.0
   dockerRepository: airbyte/source-zendesk-support
   githubIssueLabel: source-zendesk-support
   icon: zendesk-support.svg

airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/deleted_tickets.json

@@ -0,0 +1,34 @@
+{
+  "$schema": "https://json-schema.org/draft-07/schema#",
+  "title": "Deleted Tickets",
+  "type": ["null", "object"],
+  "properties": {
+    "actor": {
+      "type": ["null", "object"],
+      "properties": {
+          "id": {
+            "type": ["null", "integer"]
+          },
+          "name": {
+            "type": ["null", "string"]
+          }
+        }
+    },
+    "id": {
+      "type": ["null", "integer"]
+    },
+    "subject": {
+      "type": ["null", "string"]
+    },
+    "description": {
+      "type": ["null", "string"]
+    },
+    "deleted_at":  {
+      "type": ["null", "string"],
+      "format": "date-time"
+    },
+    "previous_state": {
+      "type": ["null", "string"]
+    }
+  }
+}

airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/source.py

@@ -12,7 +12,7 @@
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator
-from source_zendesk_support.streams import DATETIME_FORMAT, SourceZendeskException
+from source_zendesk_support.streams import DATETIME_FORMAT, ZendeskConfigException
 
 from .streams import (
     AccountAttributes,
@@ -24,6 +24,7 @@
     AuditLogs,
     Brands,
     CustomRoles,
+    DeletedTickets,
     GroupMemberships,
     Groups,
     Macros,
@@ -101,7 +102,7 @@ def get_authenticator(cls, config: Mapping[str, Any]) -> [TokenAuthenticator, Ba
             elif auth.get("credentials") == "api_token":
                 return BasicApiTokenAuthenticator(config["credentials"]["email"], config["credentials"]["api_token"])
             else:
-                raise SourceZendeskException(f"Not implemented authorization method: {config['credentials']}")
+                raise ZendeskConfigException(message=f"Not implemented authorization method: {config['credentials']}")
 
     def check_connection(self, logger, config) -> Tuple[bool, any]:
         """Connection check to validate that the user-provided config can be used to connect to the underlying API
@@ -112,17 +113,18 @@ def check_connection(self, logger, config) -> Tuple[bool, any]:
         (False, error) otherwise.
         """
         auth = self.get_authenticator(config)
-        settings = None
         try:
             datetime.strptime(config["start_date"], DATETIME_FORMAT)
             settings = UserSettingsStream(config["subdomain"], authenticator=auth, start_date=None).get_settings()
         except Exception as e:
             return False, e
-
         active_features = [k for k, v in settings.get("active_features", {}).items() if v]
-        # logger.info("available features: %s" % active_features)
         if "organization_access_enabled" not in active_features:
-            return False, "Organization access is not enabled. Please check admin permission of the current account"
+            return (
+                False,
+                "Please verify that the account linked to the API key has admin permissions and try again."
+                "For more information visit https://support.zendesk.com/hc/en-us/articles/4408832171034-About-team-member-product-roles-and-access.",
+            )
         return True, None
 
     @classmethod
@@ -148,6 +150,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
             ArticleCommentVotes(**args),
             ArticleVotes(**args),
             AuditLogs(**args),
+            DeletedTickets(**args),
             GroupMemberships(**args),
             Groups(**args),
             Macros(**args),

airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py

@@ -18,6 +18,8 @@
 from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream
 from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader
 from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
+from airbyte_cdk.utils import AirbyteTracedException
+from airbyte_protocol.models import FailureType
 
 DATETIME_FORMAT: str = "%Y-%m-%dT%H:%M:%SZ"
 LAST_END_TIME_KEY: str = "_last_end_time"
@@ -35,8 +37,12 @@ def to_int(s):
     return s
 
 
-class SourceZendeskException(Exception):
-    """default exception of custom SourceZendesk logic"""
+class ZendeskConfigException(AirbyteTracedException):
+    """default config exception to custom SourceZendesk logic"""
+
+    def __init__(self, **kwargs):
+        failure_type: FailureType = FailureType.config_error
+        super(ZendeskConfigException, self).__init__(failure_type=failure_type, **kwargs)
 
 
 class BaseZendeskSupportStream(HttpStream, ABC):
@@ -119,7 +125,9 @@ def should_retry(self, response: requests.Response) -> bool:
             except requests.exceptions.JSONDecodeError:
                 reason = response.reason
                 error = {"title": f"{reason}", "message": "Received empty JSON response"}
-            self.logger.error(f"Skipping stream {self.name}: Check permissions, error message: {error}.")
+            self.logger.error(
+                f"Skipping stream {self.name}, error message: {error}. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support."
+            )
             setattr(self, "raise_on_http_errors", False)
             return False
         return super().should_retry(response)
@@ -794,7 +802,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp
     def get_settings(self) -> Mapping[str, Any]:
         for resp in self.read_records(SyncMode.full_refresh):
             return resp
-        raise SourceZendeskException("not found settings")
+        raise ZendeskConfigException(message="Can not get access to settings endpoint; Please check provided credentials")
 
     def request_params(
         self,
@@ -947,3 +955,28 @@ def path(
         article_id = stream_slice.get("parent").get("source_id")
         comment_id = stream_slice.get("parent").get("id")
         return f"help_center/articles/{article_id}/comments/{comment_id}/votes"
+
+
+class DeletedTickets(CursorPaginationZendeskSupportStream):
+    """Deleted Tickets Stream https://developer.zendesk.com/api-reference/ticketing/tickets/tickets/#list-deleted-tickets"""
+
+    response_list_name: str = "deleted_tickets"
+    transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization)
+    cursor_field = "deleted_at"
+
+    def path(self, **kwargs) -> str:
+        return "deleted_tickets.json"
+
+    def request_params(
+        self,
+        stream_state: Mapping[str, Any],
+        stream_slice: Mapping[str, Any] = None,
+        next_page_token: Mapping[str, Any] = None,
+    ) -> MutableMapping[str, Any]:
+        params = {
+            "sort_by": self.cursor_field,
+            "page[size]": self.page_size,
+        }
+        if next_page_token:
+            params.update(next_page_token)
+        return params

airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py

@@ -158,19 +158,19 @@ def test_check(response, start_date, check_passed):
 @pytest.mark.parametrize(
     "ticket_forms_response, status_code, expected_n_streams, expected_warnings, reason",
     [
-        ('{"ticket_forms": [{"id": 1, "updated_at": "2021-07-08T00:05:45Z"}]}', 200, 34, [], None),
+        ('{"ticket_forms": [{"id": 1, "updated_at": "2021-07-08T00:05:45Z"}]}', 200, 35, [], None),
         (
                 '{"error": "Not sufficient permissions"}',
                 403,
-                31,
-                ["Skipping stream ticket_forms: Check permissions, error message: Not sufficient permissions."],
+                32,
+                ["Skipping stream ticket_forms, error message: Not sufficient permissions. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support."],
                 None
         ),
         (
                 '',
                 404,
-                31,
-                ["Skipping stream ticket_forms: Check permissions, error message: {'title': 'Not Found', 'message': 'Received empty JSON response'}."],
+                32,
+                ["Skipping stream ticket_forms, error message: {'title': 'Not Found', 'message': 'Received empty JSON response'}. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support."],
                 'Not Found'
         ),
     ],