Sourse Zendesk Talk: OAuth2.0 and unittests (#15764)

* Sourse Zendesk Talk:

- Oauth2.0 support
- unittest coverage 90%

* Source Zendesk Talk: update documentation

* Source Zendesk Talk: update changelog

* Source Zendesk Talk: update field description

* Source Zendesk Talk: fix documentation caption

* Source Zendesk Talk: fix spec, improve tests

* Source Zendesk Talk: remove Makefile

* Source Zendesk Talk: fix tests

* Source Zendesk Talk: fix tests

* Source Zendesk Talk: acceptance-test-config.yml test commit

* Source Zendesk Talk: acceptance-test-config.yml test commit

* Source Zendesk Talk: fix spec, add config to acceptance tests

* added oauth java test

* auto-bump connector version [ci skip]

Co-authored-by: Oleksandr Bazarnov <oleksandr.bazarnov@globallogic.com>
Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -1116,7 +1116,7 @@
 - name: Zendesk Talk
   sourceDefinitionId: c8630570-086d-4a40-99ae-ea5b18673071
   dockerRepository: airbyte/source-zendesk-talk
-  dockerImageTag: 0.1.3
+  dockerImageTag: 0.1.4
   documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-talk
   icon: zendesk.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -10979,49 +10979,125 @@
               path_in_connector_config:
               - "credentials"
               - "client_secret"
-- dockerImage: "airbyte/source-zendesk-talk:0.1.3"
+- dockerImage: "airbyte/source-zendesk-talk:0.1.4"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/zendesk-talk"
-    changelogUrl: "https://docs.airbyte.io/integrations/sources/zendesk-talk"
     connectionSpecification:
-      title: "Zendesk Talk Spec"
+      $schema: "http://json-schema.org/draft-07/schema#"
+      title: "Source Zendesk Talk Spec"
       type: "object"
+      required:
+      - "start_date"
+      - "subdomain"
       properties:
-        subdomain:
-          title: "Subdomain"
-          description: "The subdomain for your Zendesk Talk."
-          type: "string"
-        access_token:
-          title: "Access Token"
-          description: "The value of the API token generated. See the <a href=\"https://docs.airbyte.io/integrations/sources/zendesk-talk\"\
-            >docs</a> for more information."
-          airbyte_secret: true
-          type: "string"
-        email:
-          title: "Email"
-          description: "The user email for your Zendesk account."
-          type: "string"
         start_date:
-          title: "Replication Start Date"
-          description: "The date/datetime from which you'd like to replicate data\
-            \ for Zendesk Talk API, in the format YYYY-MM-DDT00:00:00Z. The time part\
-            \ is optional."
-          pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)?$"
+          type: "string"
+          title: "Start Date"
+          description: "The date from which you'd like to replicate data for Zendesk\
+            \ Talk API, in the format YYYY-MM-DDT00:00:00Z. All data generated after\
+            \ this date will be replicated."
           examples:
-          - "2017-01-25T00:00:00Z"
-          - "2017-01-25"
+          - "2020-10-15T00:00:00Z"
+          pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
+        subdomain:
           type: "string"
-          format: "date-time"
-      required:
-      - "subdomain"
-      - "access_token"
-      - "email"
-      - "start_date"
-    supportsIncremental: true
+          title: "Subdomain"
+          description: "This is your Zendesk subdomain that can be found in your account\
+            \ URL. For example, in https://{MY_SUBDOMAIN}.zendesk.com/, where MY_SUBDOMAIN\
+            \ is the value of your subdomain."
+        credentials:
+          title: "Authentication"
+          type: "object"
+          description: "Zendesk service provides two authentication methods. Choose\
+            \ between: `OAuth2.0` or `API token`."
+          oneOf:
+          - title: "API Token"
+            type: "object"
+            required:
+            - "email"
+            - "api_token"
+            additionalProperties: true
+            properties:
+              auth_type:
+                type: "string"
+                const: "api_token"
+              email:
+                title: "Email"
+                type: "string"
+                description: "The user email for your Zendesk account."
+              api_token:
+                title: "API Token"
+                type: "string"
+                description: "The value of the API token generated. See the <a href=\"\
+                  https://docs.airbyte.com/integrations/sources/zendesk-talk\">docs</a>\
+                  \ for more information."
+                airbyte_secret: true
+          - title: "OAuth2.0"
+            type: "object"
+            required:
+            - "access_token"
+            additionalProperties: true
+            properties:
+              auth_type:
+                type: "string"
+                const: "oauth2.0"
+                order: 0
+              access_token:
+                type: "string"
+                title: "Access Token"
+                description: "The value of the API token generated. See the <a href=\"\
+                  https://docs.airbyte.io/integrations/sources/zendesk-talk\">docs</a>\
+                  \ for more information."
+                airbyte_secret: true
     supportsNormalization: false
     supportsDBT: false
-    supported_destination_sync_modes:
-    - "append"
+    supported_destination_sync_modes: []
+    advanced_auth:
+      auth_flow_type: "oauth2.0"
+      predicate_key:
+      - "credentials"
+      - "auth_type"
+      predicate_value: "oauth2.0"
+      oauth_config_specification:
+        oauth_user_input_from_connector_config_specification:
+          type: "object"
+          additionalProperties: false
+          properties:
+            subdomain:
+              type: "string"
+              path_in_connector_config:
+              - "subdomain"
+        complete_oauth_output_specification:
+          type: "object"
+          additionalProperties: false
+          properties:
+            access_token:
+              type: "string"
+              path_in_connector_config:
+              - "credentials"
+              - "access_token"
+        complete_oauth_server_input_specification:
+          type: "object"
+          additionalProperties: false
+          properties:
+            client_id:
+              type: "string"
+            client_secret:
+              type: "string"
+        complete_oauth_server_output_specification:
+          type: "object"
+          additionalProperties: false
+          properties:
+            client_id:
+              type: "string"
+              path_in_connector_config:
+              - "credentials"
+              - "client_id"
+            client_secret:
+              type: "string"
+              path_in_connector_config:
+              - "credentials"
+              - "client_secret"
 - dockerImage: "airbyte/source-zenloop:0.1.1"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/zenloop"

airbyte-integrations/connectors/source-zendesk-talk/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.3
+LABEL io.airbyte.version=0.1.4
 LABEL io.airbyte.name=airbyte/source-zendesk-talk

airbyte-integrations/connectors/source-zendesk-talk/acceptance-test-config.yml

@@ -4,23 +4,30 @@
 connector_image: airbyte/source-zendesk-talk:dev
 tests:
   spec:
-    - spec_path: "integration_tests/spec.json"
+    - spec_path: "source_zendesk_talk/spec.json"
   connection:
     - config_path: "secrets/config.json"
       status: "succeed"
     - config_path: "integration_tests/invalid_config.json"
       status: "failed"
+    - config_path: "secrets/config_old.json"
+      status: "succeed"
   discovery:
     - config_path: "secrets/config.json"
+      backward_compatibility_tests_config:
+        disable_for_version: "0.1.3"
+    - config_path: "secrets/config_old.json"
+      backward_compatibility_tests_config:
+        disable_for_version: "0.1.3"
   basic_read:
     - config_path: "secrets/config.json"
   incremental:
     - config_path: "secrets/config.json"
       future_state_path: "integration_tests/abnormal_state.json"
   full_refresh:
     - config_path: "secrets/config.json"
-      # Statistics streams (with single record) have artificial PK that changes everytime
+      configured_catalog_path: "integration_tests/configured_catalog.json"
       ignored_fields:
-        "account_overview": ["current_timestamp"]
-        "agents_overview": ["current_timestamp"]
-        "current_queue_activity": ["current_timestamp"]
+        account_overview: ["current_timestamp"]
+        agents_overview: ["current_timestamp"]
+        current_queue_activity: ["current_timestamp"]

airbyte-integrations/connectors/source-zendesk-talk/acceptance-test-docker.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env sh
 
 # Build latest connector image
-docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2)
+docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-)
 
 # Pull latest acctest image
 docker pull airbyte/source-acceptance-test:latest

airbyte-integrations/connectors/source-zendesk-talk/integration_tests/invalid_config.json

@@ -1,6 +1,8 @@
 {
-  "email": "integration-test@airbyte.io",
-  "access_token": "some_token",
-  "subdomain": "domain",
-  "start_date": "2021-04-01T00:00:00Z"
-}
+  "credentials": {
+    "auth_type": "oauth2.0",
+    "access_token": "sometoken"
+  },
+  "subdomain": "d3v-airbyte",
+  "start_date": "2221-04-01T00:00:00Z"
+}

airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/source.py

@@ -2,14 +2,15 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
-from datetime import datetime
 from typing import Any, List, Mapping, Tuple
 
+import pendulum
+import requests
 from airbyte_cdk import AirbyteLogger
-from airbyte_cdk.models import ConnectorSpecification, DestinationSyncMode, SyncMode
+from airbyte_cdk.models import SyncMode
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
-from pydantic import BaseModel, Field
+from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator
 from requests.auth import HTTPBasicAuth
 from source_zendesk_talk.streams import (
     AccountOverview,
@@ -28,31 +29,25 @@
 )
 
 
-class ConnectorConfig(BaseModel):
-    class Config:
-        title = "Zendesk Talk Spec"
-
-    subdomain: str = Field(
-        description="The subdomain for your Zendesk Talk.",
-    )
-    access_token: str = Field(
-        description='The value of the API token generated. See the <a href="https://docs.airbyte.io/integrations/sources/zendesk-talk">docs</a> for more information.',
-        airbyte_secret=True,
-    )
-    email: str = Field(description="The user email for your Zendesk account.")
-    start_date: datetime = Field(
-        title="Replication Start Date",
-        description="The date/datetime from which you'd like to replicate data for Zendesk Talk API, in the format YYYY-MM-DDT00:00:00Z. The time part is optional.",
-        pattern="^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)?$",
-        examples=["2017-01-25T00:00:00Z", "2017-01-25"],
-    )
-
-
 class SourceZendeskTalk(AbstractSource):
+    @classmethod
+    def get_authenticator(cls, config: Mapping[str, Any]) -> requests.auth.AuthBase:
+        # old authentication flow support
+        if "access_token" in config and "email" in config:
+            return HTTPBasicAuth(username=f'{config["email"]}/token', password=config["access_token"])
+        # new authentication flow
+        auth = config["credentials"]
+        if auth:
+            if auth["auth_type"] == "oauth2.0":
+                return TokenAuthenticator(token=auth["access_token"])
+            elif auth["auth_type"] == "api_token":
+                return HTTPBasicAuth(username=f'{auth["email"]}/token', password=auth["api_token"])
+            else:
+                raise Exception(f"Not implemented authorization method: {auth['auth_type']}")
+
     def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]:
-        parsed_config = ConnectorConfig.parse_obj(config)
-        authenticator = HTTPBasicAuth(username=f"{parsed_config.email}/token", password=parsed_config.access_token)
-        stream = AccountOverview(authenticator=authenticator, subdomain=parsed_config.subdomain)
+        authenticator = self.get_authenticator(config)
+        stream = AccountOverview(authenticator=authenticator, subdomain=config["subdomain"])
 
         account_info = next(iter(stream.read_records(sync_mode=SyncMode.full_refresh)), None)
         if not account_info:
@@ -61,10 +56,9 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) ->
         return True, None
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
-        parsed_config = ConnectorConfig.parse_obj(config)
-        authenticator = HTTPBasicAuth(username=f"{parsed_config.email}/token", password=parsed_config.access_token)
-        common_kwargs = dict(authenticator=authenticator, subdomain=parsed_config.subdomain)
-        incremental_kwargs = dict(**common_kwargs, start_date=parsed_config.start_date)
+        authenticator = self.get_authenticator(config)
+        common_kwargs = {"authenticator": authenticator, "subdomain": config["subdomain"]}
+        incremental_kwargs = {**common_kwargs, **{"start_date": pendulum.parse(config["start_date"])}}
 
         return [
             AccountOverview(**common_kwargs),
@@ -81,16 +75,3 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
             IVRs(**common_kwargs),
             PhoneNumbers(**common_kwargs),
         ]
-
-    def spec(self, *args, **kwargs) -> ConnectorSpecification:
-        """
-        Returns the spec for this integration. The spec is a JSON-Schema object describing the required configurations (e.g: username and password)
-        required to run this integration.
-        """
-        return ConnectorSpecification(
-            documentationUrl="https://docs.airbyte.io/integrations/sources/zendesk-talk",
-            changelogUrl="https://docs.airbyte.io/integrations/sources/zendesk-talk",
-            supportsIncremental=True,
-            supported_destination_sync_modes=[DestinationSyncMode.append],
-            connectionSpecification=ConnectorConfig.schema(),
-        )