🐛 Source Zendesk Support: Fix wrong types for schemas, add Transformer (

airbytehq#8050)

* Fix wrong types for schemas, add Transformer

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -616,7 +616,7 @@
 - name: Zendesk Support
   sourceDefinitionId: 79c1aa37-dae3-42ae-b333-d1c105477715
   dockerRepository: airbyte/source-zendesk-support
-  dockerImageTag: 0.1.5
+  dockerImageTag: 0.1.6
   documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-support
   icon: zendesk.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -5992,7 +5992,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-zendesk-support:0.1.5"
+- dockerImage: "airbyte/source-zendesk-support:0.1.6"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/zendesk-support"
     connectionSpecification:

airbyte-integrations/connectors/source-zendesk-support/Dockerfile

@@ -25,5 +25,5 @@ COPY source_zendesk_support ./source_zendesk_support
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.5
+LABEL io.airbyte.version=0.1.6
 LABEL io.airbyte.name=airbyte/source-zendesk-support

airbyte-integrations/connectors/source-zendesk-support/setup.py

@@ -5,7 +5,7 @@
 
 from setuptools import find_packages, setup
 
-MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1.23", "pytz"]
+MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1.36", "pytz"]
 
 TEST_REQUIREMENTS = ["pytest~=6.1", "source-acceptance-test", "requests-mock==1.9.3"]
 

airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/metadata.json

@@ -1,7 +1,9 @@
 {
   "type": ["null", "object"],
   "properties": {
-    "custom": {},
+    "custom": {
+      "type": ["null", "object"]
+    },
     "trusted": {
       "type": ["null", "boolean"]
     },

airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/sla_policies.json

@@ -67,7 +67,9 @@
           "business_hours": {
             "type": ["null", "boolean"]
           },
-          "metric": {}
+          "metric": {
+            "type": ["null", "string"]
+          }
         },
         "type": ["null", "object"]
       }

airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_audits.json

@@ -125,7 +125,9 @@
           "formatted_to": {
             "type": ["null", "string"]
           },
-          "transcription_visible": {},
+          "transcription_visible": {
+            "type": ["null", "boolean"]
+          },
           "trusted": {
             "type": ["null", "boolean"]
           },
@@ -142,10 +144,7 @@
             "type": ["null", "integer"]
           },
           "value": {
-            "type": ["null", "array", "string"],
-            "items": {
-              "type": ["null", "string"]
-            }
+            "type": ["null", "string"]
           },
           "author_id": {
             "type": ["null", "integer"]
@@ -237,10 +236,7 @@
             "type": ["null", "integer"]
           },
           "previous_value": {
-            "type": ["null", "array", "string"],
-            "items": {
-              "type": ["null", "string"]
-            }
+            "type": ["null", "string"]
           },
           "macro_title": {
             "type": ["null", "string"]
@@ -260,7 +256,9 @@
     "metadata": {
       "type": ["null", "object"],
       "properties": {
-        "custom": {},
+        "custom": {
+          "type": ["null", "object"]
+        },
         "trusted": {
           "type": ["null", "boolean"]
         },

airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_fields.json

@@ -92,8 +92,7 @@
       "type": ["null", "boolean"]
     },
     "system_field_options": {
-      "type": ["null", "array"],
-      "items": {}
+      "type": ["null", "array"]
     },
     "sub_type_id": {
       "type": ["null", "integer"]

airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/tickets.json

@@ -77,7 +77,9 @@
           "id": {
             "type": ["null", "integer"]
           },
-          "value": {}
+          "value": {
+            "type": ["null", "string"]
+          }
         },
         "type": ["null", "object"]
       },

airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py

@@ -16,6 +16,7 @@
 from airbyte_cdk.models import SyncMode
 from airbyte_cdk.sources.streams.http import HttpStream
 from airbyte_cdk.sources.streams.http.auth.core import HttpAuthenticator
+from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
 
 DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
 LAST_END_TIME_KEY = "_last_end_time"
@@ -34,6 +35,8 @@ class SourceZendeskSupportStream(HttpStream, ABC):
     created_at_field = "created_at"
     updated_at_field = "updated_at"
 
+    transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization)
+
     def __init__(self, subdomain: str, **kwargs):
         super().__init__(**kwargs)
 

airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py

@@ -3,11 +3,12 @@
 #
 
 import json
-from unittest.mock import Mock
+from unittest.mock import MagicMock, Mock
 
 import pytest
 import requests
 import requests_mock
+from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode
 from requests.exceptions import HTTPError
 from source_zendesk_support import SourceZendeskSupport
 from source_zendesk_support.streams import Tags, TicketComments
@@ -22,6 +23,16 @@ def prepare_stream_args():
         return SourceZendeskSupport.convert_config2stream_args(json.loads(f.read()))
 
 
+@pytest.fixture(scope="module")
+def config():
+    """Generates fake config"""
+    return {
+        "subdomain": "fake_domain",
+        "start_date": "2020-01-01T00:00:00Z",
+        "auth_method": {"auth_method": "api_token", "email": "email@email.com", "api_token": "fake_api_token"},
+    }
+
+
 @pytest.mark.parametrize(
     "header_name,header_value,expected",
     [
@@ -94,3 +105,45 @@ def test_comments_not_found_ticket(prepare_stream_args, status_code, expected_co
                 next(comments)
         else:
             assert len(list(comments)) == expected_comment_count
+
+
+@pytest.mark.parametrize(
+    "input_data,expected_data",
+    [
+        (
+            {"id": 123, "custom_fields": [{"id": 3213212, "value": ["fake_3000", "fake_5555"]}]},
+            {"id": 123, "custom_fields": [{"id": 3213212, "value": "['fake_3000', 'fake_5555']"}]},
+        ),
+        (
+            {"id": 234, "custom_fields": [{"id": 2345234, "value": "fake_123"}]},
+            {"id": 234, "custom_fields": [{"id": 2345234, "value": "fake_123"}]},
+        ),
+        (
+            {"id": 345, "custom_fields": [{"id": 5432123, "value": 55432.321}]},
+            {"id": 345, "custom_fields": [{"id": 5432123, "value": "55432.321"}]},
+        ),
+    ],
+)
+def test_transform_for_tickets_stream(config, input_data, expected_data):
+    """Checks Transform in case when records come with invalid fields data types"""
+    test_catalog = ConfiguredAirbyteCatalog(
+        streams=[
+            ConfiguredAirbyteStream(
+                stream=AirbyteStream(name="tickets", json_schema={}),
+                sync_mode=SyncMode.full_refresh,
+                destination_sync_mode=DestinationSyncMode.overwrite,
+            )
+        ]
+    )
+
+    with requests_mock.Mocker() as ticket_mock:
+        ticket_mock.get(
+            f"https://{config['subdomain']}.zendesk.com/api/v2/incremental/tickets.json",
+            status_code=200,
+            json={"tickets": [input_data], "end_time": "2021-07-22T06:55:55Z", "end_of_stream": True},
+        )
+
+        source = SourceZendeskSupport()
+        records = source.read(MagicMock(), config, test_catalog, None)
+        for record in records:
+            assert record.record.data == expected_data

docs/integrations/sources/zendesk-support.md

@@ -97,6 +97,7 @@ We recommend creating a restricted, read-only key specifically for Airbyte acces
 
 | Version | Date | Pull Request | Subject |
 | :------ | :--------  | :-----       | :------ |
+| `0.1.6` | 2021-11-18 | [8050](https://github.com/airbytehq/airbyte/pull/8050) | Fix wrong types for schemas, add Transformer |
 | `0.1.5` | 2021-10-26 | [7679](https://github.com/airbytehq/airbyte/pull/7679) | Add ticket_id and ticket_comments |
 | `0.1.4` | 2021-10-26 | [7377](https://github.com/airbytehq/airbyte/pull/7377) | fix initially_assigned_at type in ticket metrics |
 | `0.1.3` | 2021-10-17 | [7097](https://github.com/airbytehq/airbyte/pull/7097) | correction of spec file |