diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 0396136ea2dfca..c215f11bd2241d 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -616,7 +616,7 @@ - name: Zendesk Support sourceDefinitionId: 79c1aa37-dae3-42ae-b333-d1c105477715 dockerRepository: airbyte/source-zendesk-support - dockerImageTag: 0.1.5 + dockerImageTag: 0.1.6 documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-support icon: zendesk.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index e3243aa63eab19..8f5f3530b5b6ca 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -5992,7 +5992,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-zendesk-support:0.1.5" +- dockerImage: "airbyte/source-zendesk-support:0.1.6" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/zendesk-support" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-zendesk-support/Dockerfile b/airbyte-integrations/connectors/source-zendesk-support/Dockerfile index 97639fd09705dd..e093db6cdccb18 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/Dockerfile +++ b/airbyte-integrations/connectors/source-zendesk-support/Dockerfile @@ -25,5 +25,5 @@ COPY source_zendesk_support ./source_zendesk_support ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.5 +LABEL io.airbyte.version=0.1.6 LABEL io.airbyte.name=airbyte/source-zendesk-support diff --git a/airbyte-integrations/connectors/source-zendesk-support/setup.py b/airbyte-integrations/connectors/source-zendesk-support/setup.py index 62b2a22917df2e..54e946c9e11ce2 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/setup.py +++ b/airbyte-integrations/connectors/source-zendesk-support/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1.23", "pytz"] +MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1.36", "pytz"] TEST_REQUIREMENTS = ["pytest~=6.1", "source-acceptance-test", "requests-mock==1.9.3"] diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/metadata.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/metadata.json index b68e6ae7fa1ec5..c68c6c1a3f302c 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/metadata.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/metadata.json @@ -1,7 +1,9 @@ { "type": ["null", "object"], "properties": { - "custom": {}, + "custom": { + "type": ["null", "object"] + }, "trusted": { "type": ["null", "boolean"] }, diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/sla_policies.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/sla_policies.json index 22b17661762980..3699c6e12ba6dc 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/sla_policies.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/sla_policies.json @@ -67,7 +67,9 @@ "business_hours": { "type": ["null", "boolean"] }, - "metric": {} + "metric": { + "type": ["null", "string"] + } }, "type": ["null", "object"] } diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_audits.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_audits.json index eba36112908051..d75ab135bca8fb 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_audits.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_audits.json @@ -125,7 +125,9 @@ "formatted_to": { "type": ["null", "string"] }, - "transcription_visible": {}, + "transcription_visible": { + "type": ["null", "boolean"] + }, "trusted": { "type": ["null", "boolean"] }, @@ -142,10 +144,7 @@ "type": ["null", "integer"] }, "value": { - "type": ["null", "array", "string"], - "items": { - "type": ["null", "string"] - } + "type": ["null", "string"] }, "author_id": { "type": ["null", "integer"] @@ -237,10 +236,7 @@ "type": ["null", "integer"] }, "previous_value": { - "type": ["null", "array", "string"], - "items": { - "type": ["null", "string"] - } + "type": ["null", "string"] }, "macro_title": { "type": ["null", "string"] @@ -260,7 +256,9 @@ "metadata": { "type": ["null", "object"], "properties": { - "custom": {}, + "custom": { + "type": ["null", "object"] + }, "trusted": { "type": ["null", "boolean"] }, diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_fields.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_fields.json index b84b9afdb894af..3c9df5a7335e8d 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_fields.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_fields.json @@ -92,8 +92,7 @@ "type": ["null", "boolean"] }, "system_field_options": { - "type": ["null", "array"], - "items": {} + "type": ["null", "array"] }, "sub_type_id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/tickets.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/tickets.json index 20bfc48b707473..a66d51c2a5c49d 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/tickets.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/tickets.json @@ -77,7 +77,9 @@ "id": { "type": ["null", "integer"] }, - "value": {} + "value": { + "type": ["null", "string"] + } }, "type": ["null", "object"] }, diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py index 5f68e033b6ed71..7e909d06f200e6 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py @@ -16,6 +16,7 @@ from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth.core import HttpAuthenticator +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" LAST_END_TIME_KEY = "_last_end_time" @@ -34,6 +35,8 @@ class SourceZendeskSupportStream(HttpStream, ABC): created_at_field = "created_at" updated_at_field = "updated_at" + transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) + def __init__(self, subdomain: str, **kwargs): super().__init__(**kwargs) diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py index 3243c3ba8e79c2..c9230cd59ceffe 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py @@ -3,11 +3,12 @@ # import json -from unittest.mock import Mock +from unittest.mock import MagicMock, Mock import pytest import requests import requests_mock +from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode from requests.exceptions import HTTPError from source_zendesk_support import SourceZendeskSupport from source_zendesk_support.streams import Tags, TicketComments @@ -22,6 +23,16 @@ def prepare_stream_args(): return SourceZendeskSupport.convert_config2stream_args(json.loads(f.read())) +@pytest.fixture(scope="module") +def config(): + """Generates fake config""" + return { + "subdomain": "fake_domain", + "start_date": "2020-01-01T00:00:00Z", + "auth_method": {"auth_method": "api_token", "email": "email@email.com", "api_token": "fake_api_token"}, + } + + @pytest.mark.parametrize( "header_name,header_value,expected", [ @@ -94,3 +105,45 @@ def test_comments_not_found_ticket(prepare_stream_args, status_code, expected_co next(comments) else: assert len(list(comments)) == expected_comment_count + + +@pytest.mark.parametrize( + "input_data,expected_data", + [ + ( + {"id": 123, "custom_fields": [{"id": 3213212, "value": ["fake_3000", "fake_5555"]}]}, + {"id": 123, "custom_fields": [{"id": 3213212, "value": "['fake_3000', 'fake_5555']"}]}, + ), + ( + {"id": 234, "custom_fields": [{"id": 2345234, "value": "fake_123"}]}, + {"id": 234, "custom_fields": [{"id": 2345234, "value": "fake_123"}]}, + ), + ( + {"id": 345, "custom_fields": [{"id": 5432123, "value": 55432.321}]}, + {"id": 345, "custom_fields": [{"id": 5432123, "value": "55432.321"}]}, + ), + ], +) +def test_transform_for_tickets_stream(config, input_data, expected_data): + """Checks Transform in case when records come with invalid fields data types""" + test_catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream(name="tickets", json_schema={}), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + ) + ] + ) + + with requests_mock.Mocker() as ticket_mock: + ticket_mock.get( + f"https://{config['subdomain']}.zendesk.com/api/v2/incremental/tickets.json", + status_code=200, + json={"tickets": [input_data], "end_time": "2021-07-22T06:55:55Z", "end_of_stream": True}, + ) + + source = SourceZendeskSupport() + records = source.read(MagicMock(), config, test_catalog, None) + for record in records: + assert record.record.data == expected_data diff --git a/docs/integrations/sources/zendesk-support.md b/docs/integrations/sources/zendesk-support.md index e837f007e213f7..3a3c2852a8cd92 100644 --- a/docs/integrations/sources/zendesk-support.md +++ b/docs/integrations/sources/zendesk-support.md @@ -97,6 +97,7 @@ We recommend creating a restricted, read-only key specifically for Airbyte acces | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | +| `0.1.6` | 2021-11-18 | [8050](https://github.com/airbytehq/airbyte/pull/8050) | Fix wrong types for schemas, add Transformer | | `0.1.5` | 2021-10-26 | [7679](https://github.com/airbytehq/airbyte/pull/7679) | Add ticket_id and ticket_comments | | `0.1.4` | 2021-10-26 | [7377](https://github.com/airbytehq/airbyte/pull/7377) | fix initially_assigned_at type in ticket metrics | | `0.1.3` | 2021-10-17 | [7097](https://github.com/airbytehq/airbyte/pull/7097) | correction of spec file |