From 4985408ab8b7f70fac88fb27f187a9d867ee820b Mon Sep 17 00:00:00 2001 From: Tope Folorunso Date: Wed, 9 Aug 2023 21:24:17 +0100 Subject: [PATCH 1/5] migrate zendesk sunshine to low code --- .../source-zendesk-sunshine/Dockerfile | 36 +- .../source-zendesk-sunshine/README.md | 65 +-- .../source-zendesk-sunshine/__init__.py | 3 + .../acceptance-test-config.yml | 69 ++- .../acceptance-test-docker.sh | 1 + .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 5 + .../integration_tests/acceptance.py | 2 + .../integration_tests/sample_config.json | 3 + .../integration_tests/sample_state.json | 5 + .../source-zendesk-sunshine/metadata.yaml | 20 +- .../source-zendesk-sunshine/setup.py | 7 +- .../source_zendesk_sunshine/__init__.py | 25 +- .../source_zendesk_sunshine/manifest.yaml | 519 ++++++++++++++++++ .../source_zendesk_sunshine/source.py | 76 +-- .../source_zendesk_sunshine/spec.json | 145 ----- .../source_zendesk_sunshine/streams.py | 212 ------- 17 files changed, 649 insertions(+), 547 deletions(-) create mode 100644 airbyte-integrations/connectors/source-zendesk-sunshine/__init__.py create mode 100644 airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/manifest.yaml delete mode 100644 airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/spec.json delete mode 100644 airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/streams.py diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/Dockerfile b/airbyte-integrations/connectors/source-zendesk-sunshine/Dockerfile index d8c3461984234..603cbd8b64c6d 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/Dockerfile +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/Dockerfile @@ -1,16 +1,38 @@ -FROM python:3.9-slim +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base WORKDIR /airbyte/integration_code -COPY source_zendesk_sunshine ./source_zendesk_sunshine + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only COPY main.py ./ -COPY setup.py ./ -RUN pip install . +COPY source_zendesk_sunshine ./source_zendesk_sunshine ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.2.1 LABEL io.airbyte.name=airbyte/source-zendesk-sunshine diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/README.md b/airbyte-integrations/connectors/source-zendesk-sunshine/README.md index 04b9cba3ab926..452bbe11f6158 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/README.md +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/README.md @@ -1,34 +1,10 @@ # Zendesk Sunshine Source -This is the repository for the Zendesk Sunshine source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/zendesk-sunshine). +This is the repository for the Zendesk Sunshine configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/zendesk-sunshine). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. @@ -38,22 +14,14 @@ To build using Gradle, from the Airbyte repository root, run: ``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/zendesk-sunshine) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_zendesk_sunshine/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/zendesk-sunshine) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_zendesk_sunshine/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source zendesk-sunshine test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image #### Build @@ -78,32 +46,15 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zendesk-sunshine:dev d docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-zendesk-sunshine:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` #### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run + +To run your integration tests with Docker, run: ``` -python -m pytest integration_tests -p integration_tests.acceptance +./acceptance-test-docker.sh ``` -To run your integration tests with docker ### Using gradle to run tests All commands should be run from airbyte project root. diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/__init__.py b/airbyte-integrations/connectors/source-zendesk-sunshine/__init__.py new file mode 100644 index 0000000000000..c941b30457953 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-config.yml b/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-config.yml index 66c7333146997..7e781ad69c225 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-config.yml @@ -1,35 +1,50 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-zendesk-sunshine:dev -tests: +acceptance_tests: spec: - - spec_path: "source_zendesk_sunshine/spec.json" + tests: + - spec_path: "source_zendesk_sunshine/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "secrets/config_oauth.json" - status: "succeed" - - config_path: "secrets/config_api_token.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" - - config_path: "integration_tests/invalid_config_api_token.json" - status: "failed" - - config_path: "integration_tests/invalid_config_oauth.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "secrets/config_oauth.json" + status: "succeed" + - config_path: "secrets/config_api_token.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + - config_path: "integration_tests/invalid_config_api_token.json" + status: "failed" + - config_path: "integration_tests/invalid_config_oauth.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - - config_path: "secrets/config_api_token.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - - config_path: "secrets/config_oauth.json" - configured_catalog_path: "integration_tests/configured_catalog.json" -# incremental: # complex state ( {parent_id: {cur_field: value}} still not supported ) -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config_api_token.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config_oauth.json" + configured_catalog_path: "integration_tests/configured_catalog.json" +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.jsonl" +# extra_fields: no +# exact_order: no +# extra_records: yes + incremental: + bypass_reason: "This connector does not implement incremental sync" +# TODO uncomment this block this block if your connector implements incremental sync: +# tests: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# future_state: +# future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-docker.sh index 5797d20fe9a78..b6d65deeccb43 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/__init__.py b/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/__init__.py index e69de29bb2d1d..c941b30457953 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/abnormal_state.json new file mode 100644 index 0000000000000..52b0f2c2118f4 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/acceptance.py index 82823254d2666..9e6409236281f 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/sample_config.json new file mode 100644 index 0000000000000..ecc4913b84c74 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "fix-me": "TODO" +} diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/sample_state.json new file mode 100644 index 0000000000000..3587e579822d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-sunshine/metadata.yaml index f461072506317..2ba4493edc86c 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/metadata.yaml @@ -1,24 +1,28 @@ data: + allowedHosts: + hosts: + - ${subdomain}.zendesk.com + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: 325e0640-e7b3-4e24-b823-3361008f603f - dockerImageTag: 0.1.1 + dockerImageTag: 0.2.1 dockerRepository: airbyte/source-zendesk-sunshine githubIssueLabel: source-zendesk-sunshine icon: zendesk-sunshine.svg license: MIT name: Zendesk Sunshine - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: TODO releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-sunshine tags: - - language:python + - language:lowcode ab_internal: sl: 100 ql: 200 - supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/setup.py b/airbyte-integrations/connectors/source-zendesk-sunshine/setup.py index d09cfec2ebceb..3c71876ed7d37 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/setup.py +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/setup.py @@ -6,11 +6,12 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk", + "airbyte-cdk~=0.1", ] TEST_REQUIREMENTS = [ - "pytest~=6.1", + "pytest~=6.2", + "pytest-mock~=3.6.1", "connector-acceptance-test", ] @@ -21,7 +22,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/__init__.py b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/__init__.py index f1f84df115219..d6f54ad6589ff 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/__init__.py +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/__init__.py @@ -1,26 +1,7 @@ -""" -MIT License +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" from .source import SourceZendeskSunshine diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/manifest.yaml b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/manifest.yaml new file mode 100644 index 0000000000000..97022f85a8605 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/manifest.yaml @@ -0,0 +1,519 @@ +version: 0.50.2 +type: DeclarativeSource + +check: + type: CheckStream + stream_names: + - limits + +streams: + - type: DeclarativeStream + name: limits + primary_key: + - key + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://{{ config['subdomain'] }}.zendesk.com/api/sunshine/ + path: limits + http_method: GET + request_parameters: {} + request_headers: + Content-Type: application/json + authenticator: + type: BasicHttpAuthenticator + password: '{{ config[''password''] }}' + username: '{{ config[''username''] }}' + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + backoff_strategies: + - type: WaitTimeFromHeader + header: Retry-After + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + page_size_option: + type: RequestOption + field_name: per_page + inject_into: request_parameter + pagination_strategy: + type: CursorPagination + page_size: 100 + cursor_value: '{{ response.get("links", {}).get("next", {}) }}' + stop_condition: '{{ not response.get("links", {}).get("next", {}) }}' + - type: DeclarativeStream + name: object_records + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://{{ config['subdomain'] }}.zendesk.com/api/sunshine/ + path: objects/query + http_method: POST + request_parameters: + query: '{"_type": {"$eq": {{ stream_partition.type }} }' + sort_by: _updated_at asc + _updated_at: '{ "start": {{ stream_interval.start_time }} }' + request_headers: + Content-Type: application/json + authenticator: + type: BasicHttpAuthenticator + password: '{{ config[''password''] }}' + username: '{{ config[''username''] }}' + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + backoff_strategies: + - type: WaitTimeFromHeader + header: Retry-After + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + page_size_option: + type: RequestOption + field_name: per_page + inject_into: request_parameter + pagination_strategy: + type: CursorPagination + page_size: 100 + cursor_value: '{{ response.get("links", {}).get("next", {}) }}' + stop_condition: '{{ not response.get("links", {}).get("next", {}) }}' + partition_router: + - type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: key + partition_field: type + stream: + type: DeclarativeStream + name: object_types + primary_key: + - key + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + properties: {} + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: >- + https://{{ config['subdomain'] + }}.zendesk.com/api/sunshine/ + path: objects/types + http_method: GET + request_parameters: {} + request_headers: + Content-Type: application/json + authenticator: + type: BasicHttpAuthenticator + password: '{{ config[''password''] }}' + username: '{{ config[''username''] }}' + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + backoff_strategies: + - type: WaitTimeFromHeader + header: Retry-After + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + page_size_option: + type: RequestOption + field_name: per_page + inject_into: request_parameter + pagination_strategy: + type: CursorPagination + page_size: 100 + cursor_value: '{{ response.get("links", {}).get("next", {}) }}' + stop_condition: '{{ not response.get("links", {}).get("next", {}) }}' + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updated_at + cursor_datetime_formats: + - '%Y-%m-%dT%H:%M:%S.%f%z' + datetime_format: '%Y-%m-%dT%H:%M:%S.%f%z' + start_datetime: + type: MinMaxDatetime + datetime: '{{ config[''start_date''] }}' + datetime_format: '%Y-%m-%dT%H:%M:%SZ' + end_datetime: + type: MinMaxDatetime + datetime: '{{ now_utc().strftime(''%Y-%m-%dT%H:%M:%SZ'') }}' + datetime_format: '%Y-%m-%dT%H:%M:%SZ' + - type: DeclarativeStream + name: object_type_policies + primary_key: [] + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://{{ config['subdomain'] }}.zendesk.com/api/sunshine/ + path: objects/types/{{ stream_partition.type }}/permissions + http_method: GET + request_parameters: {} + request_headers: + Content-Type: application/json + authenticator: + type: BasicHttpAuthenticator + password: '{{ config[''password''] }}' + username: '{{ config[''username''] }}' + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + backoff_strategies: + - type: WaitTimeFromHeader + header: Retry-After + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + page_size_option: + type: RequestOption + field_name: per_page + inject_into: request_parameter + pagination_strategy: + type: CursorPagination + page_size: 100 + cursor_value: '{{ response.get("links", {}).get("next", {}) }}' + stop_condition: '{{ not response.get("links", {}).get("next", {}) }}' + partition_router: + - type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: key + partition_field: type + stream: + type: DeclarativeStream + name: object_types + primary_key: + - key + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + properties: {} + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: >- + https://{{ config['subdomain'] + }}.zendesk.com/api/sunshine/ + path: objects/types + http_method: GET + request_parameters: {} + request_headers: + Content-Type: application/json + authenticator: + type: BasicHttpAuthenticator + password: '{{ config[''password''] }}' + username: '{{ config[''username''] }}' + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + backoff_strategies: + - type: WaitTimeFromHeader + header: Retry-After + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + page_size_option: + type: RequestOption + field_name: per_page + inject_into: request_parameter + pagination_strategy: + type: CursorPagination + page_size: 100 + cursor_value: '{{ response.get("links", {}).get("next", {}) }}' + stop_condition: '{{ not response.get("links", {}).get("next", {}) }}' + transformations: + - type: AddFields + fields: + - path: + - object_type + value: '{{ stream_partition.type }}' + - type: DeclarativeStream + name: object_types + primary_key: + - key + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://{{ config['subdomain'] }}.zendesk.com/api/sunshine/ + path: objects/types + http_method: GET + request_parameters: {} + request_headers: + Content-Type: application/json + authenticator: + type: BasicHttpAuthenticator + password: '{{ config[''password''] }}' + username: '{{ config[''username''] }}' + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + backoff_strategies: + - type: WaitTimeFromHeader + header: Retry-After + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + page_size_option: + type: RequestOption + field_name: per_page + inject_into: request_parameter + pagination_strategy: + type: CursorPagination + page_size: 100 + cursor_value: '{{ response.get("links", {}).get("next", {}) }}' + stop_condition: '{{ not response.get("links", {}).get("next", {}) }}' + - type: DeclarativeStream + name: relationship_records + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://{{ config['subdomain'] }}.zendesk.com/api/sunshine/ + path: relationships/records + http_method: GET + request_parameters: + type: '{{ stream_partition.type }}' + request_headers: + Content-Type: application/json + authenticator: + type: BasicHttpAuthenticator + password: '{{ config[''password''] }}' + username: '{{ config[''username''] }}' + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + backoff_strategies: + - type: WaitTimeFromHeader + header: Retry-After + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + page_size_option: + type: RequestOption + field_name: per_page + inject_into: request_parameter + pagination_strategy: + type: CursorPagination + page_size: 100 + cursor_value: '{{ response.get("links", {}).get("next", {}) }}' + stop_condition: '{{ not response.get("links", {}).get("next", {}) }}' + partition_router: + - type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: key + partition_field: type + stream: + type: DeclarativeStream + name: relationship_types + primary_key: + - key + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + properties: {} + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: >- + https://{{ config['subdomain'] + }}.zendesk.com/api/sunshine/ + path: relationships/types + http_method: GET + request_parameters: {} + request_headers: + Content-Type: application/json + authenticator: + type: BasicHttpAuthenticator + password: '{{ config[''password''] }}' + username: '{{ config[''username''] }}' + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + backoff_strategies: + - type: WaitTimeFromHeader + header: Retry-After + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + page_size_option: + type: RequestOption + field_name: per_page + inject_into: request_parameter + pagination_strategy: + type: CursorPagination + page_size: 100 + cursor_value: '{{ response.get("links", {}).get("next", {}) }}' + stop_condition: '{{ not response.get("links", {}).get("next", {}) }}' + - type: DeclarativeStream + name: relationship_types + primary_key: + - key + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://{{ config['subdomain'] }}.zendesk.com/api/sunshine/ + path: relationships/types + http_method: GET + request_parameters: {} + request_headers: + Content-Type: application/json + authenticator: + type: BasicHttpAuthenticator + password: '{{ config[''password''] }}' + username: '{{ config[''username''] }}' + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + backoff_strategies: + - type: WaitTimeFromHeader + header: Retry-After + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + page_size_option: + type: RequestOption + field_name: per_page + inject_into: request_parameter + pagination_strategy: + type: CursorPagination + page_size: 100 + cursor_value: '{{ response.get("links", {}).get("next", {}) }}' + stop_condition: '{{ not response.get("links", {}).get("next", {}) }}' + +spec: + documentation_url: https://example.org + type: Spec + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + type: object + additionalProperties: true + required: + - start_date + - subdomain + properties: + start_date: + type: string + title: Start date + format: date-time + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ + order: 0 + username: + type: string + title: Username + order: 1 + password: + type: string + title: Password + always_show: true + airbyte_secret: true + order: 2 + subdomain: + type: string + order: 3 + title: Subdomain + description: The subdomain for your Zendesk Account. + diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/source.py b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/source.py index c848aa744881d..aac70fbfffcbe 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/source.py +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/source.py @@ -2,73 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import base64 -from typing import Any, List, Mapping, Tuple, Union +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import pendulum -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +WARNING: Do not modify this file. +""" -from .streams import Limits, ObjectRecords, ObjectTypePolicies, ObjectTypes, RelationshipRecords, RelationshipTypes - -class Base64HttpAuthenticator(TokenAuthenticator): - def __init__(self, auth: Tuple[str, str], auth_method: str = "Basic", **kwargs): - auth_string = f"{auth[0]}:{auth[1]}".encode("utf8") - b64_encoded = base64.b64encode(auth_string).decode("utf8") - super().__init__(token=b64_encoded, auth_method=auth_method, **kwargs) - - -class ZendeskSunshineAuthenticator: - """Provides the authentication capabilities for both old and new methods.""" - - @staticmethod - def get_auth(config: Mapping[str, Any]) -> Union[Base64HttpAuthenticator, TokenAuthenticator]: - credentials = config.get("credentials", {}) - token = config.get("api_token") or credentials.get("api_token") - email = config.get("email") or credentials.get("email") - if email and token: - return Base64HttpAuthenticator(auth=(f"{email}/token", token)) - return TokenAuthenticator(token=credentials["access_token"]) - - -class SourceZendeskSunshine(AbstractSource): - def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: - try: - pendulum.parse(config["start_date"], strict=True) - authenticator = ZendeskSunshineAuthenticator.get_auth(config) - stream = Limits(authenticator=authenticator, subdomain=config["subdomain"], start_date=pendulum.parse(config["start_date"])) - records = stream.read_records(sync_mode=SyncMode.full_refresh) - next(records) - return True, None - except Exception as e: - return False, repr(e) - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - CustomObjectEvents stream is an early access stream. (looks like it is a new feature) - It requires activation in site ui + manual activation from Zendesk via call. - I requested the call, but since they did not approve it, - this endpoint will return 403 Forbidden. Thats why it is disabled here. - - Jobs stream is also commented out. Reason: It is dynamic. - It can have the data, but this data have time to live. - After this time is passed we have no data. It will require permanent population, to pass - the test criteria `stream should contain at least 1 record) - """ - authenticator = ZendeskSunshineAuthenticator.get_auth(config) - args = {"authenticator": authenticator, "subdomain": config["subdomain"], "start_date": config["start_date"]} - return [ - ObjectTypes(**args), - ObjectRecords(**args), - RelationshipTypes(**args), - RelationshipRecords(**args), - # CustomObjectEvents(**args), - ObjectTypePolicies(**args), - # Jobs(**args), - Limits(**args), - ] +# Declarative Source +class SourceZendeskSunshine(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/spec.json b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/spec.json deleted file mode 100644 index 29e900a4b1a5c..0000000000000 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/spec.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/zendesk_sunshine", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Zendesk Sunshine Spec", - "type": "object", - "required": ["start_date", "subdomain"], - "additionalProperties": true, - "properties": { - "subdomain": { - "title": "Subdomain", - "type": "string", - "description": "The subdomain for your Zendesk Account." - }, - "start_date": { - "title": "Start Date", - "type": "string", - "description": "The date from which you'd like to replicate data for Zendesk Sunshine API, in the format YYYY-MM-DDT00:00:00Z.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2021-01-01T00:00:00Z"] - }, - "credentials": { - "title": "Authorization Method", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "OAuth2.0", - "required": [ - "auth_method", - "client_id", - "client_secret", - "access_token" - ], - "properties": { - "auth_method": { - "type": "string", - "const": "oauth2.0", - "enum": ["oauth2.0"], - "default": "oauth2.0", - "order": 0 - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The Client ID of your OAuth application.", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "description": "The Client Secret of your OAuth application.", - "airbyte_secret": true - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "Long-term access Token for making authenticated requests.", - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "API Token", - "required": ["auth_method", "api_token", "email"], - "properties": { - "auth_method": { - "type": "string", - "const": "api_token", - "enum": ["api_token"], - "default": "api_token", - "order": 1 - }, - "api_token": { - "type": "string", - "title": "API Token", - "description": "API Token. See the docs for information on how to generate this key.", - "airbyte_secret": true - }, - "email": { - "type": "string", - "title": "Email", - "description": "The user email for your Zendesk account" - } - } - } - ] - } - } - }, - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_method"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - }, - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "subdomain": { - "type": "string", - "path_in_connector_config": ["subdomain"] - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/streams.py b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/streams.py deleted file mode 100644 index b68ee15d8c586..0000000000000 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/streams.py +++ /dev/null @@ -1,212 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import urllib.parse -from abc import ABC -from typing import Any, Iterable, Mapping, MutableMapping, Optional - -import pendulum -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http import HttpStream - - -class SunshineStream(HttpStream, ABC): - primary_key = "id" - data_field = "data" - page_size = 100 - - def __init__(self, subdomain: str, start_date: pendulum.datetime, **kwargs): - self._start_date = start_date - self.subdomain = subdomain - super().__init__(**kwargs) - - @property - def url_base(self) -> str: - return f"https://{self.subdomain}.zendesk.com/api/sunshine/" - - def backoff_time(self, response: requests.Response) -> Optional[float]: - delay_time = response.headers.get("Retry-After") - if delay_time: - return float(delay_time) - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - resp_json = response.json() - if resp_json.get("links") and resp_json.get("links").get("next"): - next_query_string = urllib.parse.urlsplit(resp_json.get("links").get("next")).query - params = dict(urllib.parse.parse_qsl(next_query_string)) - return params - return {} - - def request_headers(self, **kwargs) -> Mapping[str, Any]: - return {"Content-Type": "application/json"} - - def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: - """ - The response data field is mostly a list of objects. Sometimes we can have object in data field. - (example `ObjectTypePolicies`). In this case this method should be overridden. - """ - response_json = response.json() - yield from response_json.get(self.data_field, []) - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - - params = {"per_page": self.page_size} - if next_page_token: - params.update(next_page_token) - return params - - -class IncrementalSunshineStream(SunshineStream, ABC): - state_checkpoint_interval = 1000 - cursor_field = "updated_at" # most common - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object - and returning an updated state object. - """ - latest_state = latest_record.get(self.cursor_field) - current_state = current_stream_state.get(self.cursor_field) or latest_state - # dates are ISO-formatted, no need to parse - return {self.cursor_field: max(latest_state, current_state)} - - -class ObjectTypes(SunshineStream): - primary_key = "key" - - def path(self, **kwargs) -> str: - return "objects/types" - - -class ObjectRecords(IncrementalSunshineStream): - """ - The get method supports only the full-refresh way to get the information fron this source. - This source has date fields in all the endpoints, but we cannot query this field during GET requests. - To support Incremental for this stream I had to use `query` endpoint instead of `objects/records` - - this allows me to use date filters. This is the only way to have incremental support. - """ - - http_method = "POST" - - def request_body_json( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Optional[Mapping]: - type_ = stream_slice["type"] - state_value = stream_state.get(type_, {}).get(self.cursor_field) - start_date = state_value or self._start_date - formatted_start_date = pendulum.parse(start_date).strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] - query = { - "query": {"_type": {"$eq": type_}}, - "_updated_at": { - "start": formatted_start_date, - }, - "sort_by": "_updated_at asc", - } - return query - - def path(self, **kwargs) -> str: - return "objects/query" - - def stream_slices(self, **kwargs): - parent_stream = ObjectTypes(authenticator=self.authenticator, subdomain=self.subdomain, start_date=self._start_date) - for obj_type in parent_stream.read_records(sync_mode=SyncMode.full_refresh): - yield {"type": obj_type["key"]} - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - type_ = latest_record.get("type") - latest_cursor_value = latest_record.get(self.cursor_field) - current_stream_state = current_stream_state or {} - current_state = current_stream_state.get(type_) if current_stream_state else None - if current_state: - current_state = current_state.get(self.cursor_field) - current_state_value = current_state or latest_cursor_value - max_value = max(current_state_value, latest_cursor_value) - new_value = {self.cursor_field: max_value} - - current_stream_state[type_] = new_value - return current_stream_state - - -class RelationshipTypes(SunshineStream): - primary_key = "key" - - def path(self, **kwargs) -> str: - return "relationships/types" - - -class RelationshipRecords(SunshineStream): - def path(self, **kwargs) -> str: - return "relationships/records" - - def stream_slices(self, **kwargs): - parent_stream = RelationshipTypes(authenticator=self.authenticator, subdomain=self.subdomain, start_date=self._start_date) - for rel_type in parent_stream.read_records(sync_mode=SyncMode.full_refresh): - yield {"type": rel_type["key"]} - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - - params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - type_ = stream_slice["type"] - params["type"] = type_ - return params - - -class CustomObjectEvents(SunshineStream): - """ - This stream is early access stream. (look like a new feature) - It requires activation in site ui + manual activation from Zendesk via call. - I requested the call, but since they did not approve it, - this endpoint will return 403 Forbidden - """ - - def path(self, **kwargs) -> str: - return "objects/events" - - -class ObjectTypePolicies(SunshineStream): - primary_key = None - - def stream_slices(self, **kwargs): - parent_stream = ObjectTypes(authenticator=self.authenticator, subdomain=self.subdomain, start_date=self._start_date) - for obj_type in parent_stream.read_records(sync_mode=SyncMode.full_refresh): - yield {"type": obj_type["key"]} - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - obj_type = stream_slice["type"] - return f"objects/types/{obj_type}/permissions" - - def parse_response( - self, response: requests.Response, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs - ) -> Iterable[Mapping]: - response_json = response.json() - data = response_json.get(self.data_field, {}) - # the response does not contain info about parent itself - only rules. Need to add this. - data["object_type"] = stream_slice["type"] - yield data - - -class Jobs(SunshineStream): - """ - This stream is dynamic. The data can exist today, but may be absent tomorrow. - Since we need to have some data in the stream this stream is disabled. - """ - - def path(self, **kwargs) -> str: - return "jobs" - - -class Limits(SunshineStream): - primary_key = "key" - - def path(self, **kwargs) -> str: - return "limits" From 12f8d6e4b1f3b961e67fdea760480a0c4094431c Mon Sep 17 00:00:00 2001 From: Tope Folorunso Date: Tue, 22 Aug 2023 04:13:44 +0100 Subject: [PATCH 2/5] add oauth option --- .../source-zendesk-sunshine/metadata.yaml | 2 +- .../source_zendesk_sunshine/components.py | 22 + .../source_zendesk_sunshine/manifest.yaml | 628 ++++++------------ docs/integrations/sources/zendesk-sunshine.md | 1 + 4 files changed, 232 insertions(+), 421 deletions(-) create mode 100644 airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/components.py diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-sunshine/metadata.yaml index bb06f5908785f..de852cc4aaa3c 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/metadata.yaml @@ -16,7 +16,7 @@ data: icon: zendesk-sunshine.svg license: MIT name: Zendesk Sunshine - releaseDate: TODO + releaseDate: 2021-07-08 releaseStage: alpha supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-sunshine diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/components.py b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/components.py new file mode 100644 index 0000000000000..11e569eac735f --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/components.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dataclasses import dataclass +from typing import Any, Mapping + +from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator +from airbyte_cdk.sources.declarative.auth.token import BasicHttpAuthenticator, BearerAuthenticator + + +@dataclass +class AuthenticatorZendeskSunshine(DeclarativeAuthenticator): + config: Mapping[str, Any] + basic_auth: BasicHttpAuthenticator + oauth2: BearerAuthenticator + + def __new__(cls, basic_auth, oauth2, config, *args, **kwargs): + if config["credentials"]["email"] and config["credentials"]["api_token"]: + return basic_auth + else: + return oauth2 \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/manifest.yaml b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/manifest.yaml index 97022f85a8605..2e0803950c767 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/manifest.yaml +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/manifest.yaml @@ -6,100 +6,95 @@ check: stream_names: - limits -streams: - - type: DeclarativeStream - name: limits - primary_key: - - key +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + page_size_option: + type: RequestOption + field_name: per_page + inject_into: request_parameter + pagination_strategy: + type: CursorPagination + page_size: 100 + cursor_value: '{{ response.get("links", {}).get("next", {}) }}' + stop_condition: '{{ not response.get("links", {}).get("next", {}) }}' + basic_authenticator: + type: BasicHttpAuthenticator + password: "{{ config['credentials']['api_token'] }}" + username: "{{ config['credentials']['email'] }}/token" + oauth2_authenticator: + type: BearerAuthenticator + api_token: "{{ config['credentials']['access_token'] }}" + requester: + type: HttpRequester + url_base: https://{{ config['subdomain'] }}.zendesk.com/api/sunshine/ + http_method: GET + request_headers: + Content-Type: application/json + authenticator: + class_name: source_zendesk_sunshine.components.AuthenticatorZendeskSunshine + basic_auth: "#/definitions/basic_authenticator" + oauth2: "#/definitions/oauth2_authenticator" + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + backoff_strategies: + - type: WaitTimeFromHeader + header: Retry-After + request_body_json: {} + base_stream: + type: DeclarativeStream + primary_key: key retriever: type: SimpleRetriever requester: - type: HttpRequester - url_base: https://{{ config['subdomain'] }}.zendesk.com/api/sunshine/ - path: limits - http_method: GET - request_parameters: {} - request_headers: - Content-Type: application/json - authenticator: - type: BasicHttpAuthenticator - password: '{{ config[''password''] }}' - username: '{{ config[''username''] }}' - error_handler: - type: CompositeErrorHandler - error_handlers: - - type: DefaultErrorHandler - backoff_strategies: - - type: WaitTimeFromHeader - header: Retry-After - request_body_json: {} + $ref: "#/definitions/requester" + path: "{{ parameters.path }}" record_selector: - type: RecordSelector - extractor: - type: DpathExtractor - field_path: - - data + $ref: "#/definitions/selector" paginator: - type: DefaultPaginator - page_token_option: - type: RequestPath - page_size_option: - type: RequestOption - field_name: per_page - inject_into: request_parameter - pagination_strategy: - type: CursorPagination - page_size: 100 - cursor_value: '{{ response.get("links", {}).get("next", {}) }}' - stop_condition: '{{ not response.get("links", {}).get("next", {}) }}' - - type: DeclarativeStream + $ref: "#/definitions/paginator" + limits_stream: + $ref: "#/definitions/base_stream" + name: limits + $parameters: + path: limits + relationship_types_stream: + $ref: "#/definitions/base_stream" + name: relationship_types + $parameters: + path: relationships/types + object_types_stream: + $ref: "#/definitions/base_stream" + name: object_types + $parameters: + path: objects/types + object_records_stream: + type: DeclarativeStream name: object_records - primary_key: - - id + primary_key: id retriever: type: SimpleRetriever requester: - type: HttpRequester - url_base: https://{{ config['subdomain'] }}.zendesk.com/api/sunshine/ + $ref: "#/definitions/requester" path: objects/query - http_method: POST request_parameters: query: '{"_type": {"$eq": {{ stream_partition.type }} }' sort_by: _updated_at asc _updated_at: '{ "start": {{ stream_interval.start_time }} }' - request_headers: - Content-Type: application/json - authenticator: - type: BasicHttpAuthenticator - password: '{{ config[''password''] }}' - username: '{{ config[''username''] }}' - error_handler: - type: CompositeErrorHandler - error_handlers: - - type: DefaultErrorHandler - backoff_strategies: - - type: WaitTimeFromHeader - header: Retry-After - request_body_json: {} record_selector: - type: RecordSelector - extractor: - type: DpathExtractor - field_path: - - data + $ref: "#/definitions/selector" paginator: - type: DefaultPaginator - page_token_option: - type: RequestPath - page_size_option: - type: RequestOption - field_name: per_page - inject_into: request_parameter - pagination_strategy: - type: CursorPagination - page_size: 100 - cursor_value: '{{ response.get("links", {}).get("next", {}) }}' - stop_condition: '{{ not response.get("links", {}).get("next", {}) }}' + $ref: "#/definitions/paginator" partition_router: - type: SubstreamPartitionRouter parent_stream_configs: @@ -107,60 +102,7 @@ streams: parent_key: key partition_field: type stream: - type: DeclarativeStream - name: object_types - primary_key: - - key - schema_loader: - type: InlineSchemaLoader - schema: - $schema: http://json-schema.org/draft-07/schema# - additionalProperties: true - properties: {} - type: object - retriever: - type: SimpleRetriever - requester: - type: HttpRequester - url_base: >- - https://{{ config['subdomain'] - }}.zendesk.com/api/sunshine/ - path: objects/types - http_method: GET - request_parameters: {} - request_headers: - Content-Type: application/json - authenticator: - type: BasicHttpAuthenticator - password: '{{ config[''password''] }}' - username: '{{ config[''username''] }}' - error_handler: - type: CompositeErrorHandler - error_handlers: - - type: DefaultErrorHandler - backoff_strategies: - - type: WaitTimeFromHeader - header: Retry-After - request_body_json: {} - record_selector: - type: RecordSelector - extractor: - type: DpathExtractor - field_path: - - data - paginator: - type: DefaultPaginator - page_token_option: - type: RequestPath - page_size_option: - type: RequestOption - field_name: per_page - inject_into: request_parameter - pagination_strategy: - type: CursorPagination - page_size: 100 - cursor_value: '{{ response.get("links", {}).get("next", {}) }}' - stop_condition: '{{ not response.get("links", {}).get("next", {}) }}' + $ref: "#/definitions/object_types_stream" incremental_sync: type: DatetimeBasedCursor cursor_field: updated_at @@ -175,50 +117,19 @@ streams: type: MinMaxDatetime datetime: '{{ now_utc().strftime(''%Y-%m-%dT%H:%M:%SZ'') }}' datetime_format: '%Y-%m-%dT%H:%M:%SZ' - - type: DeclarativeStream + object_type_policies_stream: + type: DeclarativeStream name: object_type_policies primary_key: [] retriever: type: SimpleRetriever requester: - type: HttpRequester - url_base: https://{{ config['subdomain'] }}.zendesk.com/api/sunshine/ + $ref: "#/definitions/requester" path: objects/types/{{ stream_partition.type }}/permissions - http_method: GET - request_parameters: {} - request_headers: - Content-Type: application/json - authenticator: - type: BasicHttpAuthenticator - password: '{{ config[''password''] }}' - username: '{{ config[''username''] }}' - error_handler: - type: CompositeErrorHandler - error_handlers: - - type: DefaultErrorHandler - backoff_strategies: - - type: WaitTimeFromHeader - header: Retry-After - request_body_json: {} record_selector: - type: RecordSelector - extractor: - type: DpathExtractor - field_path: - - data + $ref: "#/definitions/selector" paginator: - type: DefaultPaginator - page_token_option: - type: RequestPath - page_size_option: - type: RequestOption - field_name: per_page - inject_into: request_parameter - pagination_strategy: - type: CursorPagination - page_size: 100 - cursor_value: '{{ response.get("links", {}).get("next", {}) }}' - stop_condition: '{{ not response.get("links", {}).get("next", {}) }}' + $ref: "#/definitions/paginator" partition_router: - type: SubstreamPartitionRouter parent_stream_configs: @@ -226,157 +137,28 @@ streams: parent_key: key partition_field: type stream: - type: DeclarativeStream - name: object_types - primary_key: - - key - schema_loader: - type: InlineSchemaLoader - schema: - $schema: http://json-schema.org/draft-07/schema# - additionalProperties: true - properties: {} - type: object - retriever: - type: SimpleRetriever - requester: - type: HttpRequester - url_base: >- - https://{{ config['subdomain'] - }}.zendesk.com/api/sunshine/ - path: objects/types - http_method: GET - request_parameters: {} - request_headers: - Content-Type: application/json - authenticator: - type: BasicHttpAuthenticator - password: '{{ config[''password''] }}' - username: '{{ config[''username''] }}' - error_handler: - type: CompositeErrorHandler - error_handlers: - - type: DefaultErrorHandler - backoff_strategies: - - type: WaitTimeFromHeader - header: Retry-After - request_body_json: {} - record_selector: - type: RecordSelector - extractor: - type: DpathExtractor - field_path: - - data - paginator: - type: DefaultPaginator - page_token_option: - type: RequestPath - page_size_option: - type: RequestOption - field_name: per_page - inject_into: request_parameter - pagination_strategy: - type: CursorPagination - page_size: 100 - cursor_value: '{{ response.get("links", {}).get("next", {}) }}' - stop_condition: '{{ not response.get("links", {}).get("next", {}) }}' + $ref: "#/definitions/object_types_stream" transformations: - type: AddFields fields: - path: - object_type value: '{{ stream_partition.type }}' - - type: DeclarativeStream - name: object_types - primary_key: - - key - retriever: - type: SimpleRetriever - requester: - type: HttpRequester - url_base: https://{{ config['subdomain'] }}.zendesk.com/api/sunshine/ - path: objects/types - http_method: GET - request_parameters: {} - request_headers: - Content-Type: application/json - authenticator: - type: BasicHttpAuthenticator - password: '{{ config[''password''] }}' - username: '{{ config[''username''] }}' - error_handler: - type: CompositeErrorHandler - error_handlers: - - type: DefaultErrorHandler - backoff_strategies: - - type: WaitTimeFromHeader - header: Retry-After - request_body_json: {} - record_selector: - type: RecordSelector - extractor: - type: DpathExtractor - field_path: - - data - paginator: - type: DefaultPaginator - page_token_option: - type: RequestPath - page_size_option: - type: RequestOption - field_name: per_page - inject_into: request_parameter - pagination_strategy: - type: CursorPagination - page_size: 100 - cursor_value: '{{ response.get("links", {}).get("next", {}) }}' - stop_condition: '{{ not response.get("links", {}).get("next", {}) }}' - - type: DeclarativeStream + relationship_records_stream: + type: DeclarativeStream name: relationship_records - primary_key: - - id + primary_key: id retriever: type: SimpleRetriever requester: - type: HttpRequester - url_base: https://{{ config['subdomain'] }}.zendesk.com/api/sunshine/ + $ref: "#/definitions/requester" path: relationships/records - http_method: GET request_parameters: type: '{{ stream_partition.type }}' - request_headers: - Content-Type: application/json - authenticator: - type: BasicHttpAuthenticator - password: '{{ config[''password''] }}' - username: '{{ config[''username''] }}' - error_handler: - type: CompositeErrorHandler - error_handlers: - - type: DefaultErrorHandler - backoff_strategies: - - type: WaitTimeFromHeader - header: Retry-After - request_body_json: {} record_selector: - type: RecordSelector - extractor: - type: DpathExtractor - field_path: - - data + $ref: "#/definitions/selector" paginator: - type: DefaultPaginator - page_token_option: - type: RequestPath - page_size_option: - type: RequestOption - field_name: per_page - inject_into: request_parameter - pagination_strategy: - type: CursorPagination - page_size: 100 - cursor_value: '{{ response.get("links", {}).get("next", {}) }}' - stop_condition: '{{ not response.get("links", {}).get("next", {}) }}' + $ref: "#/definitions/paginator" partition_router: - type: SubstreamPartitionRouter parent_stream_configs: @@ -384,108 +166,18 @@ streams: parent_key: key partition_field: type stream: - type: DeclarativeStream - name: relationship_types - primary_key: - - key - schema_loader: - type: InlineSchemaLoader - schema: - $schema: http://json-schema.org/draft-07/schema# - additionalProperties: true - properties: {} - type: object - retriever: - type: SimpleRetriever - requester: - type: HttpRequester - url_base: >- - https://{{ config['subdomain'] - }}.zendesk.com/api/sunshine/ - path: relationships/types - http_method: GET - request_parameters: {} - request_headers: - Content-Type: application/json - authenticator: - type: BasicHttpAuthenticator - password: '{{ config[''password''] }}' - username: '{{ config[''username''] }}' - error_handler: - type: CompositeErrorHandler - error_handlers: - - type: DefaultErrorHandler - backoff_strategies: - - type: WaitTimeFromHeader - header: Retry-After - request_body_json: {} - record_selector: - type: RecordSelector - extractor: - type: DpathExtractor - field_path: - - data - paginator: - type: DefaultPaginator - page_token_option: - type: RequestPath - page_size_option: - type: RequestOption - field_name: per_page - inject_into: request_parameter - pagination_strategy: - type: CursorPagination - page_size: 100 - cursor_value: '{{ response.get("links", {}).get("next", {}) }}' - stop_condition: '{{ not response.get("links", {}).get("next", {}) }}' - - type: DeclarativeStream - name: relationship_types - primary_key: - - key - retriever: - type: SimpleRetriever - requester: - type: HttpRequester - url_base: https://{{ config['subdomain'] }}.zendesk.com/api/sunshine/ - path: relationships/types - http_method: GET - request_parameters: {} - request_headers: - Content-Type: application/json - authenticator: - type: BasicHttpAuthenticator - password: '{{ config[''password''] }}' - username: '{{ config[''username''] }}' - error_handler: - type: CompositeErrorHandler - error_handlers: - - type: DefaultErrorHandler - backoff_strategies: - - type: WaitTimeFromHeader - header: Retry-After - request_body_json: {} - record_selector: - type: RecordSelector - extractor: - type: DpathExtractor - field_path: - - data - paginator: - type: DefaultPaginator - page_token_option: - type: RequestPath - page_size_option: - type: RequestOption - field_name: per_page - inject_into: request_parameter - pagination_strategy: - type: CursorPagination - page_size: 100 - cursor_value: '{{ response.get("links", {}).get("next", {}) }}' - stop_condition: '{{ not response.get("links", {}).get("next", {}) }}' + $ref: "#/definitions/relationship_types_stream" + +streams: + - "#/definitions/limits_stream" + - "#/definitions/object_types_stream" + - "#/definitions/object_records_stream" + - "#/definitions/object_type_policies_stream" + - "#/definitions/relationship_types_stream" + - "#/definitions/relationship_records_stream" spec: - documentation_url: https://example.org + documentation_url: https://docs.airbyte.com/integrations/sources/zendesk_sunshine type: Spec connection_specification: $schema: http://json-schema.org/draft-07/schema# @@ -495,25 +187,121 @@ spec: - start_date - subdomain properties: + subdomain: + type: string + order: 0 + title: Subdomain + description: The subdomain for your Zendesk Account. start_date: type: string title: Start date format: date-time + description: The date from which you'd like to replicate data for Zendesk Sunshine API, in the format YYYY-MM-DDT00:00:00Z. pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ - order: 0 - username: - type: string - title: Username + examples: + - '2021-01-01T00:00:00Z' order: 1 - password: - type: string - title: Password - always_show: true - airbyte_secret: true - order: 2 - subdomain: - type: string - order: 3 - title: Subdomain - description: The subdomain for your Zendesk Account. - + credentials: + title: Authorization Method + type: object + oneOf: + - type: object + title: OAuth2.0 + required: + - auth_method + - client_id + - client_secret + - access_token + properties: + auth_method: + type: string + const: oauth2.0 + enum: + - oauth2.0 + default: oauth2.0 + order: 0 + client_id: + type: string + title: Client ID + description: The Client ID of your OAuth application. + airbyte_secret: true + client_secret: + type: string + title: Client Secret + description: The Client Secret of your OAuth application. + airbyte_secret: true + access_token: + type: string + title: Access Token + description: Long-term access Token for making authenticated requests. + airbyte_secret: true + - type: object + title: API Token + required: + - auth_method + - api_token + - email + properties: + auth_method: + type: string + const: api_token + enum: + - api_token + default: api_token + order: 1 + api_token: + type: string + title: API Token + description: API Token. See the docs + for information on how to generate this key. + airbyte_secret: true + email: + type: string + title: Email + description: The user email for your Zendesk account + advanced_auth: + auth_flow_type: oauth2.0 + predicate_key: + - credentials + - auth_method + predicate_value: oauth2.0 + oauth_config_specification: + complete_oauth_output_specification: + type: object + additionalProperties: false + properties: + access_token: + type: string + path_in_connector_config: + - credentials + - access_token + complete_oauth_server_input_specification: + type: object + additionalProperties: false + properties: + client_id: + type: string + client_secret: + type: string + complete_oauth_server_output_specification: + type: object + additionalProperties: false + properties: + client_id: + type: string + path_in_connector_config: + - credentials + - client_id + client_secret: + type: string + path_in_connector_config: + - credentials + - client_secret + oauth_user_input_from_connector_config_specification: + type: object + additionalProperties: false + properties: + subdomain: + type: string + path_in_connector_config: + - subdomain diff --git a/docs/integrations/sources/zendesk-sunshine.md b/docs/integrations/sources/zendesk-sunshine.md index 8a68798ba6cad..0b957eee840e2 100644 --- a/docs/integrations/sources/zendesk-sunshine.md +++ b/docs/integrations/sources/zendesk-sunshine.md @@ -64,6 +64,7 @@ We recommend creating a restricted, read-only key specifically for Airbyte acces | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.2.0 | 2023-08-22 | [29310](https://github.com/airbytehq/airbyte/pull/29310) | Migrate Python CDK to Low Code | | 0.1.2 | 2023-08-15 | [7976](https://github.com/airbytehq/airbyte/pull/7976) | Fix schemas and tests | | 0.1.1 | 2021-11-15 | [7976](https://github.com/airbytehq/airbyte/pull/7976) | Add oauth2.0 support | | 0.1.0 | 2021-07-08 | [4359](https://github.com/airbytehq/airbyte/pull/4359) | Initial Release | From 8d869da015ccf5cc1914c40537ce1bd1abf00027 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Tue, 22 Aug 2023 16:42:25 -0300 Subject: [PATCH 3/5] fix dockerfile and metadata --- .../connectors/source-zendesk-sunshine/Dockerfile | 2 -- .../connectors/source-zendesk-sunshine/metadata.yaml | 2 +- .../connectors/source-zendesk-sunshine/setup.py | 3 +-- .../source_zendesk_sunshine/components.py | 2 +- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/Dockerfile b/airbyte-integrations/connectors/source-zendesk-sunshine/Dockerfile index 26c8be2c04912..ce27f0f1141e3 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/Dockerfile +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/Dockerfile @@ -16,8 +16,6 @@ COPY --from=builder /install /usr/local COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime RUN echo "Etc/UTC" > /etc/timezone -# build a clean environment -FROM base WORKDIR /airbyte/integration_code COPY main.py ./ COPY source_zendesk_sunshine ./source_zendesk_sunshine diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-sunshine/metadata.yaml index de852cc4aaa3c..ceb4dfb3a5549 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/metadata.yaml @@ -21,7 +21,7 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-sunshine tags: - - language:lowcode + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/setup.py b/airbyte-integrations/connectors/source-zendesk-sunshine/setup.py index b3f1345f3ac01..1c47ce8ab0a99 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/setup.py +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/setup.py @@ -6,14 +6,13 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.2", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/components.py b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/components.py index 11e569eac735f..a74659d52544a 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/components.py +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/components.py @@ -19,4 +19,4 @@ def __new__(cls, basic_auth, oauth2, config, *args, **kwargs): if config["credentials"]["email"] and config["credentials"]["api_token"]: return basic_auth else: - return oauth2 \ No newline at end of file + return oauth2 From 2c578610ac274b52652ea684782225b8ce507f76 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Tue, 22 Aug 2023 17:06:18 -0300 Subject: [PATCH 4/5] udpate acept test yaml file --- .../acceptance-test-config.yml | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-config.yml b/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-config.yml index e41c808f964af..6f867969ea6c4 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-config.yml @@ -1,6 +1,7 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-zendesk-sunshine:dev +test_strictness_level: low acceptance_tests: spec: tests: @@ -20,20 +21,22 @@ acceptance_tests: - config_path: "integration_tests/invalid_config_oauth.json" status: "failed" discovery: - - config_path: "secrets/config.json" - backward_compatibility_tests_config: - disable_for_version: "0.1.1" + tests: + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.1" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - - config_path: "secrets/config_api_token.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - - config_path: "secrets/config_oauth.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - # incremental: # complex state ( {parent_id: {cur_field: value}} still not supported ) - # - config_path: "secrets/config.json" - # configured_catalog_path: "integration_tests/configured_catalog.json" - # future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config_api_token.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config_oauth.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + # incremental: # complex state ( {parent_id: {cur_field: value}} still not supported ) + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" From c4050602a1d21ae2b601e6b81f245f9872ad540a Mon Sep 17 00:00:00 2001 From: Mal Hancock Date: Thu, 24 Aug 2023 11:58:18 -0700 Subject: [PATCH 5/5] fix auth, object_records stream, and tests --- .../acceptance-test-config.yml | 13 ++----------- .../source_zendesk_sunshine/components.py | 2 +- .../source_zendesk_sunshine/manifest.yaml | 10 +++++++--- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-config.yml b/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-config.yml index 6f867969ea6c4..c418358b0f897 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-config.yml @@ -3,32 +3,23 @@ connector_image: airbyte/source-zendesk-sunshine:dev test_strictness_level: low acceptance_tests: - spec: - tests: - - spec_path: "source_zendesk_sunshine/spec.yaml" connection: tests: - - config_path: "secrets/config.json" - status: "succeed" - config_path: "secrets/config_oauth.json" status: "succeed" - config_path: "secrets/config_api_token.json" status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" - config_path: "integration_tests/invalid_config_api_token.json" status: "failed" - config_path: "integration_tests/invalid_config_oauth.json" status: "failed" discovery: tests: - - config_path: "secrets/config.json" + - config_path: "secrets/config_oauth.json" backward_compatibility_tests_config: disable_for_version: "0.1.1" basic_read: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - config_path: "secrets/config_api_token.json" configured_catalog_path: "integration_tests/configured_catalog.json" - config_path: "secrets/config_oauth.json" @@ -39,5 +30,5 @@ acceptance_tests: # future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - - config_path: "secrets/config.json" + - config_path: "secrets/config_oauth.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/components.py b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/components.py index a74659d52544a..23a10b7587fb7 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/components.py +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/components.py @@ -16,7 +16,7 @@ class AuthenticatorZendeskSunshine(DeclarativeAuthenticator): oauth2: BearerAuthenticator def __new__(cls, basic_auth, oauth2, config, *args, **kwargs): - if config["credentials"]["email"] and config["credentials"]["api_token"]: + if config["credentials"]["auth_method"] == "api_token": return basic_auth else: return oauth2 diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/manifest.yaml b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/manifest.yaml index 2e0803950c767..071a829680091 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/manifest.yaml +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/manifest.yaml @@ -87,10 +87,14 @@ definitions: requester: $ref: "#/definitions/requester" path: objects/query - request_parameters: - query: '{"_type": {"$eq": {{ stream_partition.type }} }' + http_method: POST + request_body_json: + query: + _type: + $eq: '{{ stream_partition.type }}' sort_by: _updated_at asc - _updated_at: '{ "start": {{ stream_interval.start_time }} }' + _updated_at: + start: '{{ stream_interval.start_time.strftime(''%Y-%m-%d %H:%M:%s.%f'')[:-3] }}' record_selector: $ref: "#/definitions/selector" paginator: