From a2e2200125c152657f92d90f70cbc1a288ffd0ab Mon Sep 17 00:00:00 2001 From: Vitalii Vdovenko Date: Thu, 21 Oct 2021 19:28:13 +0300 Subject: [PATCH 1/3] Drift #5190 - migrate to CDK, add acceptance tests --- .../connectors/source-drift/.dockerignore | 7 ++++ .../connectors/source-drift/Dockerfile | 40 ++++++++++++++----- .../source-drift/acceptance-test-config.yml | 20 ++++++++++ .../source-drift/acceptance-test-docker.sh | 16 ++++++++ .../connectors/source-drift/build.gradle | 11 +---- .../integration_tests/__init__.py | 3 ++ .../integration_tests/acceptance.py | 14 +++++++ .../integration_tests/catalog.json | 13 ++++++ .../integration_tests/configured_catalog.json | 31 ++++++++++++++ .../integration_tests/invalid_config.json | 3 ++ .../integration_tests/sample_config.json | 3 ++ .../source-drift/{main_dev.py => main.py} | 2 +- .../connectors/source-drift/requirements.txt | 3 +- .../connectors/source-drift/setup.py | 15 ++++++- .../source_drift/client/client.py | 2 +- .../source-drift/source_drift/source.py | 2 +- 16 files changed, 159 insertions(+), 26 deletions(-) create mode 100644 airbyte-integrations/connectors/source-drift/.dockerignore create mode 100644 airbyte-integrations/connectors/source-drift/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-drift/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-drift/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-drift/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-drift/integration_tests/catalog.json create mode 100644 airbyte-integrations/connectors/source-drift/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-drift/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-drift/integration_tests/sample_config.json rename airbyte-integrations/connectors/source-drift/{main_dev.py => main.py} (82%) diff --git a/airbyte-integrations/connectors/source-drift/.dockerignore b/airbyte-integrations/connectors/source-drift/.dockerignore new file mode 100644 index 0000000000000..3d4b2b31af5b5 --- /dev/null +++ b/airbyte-integrations/connectors/source-drift/.dockerignore @@ -0,0 +1,7 @@ +* +!Dockerfile +!Dockerfile.test +!main.py +!source_drift +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-drift/Dockerfile b/airbyte-integrations/connectors/source-drift/Dockerfile index 33be13ad04ecd..3c010886e1289 100644 --- a/airbyte-integrations/connectors/source-drift/Dockerfile +++ b/airbyte-integrations/connectors/source-drift/Dockerfile @@ -1,18 +1,38 @@ -FROM airbyte/integration-base-python:0.1.1 +FROM python:3.7.11-alpine3.14 as base -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base -ENV CODE_PATH="source_drift" -ENV AIRBYTE_IMPL_MODULE="source_drift" -ENV AIRBYTE_IMPL_PATH="SourceDrift" -WORKDIR /airbyte/integration_code -COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ -RUN pip install . +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_drift ./source_drift -ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh" +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] LABEL io.airbyte.version=0.2.2 LABEL io.airbyte.name=airbyte/source-drift diff --git a/airbyte-integrations/connectors/source-drift/acceptance-test-config.yml b/airbyte-integrations/connectors/source-drift/acceptance-test-config.yml new file mode 100644 index 0000000000000..008cf40c888c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-drift/acceptance-test-config.yml @@ -0,0 +1,20 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-drift:dev +tests: + spec: + - spec_path: "source_drift/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: ["accounts", ] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-drift/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-drift/acceptance-test-docker.sh new file mode 100644 index 0000000000000..e4d8b1cef8961 --- /dev/null +++ b/airbyte-integrations/connectors/source-drift/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-drift/build.gradle b/airbyte-integrations/connectors/source-drift/build.gradle index cbea73a13ed53..f88c8f7fa9b06 100644 --- a/airbyte-integrations/connectors/source-drift/build.gradle +++ b/airbyte-integrations/connectors/source-drift/build.gradle @@ -1,20 +1,13 @@ plugins { id 'airbyte-python' id 'airbyte-docker' - id 'airbyte-standard-source-test-file' + id 'airbyte-source-acceptance-test' } airbytePython { moduleDirectory 'source_drift' } -airbyteStandardSourceTestFile { - specPath = "source_drift/spec.json" - configPath = "secrets/config.json" - configuredCatalogPath = "sample_files/configured_catalog.json" -} - dependencies { - implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) - implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) + implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-drift/integration_tests/__init__.py b/airbyte-integrations/connectors/source-drift/integration_tests/__init__.py new file mode 100644 index 0000000000000..46b7376756ec6 --- /dev/null +++ b/airbyte-integrations/connectors/source-drift/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-drift/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-drift/integration_tests/acceptance.py new file mode 100644 index 0000000000000..108075487440f --- /dev/null +++ b/airbyte-integrations/connectors/source-drift/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """ This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-drift/integration_tests/catalog.json b/airbyte-integrations/connectors/source-drift/integration_tests/catalog.json new file mode 100644 index 0000000000000..3fdb9270c00d3 --- /dev/null +++ b/airbyte-integrations/connectors/source-drift/integration_tests/catalog.json @@ -0,0 +1,13 @@ +{ + "streams": [ + { + "stream": { + "name": "accounts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-drift/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-drift/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..abfe040b2667f --- /dev/null +++ b/airbyte-integrations/connectors/source-drift/integration_tests/configured_catalog.json @@ -0,0 +1,31 @@ +{ + "streams": [ + { + "stream": { + "name": "accounts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "conversations", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-drift/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-drift/integration_tests/invalid_config.json new file mode 100644 index 0000000000000..d5d7fea32f7c8 --- /dev/null +++ b/airbyte-integrations/connectors/source-drift/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "access_token": "invalid_access_token" +} diff --git a/airbyte-integrations/connectors/source-drift/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-drift/integration_tests/sample_config.json new file mode 100644 index 0000000000000..cbf0e0e8fc258 --- /dev/null +++ b/airbyte-integrations/connectors/source-drift/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "access_token": "1234567890abcdefghijk" +} diff --git a/airbyte-integrations/connectors/source-drift/main_dev.py b/airbyte-integrations/connectors/source-drift/main.py similarity index 82% rename from airbyte-integrations/connectors/source-drift/main_dev.py rename to airbyte-integrations/connectors/source-drift/main.py index 6ca2480d5ffba..88080fd895a53 100644 --- a/airbyte-integrations/connectors/source-drift/main_dev.py +++ b/airbyte-integrations/connectors/source-drift/main.py @@ -5,7 +5,7 @@ import sys -from base_python.entrypoint import launch +from airbyte_cdk.entrypoint import launch from source_drift import SourceDrift if __name__ == "__main__": diff --git a/airbyte-integrations/connectors/source-drift/requirements.txt b/airbyte-integrations/connectors/source-drift/requirements.txt index dd447512e620a..7be17a56d745d 100644 --- a/airbyte-integrations/connectors/source-drift/requirements.txt +++ b/airbyte-integrations/connectors/source-drift/requirements.txt @@ -1,4 +1,3 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/airbyte-protocol --e ../../bases/base-python +-e ../../bases/source-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-drift/setup.py b/airbyte-integrations/connectors/source-drift/setup.py index 60fdd93f88ec9..1c58185fe1a91 100644 --- a/airbyte-integrations/connectors/source-drift/setup.py +++ b/airbyte-integrations/connectors/source-drift/setup.py @@ -5,12 +5,23 @@ from setuptools import find_packages, setup +MAIN_REQUIREMENTS = ["airbyte-cdk", "requests~=2.22"] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + setup( name="source_drift", description="Source implementation for Drift.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-protocol", "base-python", "requests~=2.22", "pytest==6.1.2"], - package_data={"": ["*.json", "schemas/*.json"]}, + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, ) diff --git a/airbyte-integrations/connectors/source-drift/source_drift/client/client.py b/airbyte-integrations/connectors/source-drift/source_drift/client/client.py index e3f0e03c868d1..6a88d740f7634 100644 --- a/airbyte-integrations/connectors/source-drift/source_drift/client/client.py +++ b/airbyte-integrations/connectors/source-drift/source_drift/client/client.py @@ -5,7 +5,7 @@ from typing import Iterator, Tuple -from base_python import BaseClient +from airbyte_cdk.sources.deprecated.client import BaseClient from .api import APIClient from .common import AuthError, ValidationError diff --git a/airbyte-integrations/connectors/source-drift/source_drift/source.py b/airbyte-integrations/connectors/source-drift/source_drift/source.py index 2608aef85028d..995fb5f2fa0dc 100644 --- a/airbyte-integrations/connectors/source-drift/source_drift/source.py +++ b/airbyte-integrations/connectors/source-drift/source_drift/source.py @@ -3,7 +3,7 @@ # -from base_python import BaseSource +from airbyte_cdk.sources.deprecated.base_source import BaseSource from .client import Client From 4a34fb1a4684de3ba5b496c582f97fff53302a59 Mon Sep 17 00:00:00 2001 From: Vitalii Vdovenko Date: Fri, 22 Oct 2021 17:33:52 +0300 Subject: [PATCH 2/3] Source Drift #7041 - fixing according to PR review --- .../connectors/source-drift/.dockerignore | 1 - .../integration_tests/catalog.json | 166 +++++++++++++++++- .../connectors/source-drift/setup.py | 2 +- 3 files changed, 161 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/connectors/source-drift/.dockerignore b/airbyte-integrations/connectors/source-drift/.dockerignore index 3d4b2b31af5b5..0e6365a8f7a06 100644 --- a/airbyte-integrations/connectors/source-drift/.dockerignore +++ b/airbyte-integrations/connectors/source-drift/.dockerignore @@ -1,6 +1,5 @@ * !Dockerfile -!Dockerfile.test !main.py !source_drift !setup.py diff --git a/airbyte-integrations/connectors/source-drift/integration_tests/catalog.json b/airbyte-integrations/connectors/source-drift/integration_tests/catalog.json index 3fdb9270c00d3..54a9f62bbcd36 100644 --- a/airbyte-integrations/connectors/source-drift/integration_tests/catalog.json +++ b/airbyte-integrations/connectors/source-drift/integration_tests/catalog.json @@ -1,13 +1,167 @@ { "streams": [ { - "stream": { - "name": "accounts", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "name": "accounts", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ownerId": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "domain": { + "type": "string" + }, + "accountId": { + "type": "string" + }, + "customProperties": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "name": { + "type": "string" + }, + "value": {}, + "type": { + "type": "string" + } + } + } + }, + "deleted": { + "type": "boolean" + }, + "createDateTime": { + "type": "integer" + }, + "updateDateTime": { + "type": "integer" + }, + "targeted": { + "type": "boolean" + } + } }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + { + "name": "conversations", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "participants": { + "type": "array", + "items": { + "type": "integer" + } + }, + "status": { + "type": "string", + "enum": ["open", "closed", "pending", "bulk_sent"] + }, + "contactId": { + "type": "integer" + }, + "inboxId": { + "type": "integer" + }, + "createdAt": { + "type": "integer" + }, + "updatedAt": { + "type": "integer" + }, + "relatedPlaybookId": { + "type": ["null", "string"] + }, + "conversationTags": { + "type": "array", + "items": { + "type": "object", + "properties": { + "color": { + "type": "string", + "description": "HEX value" + }, + "name": { + "type": "string" + } + } + } + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + { + "name": "users", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "orgId": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "alias": { + "type": "string" + }, + "email": { + "type": "string" + }, + "phone": { + "type": "string" + }, + "locale": { + "type": "string" + }, + "availability": { + "type": "string" + }, + "role": { + "type": "string" + }, + "timeZone": { + "type": "string" + }, + "avatarUrl": { + "type": "string" + }, + "verified": { + "type": "boolean" + }, + "bot": { + "type": "boolean" + }, + "createdAt": { + "type": "integer" + }, + "updatedAt": { + "type": "integer" + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false } ] } diff --git a/airbyte-integrations/connectors/source-drift/setup.py b/airbyte-integrations/connectors/source-drift/setup.py index 1c58185fe1a91..26d968fc43858 100644 --- a/airbyte-integrations/connectors/source-drift/setup.py +++ b/airbyte-integrations/connectors/source-drift/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk", "requests~=2.22"] +MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "requests~=2.22"] TEST_REQUIREMENTS = [ "pytest~=6.1", From be7cf9976255ae00d33139a2e5cf5f42d0b519c4 Mon Sep 17 00:00:00 2001 From: Vitalii Vdovenko Date: Wed, 27 Oct 2021 16:52:22 +0300 Subject: [PATCH 3/3] Source Drift #7041 - bump version and update changelogs --- .../445831eb-78db-4b1f-8f1f-0d96ad8739e2.json | 2 +- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-integrations/connectors/source-drift/Dockerfile | 2 +- docs/integrations/sources/drift.md | 5 +++++ 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/445831eb-78db-4b1f-8f1f-0d96ad8739e2.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/445831eb-78db-4b1f-8f1f-0d96ad8739e2.json index d61791e94b33b..2fe62dfeeb923 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/445831eb-78db-4b1f-8f1f-0d96ad8739e2.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/445831eb-78db-4b1f-8f1f-0d96ad8739e2.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "445831eb-78db-4b1f-8f1f-0d96ad8739e2", "name": "Drift", "dockerRepository": "airbyte/source-drift", - "dockerImageTag": "0.2.2", + "dockerImageTag": "0.2.3", "documentationUrl": "https://docs.airbyte.io/integrations/sources/drift", "icon": "drift.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index e8dbbf97fd9e3..018f96b87aa89 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -117,7 +117,7 @@ - name: Drift sourceDefinitionId: 445831eb-78db-4b1f-8f1f-0d96ad8739e2 dockerRepository: airbyte/source-drift - dockerImageTag: 0.2.2 + dockerImageTag: 0.2.3 documentationUrl: https://docs.airbyte.io/integrations/sources/drift icon: drift.svg sourceType: api diff --git a/airbyte-integrations/connectors/source-drift/Dockerfile b/airbyte-integrations/connectors/source-drift/Dockerfile index 3c010886e1289..ebc1d1beda2d9 100644 --- a/airbyte-integrations/connectors/source-drift/Dockerfile +++ b/airbyte-integrations/connectors/source-drift/Dockerfile @@ -34,5 +34,5 @@ COPY source_drift ./source_drift ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.2 +LABEL io.airbyte.version=0.2.3 LABEL io.airbyte.name=airbyte/source-drift diff --git a/docs/integrations/sources/drift.md b/docs/integrations/sources/drift.md index c920707f8609d..3cf030b5b3d89 100644 --- a/docs/integrations/sources/drift.md +++ b/docs/integrations/sources/drift.md @@ -41,3 +41,8 @@ The Drift connector should not run into Drift API limitations under normal usage Follow Drift's [Setting Things Up ](https://devdocs.drift.com/docs/quick-start)guide for a more detailed description of how to obtain the API token. +## CHANGELOG + +| Version | Date | Pull Request | Subject | +| :--- | :--- | :--- | :--- | +| `0.2.3` | 2021-10-27 | [7247](https://github.com/airbytehq/airbyte/pull/7247) | Migrate to the CDK | \ No newline at end of file