🎉 Source Drift: migrate to the CDK

* Drift #5190 - migrate to CDK, add acceptance tests * Source Drift #7041 - fixing according to PR review * Source Drift #7041 - bump version and update changelogs
airbytehq · Oct 27, 2021 · 7592bca · 7592bca
1 parent 62832cd
commit 7592bca
Show file tree

Hide file tree

Showing 19 changed files with 320 additions and 29 deletions.
diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/445831eb-78db-4b1f-8f1f-0d96ad8739e2.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/445831eb-78db-4b1f-8f1f-0d96ad8739e2.json
@@ -2,7 +2,7 @@
   "sourceDefinitionId": "445831eb-78db-4b1f-8f1f-0d96ad8739e2",
   "name": "Drift",
   "dockerRepository": "airbyte/source-drift",
-  "dockerImageTag": "0.2.2",
+  "dockerImageTag": "0.2.3",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/drift",
   "icon": "drift.svg"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -117,7 +117,7 @@
 - name: Drift
   sourceDefinitionId: 445831eb-78db-4b1f-8f1f-0d96ad8739e2
   dockerRepository: airbyte/source-drift
-  dockerImageTag: 0.2.2
+  dockerImageTag: 0.2.3
   documentationUrl: https://docs.airbyte.io/integrations/sources/drift
   icon: drift.svg
   sourceType: api

diff --git a/airbyte-integrations/connectors/source-drift/.dockerignore b/airbyte-integrations/connectors/source-drift/.dockerignore
@@ -0,0 +1,6 @@
+*
+!Dockerfile
+!main.py
+!source_drift
+!setup.py
+!secrets
diff --git a/airbyte-integrations/connectors/source-drift/Dockerfile b/airbyte-integrations/connectors/source-drift/Dockerfile
@@ -1,18 +1,38 @@
-FROM airbyte/integration-base-python:0.1.1
+FROM python:3.7.11-alpine3.14 as base
 
-# Bash is installed for more convenient debugging.
-RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/*
+# build and load all requirements
+FROM base as builder
+WORKDIR /airbyte/integration_code
+
+# upgrade pip to the latest version
+RUN apk --no-cache upgrade \
+    && pip install --upgrade pip \
+    && apk --no-cache add tzdata build-base
 
-ENV CODE_PATH="source_drift"
-ENV AIRBYTE_IMPL_MODULE="source_drift"
-ENV AIRBYTE_IMPL_PATH="SourceDrift"
 
-WORKDIR /airbyte/integration_code
-COPY $CODE_PATH ./$CODE_PATH
 COPY setup.py ./
-RUN pip install .
+# install necessary packages to a temporary folder
+RUN pip install --prefix=/install .
+
+# build a clean environment
+FROM base
+WORKDIR /airbyte/integration_code
+
+# copy all loaded and built libraries to a pure basic image
+COPY --from=builder /install /usr/local
+# add default timezone settings
+COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime
+RUN echo "Etc/UTC" > /etc/timezone
+
+# bash is installed for more convenient debugging.
+RUN apk --no-cache add bash
+
+# copy payload code only
+COPY main.py ./
+COPY source_drift ./source_drift
 
-ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh"
+ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
+ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.2
+LABEL io.airbyte.version=0.2.3
 LABEL io.airbyte.name=airbyte/source-drift
diff --git a/airbyte-integrations/connectors/source-drift/acceptance-test-config.yml b/airbyte-integrations/connectors/source-drift/acceptance-test-config.yml
@@ -0,0 +1,20 @@
+# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference)
+# for more information about how to configure these tests
+connector_image: airbyte/source-drift:dev
+tests:
+  spec:
+    - spec_path: "source_drift/spec.json"
+  connection:
+    - config_path: "secrets/config.json"
+      status: "succeed"
+    - config_path: "integration_tests/invalid_config.json"
+      status: "failed"
+  discovery:
+    - config_path: "secrets/config.json"
+  basic_read:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
+      empty_streams: ["accounts", ]
+  full_refresh:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
diff --git a/airbyte-integrations/connectors/source-drift/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-drift/acceptance-test-docker.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env sh
+
+# Build latest connector image
+docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2)
+
+# Pull latest acctest image
+docker pull airbyte/source-acceptance-test:latest
+
+# Run
+docker run --rm -it \
+    -v /var/run/docker.sock:/var/run/docker.sock \
+    -v /tmp:/tmp \
+    -v $(pwd):/test_input \
+    airbyte/source-acceptance-test \
+    --acceptance-test-config /test_input
+
diff --git a/airbyte-integrations/connectors/source-drift/build.gradle b/airbyte-integrations/connectors/source-drift/build.gradle
@@ -1,20 +1,13 @@
 plugins {
     id 'airbyte-python'
     id 'airbyte-docker'
-    id 'airbyte-standard-source-test-file'
+    id 'airbyte-source-acceptance-test'
 }
 
 airbytePython {
     moduleDirectory 'source_drift'
 }
 
-airbyteStandardSourceTestFile {
-    specPath = "source_drift/spec.json"
-    configPath = "secrets/config.json"
-    configuredCatalogPath = "sample_files/configured_catalog.json"
-}
-
 dependencies {
-    implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs)
-    implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs)
+    implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs)
 }
diff --git a/airbyte-integrations/connectors/source-drift/integration_tests/__init__.py b/airbyte-integrations/connectors/source-drift/integration_tests/__init__.py
@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
diff --git a/airbyte-integrations/connectors/source-drift/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-drift/integration_tests/acceptance.py
@@ -0,0 +1,14 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+
+
+import pytest
+
+pytest_plugins = ("source_acceptance_test.plugin",)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def connector_setup():
+    """ This fixture is a placeholder for external resources that acceptance test might require."""
+    yield
diff --git a/airbyte-integrations/connectors/source-drift/integration_tests/catalog.json b/airbyte-integrations/connectors/source-drift/integration_tests/catalog.json
@@ -0,0 +1,167 @@
+{
+  "streams": [
+    {
+      "name": "accounts",
+      "json_schema": {
+        "$schema": "http://json-schema.org/draft-07/schema#",
+        "type": "object",
+        "properties": {
+          "ownerId": {
+            "type": "integer"
+          },
+          "name": {
+            "type": "string"
+          },
+          "domain": {
+            "type": "string"
+          },
+          "accountId": {
+            "type": "string"
+          },
+          "customProperties": {
+            "type": "array",
+            "items": {
+              "type": "object",
+              "properties": {
+                "label": {
+                  "type": "string"
+                },
+                "name": {
+                  "type": "string"
+                },
+                "value": {},
+                "type": {
+                  "type": "string"
+                }
+              }
+            }
+          },
+          "deleted": {
+            "type": "boolean"
+          },
+          "createDateTime": {
+            "type": "integer"
+          },
+          "updateDateTime": {
+            "type": "integer"
+          },
+          "targeted": {
+            "type": "boolean"
+          }
+        }
+      },
+      "supported_sync_modes": ["full_refresh"],
+      "source_defined_cursor": false
+    },
+    {
+      "name": "conversations",
+      "json_schema": {
+        "$schema": "http://json-schema.org/draft-07/schema#",
+        "type": "object",
+        "properties": {
+          "id": {
+            "type": "integer"
+          },
+          "participants": {
+            "type": "array",
+            "items": {
+              "type": "integer"
+            }
+          },
+          "status": {
+            "type": "string",
+            "enum": ["open", "closed", "pending", "bulk_sent"]
+          },
+          "contactId": {
+            "type": "integer"
+          },
+          "inboxId": {
+            "type": "integer"
+          },
+          "createdAt": {
+            "type": "integer"
+          },
+          "updatedAt": {
+            "type": "integer"
+          },
+          "relatedPlaybookId": {
+            "type": ["null", "string"]
+          },
+          "conversationTags": {
+            "type": "array",
+            "items": {
+              "type": "object",
+              "properties": {
+                "color": {
+                  "type": "string",
+                  "description": "HEX value"
+                },
+                "name": {
+                  "type": "string"
+                }
+              }
+            }
+          }
+        }
+      },
+      "supported_sync_modes": ["full_refresh"],
+      "source_defined_cursor": false
+    },
+    {
+      "name": "users",
+      "json_schema": {
+        "$schema": "http://json-schema.org/draft-07/schema#",
+        "type": "object",
+        "properties": {
+          "id": {
+            "type": "integer"
+          },
+          "orgId": {
+            "type": "integer"
+          },
+          "name": {
+            "type": "string"
+          },
+          "alias": {
+            "type": "string"
+          },
+          "email": {
+            "type": "string"
+          },
+          "phone": {
+            "type": "string"
+          },
+          "locale": {
+            "type": "string"
+          },
+          "availability": {
+            "type": "string"
+          },
+          "role": {
+            "type": "string"
+          },
+          "timeZone": {
+            "type": "string"
+          },
+          "avatarUrl": {
+            "type": "string"
+          },
+          "verified": {
+            "type": "boolean"
+          },
+          "bot": {
+            "type": "boolean"
+          },
+          "createdAt": {
+            "type": "integer"
+          },
+          "updatedAt": {
+            "type": "integer"
+          }
+        }
+      },
+      "supported_sync_modes": ["full_refresh"],
+      "source_defined_cursor": false
+    }
+  ]
+}
diff --git a/airbyte-integrations/connectors/source-drift/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-drift/integration_tests/configured_catalog.json
@@ -0,0 +1,31 @@
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "accounts",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "conversations",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "users",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    }
+  ]
+}
diff --git a/airbyte-integrations/connectors/source-drift/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-drift/integration_tests/invalid_config.json
@@ -0,0 +1,3 @@
+{
+  "access_token": "invalid_access_token"
+}
diff --git a/airbyte-integrations/connectors/source-drift/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-drift/integration_tests/sample_config.json
@@ -0,0 +1,3 @@
+{
+  "access_token": "1234567890abcdefghijk"
+}
diff --git a/...tions/connectors/source-drift/main_dev.py → ...egrations/connectors/source-drift/main.py b/...tions/connectors/source-drift/main_dev.py → ...egrations/connectors/source-drift/main.py
@@ -5,7 +5,7 @@
 
 import sys
 
-from base_python.entrypoint import launch
+from airbyte_cdk.entrypoint import launch
 from source_drift import SourceDrift
 
 if __name__ == "__main__":

diff --git a/airbyte-integrations/connectors/source-drift/requirements.txt b/airbyte-integrations/connectors/source-drift/requirements.txt
@@ -1,4 +1,3 @@
 # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies.
--e ../../bases/airbyte-protocol
--e ../../bases/base-python
+-e ../../bases/source-acceptance-test
 -e .