🎉 Source Google Directory: migrate to the CDK

* Source Google Directory airbytehq#7415 - migrate to the CDK

* Source Google Directory airbytehq#5190 - fix timeout error

* Source Google Directory airbytehq#7415 - fix according to PR review

* Source Google Directory airbytehq#7415 - added etag and lastLoginTime to ignored fields for full refresh acceptance test

* Source Google Directory airbytehq#7415 - fix full refresh acceptance test config

* Source Google Directory airbytehq#7415 - bump version

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d19ae824-e289-4b14-995a-0632eb46d246.json

@@ -2,6 +2,6 @@
   "sourceDefinitionId": "d19ae824-e289-4b14-995a-0632eb46d246",
   "name": "Google Directory",
   "dockerRepository": "airbyte/source-google-directory",
-  "dockerImageTag": "0.1.5",
+  "dockerImageTag": "0.1.6",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-directory"
 }

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -192,7 +192,7 @@
 - name: Google Directory
   sourceDefinitionId: d19ae824-e289-4b14-995a-0632eb46d246
   dockerRepository: airbyte/source-google-directory
-  dockerImageTag: 0.1.5
+  dockerImageTag: 0.1.6
   documentationUrl: https://docs.airbyte.io/integrations/sources/google-directory
   sourceType: api
 - name: Google Search Console

airbyte-integrations/connectors/source-google-directory/.dockerignore

@@ -1,6 +1,6 @@
 *
 !Dockerfile
-!Dockerfile.test
+!main.py
 !source_google_directory
 !setup.py
 !secrets

airbyte-integrations/connectors/source-google-directory/Dockerfile

@@ -1,18 +1,38 @@
-FROM airbyte/integration-base-python:0.1.1
+FROM python:3.7.11-alpine3.14 as base
 
-# Bash is installed for more convenient debugging.
-RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/*
+# build and load all requirements
+FROM base as builder
+WORKDIR /airbyte/integration_code
+
+# upgrade pip to the latest version
+RUN apk --no-cache upgrade \
+    && pip install --upgrade pip \
+    && apk --no-cache add tzdata build-base
 
-ENV CODE_PATH="source_google_directory"
-ENV AIRBYTE_IMPL_MODULE="source_google_directory"
-ENV AIRBYTE_IMPL_PATH="SourceGoogleDirectory"
 
-WORKDIR /airbyte/integration_code
-COPY $CODE_PATH ./$CODE_PATH
 COPY setup.py ./
-RUN pip install ".[main]"
+# install necessary packages to a temporary folder
+RUN pip install --prefix=/install .
+
+# build a clean environment
+FROM base
+WORKDIR /airbyte/integration_code
+
+# copy all loaded and built libraries to a pure basic image
+COPY --from=builder /install /usr/local
+# add default timezone settings
+COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime
+RUN echo "Etc/UTC" > /etc/timezone
+
+# bash is installed for more convenient debugging.
+RUN apk --no-cache add bash
+
+# copy payload code only
+COPY main.py ./
+COPY source_google_directory ./source_google_directory
 
-ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh"
+ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
+ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.5
+LABEL io.airbyte.version=0.1.6
 LABEL io.airbyte.name=airbyte/source-google-directory

airbyte-integrations/connectors/source-google-directory/acceptance-test-config.yml

@@ -0,0 +1,22 @@
+# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference)
+# for more information about how to configure these tests
+connector_image: airbyte/source-google-directory:dev
+tests:
+  spec:
+    - spec_path: "source_google_directory/spec.json"
+  connection:
+    - config_path: "secrets/config.json"
+      status: "succeed"
+    - config_path: "integration_tests/invalid_config.json"
+      status: "failed"
+  discovery:
+    - config_path: "secrets/config.json"
+  basic_read:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
+  full_refresh:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
+      # API returns different lastLoginTime for some users, eteg is generated on all data and also same time different
+      ignored_fields:
+        "users": ["etag", "lastLoginTime"]

airbyte-integrations/connectors/source-google-directory/acceptance-test-docker.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env sh
+
+# Build latest connector image
+docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2)
+
+# Pull latest acctest image
+docker pull airbyte/source-acceptance-test:latest
+
+# Run
+docker run --rm -it \
+    -v /var/run/docker.sock:/var/run/docker.sock \
+    -v /tmp:/tmp \
+    -v $(pwd):/test_input \
+    airbyte/source-acceptance-test \
+    --acceptance-test-config /test_input
+

airbyte-integrations/connectors/source-google-directory/build.gradle

@@ -1,22 +1,9 @@
 plugins {
     id 'airbyte-python'
     id 'airbyte-docker'
-    id 'airbyte-standard-source-test-file'
+    id 'airbyte-source-acceptance-test'
 }
 
 airbytePython {
     moduleDirectory 'source_google_directory'
 }
-
-airbyteStandardSourceTestFile {
-    // For more information on standard source tests, see https://docs.airbyte.io/connector-development/testing-connectors
-    specPath = "source_google_directory/spec.json"
-    configPath = "secrets/config.json"
-    configuredCatalogPath = "sample_files/configured_catalog.json"
-}
-
-
-dependencies {
-    implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs)
-    implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs)
-}

airbyte-integrations/connectors/source-google-directory/integration_tests/__init__.py

@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#

airbyte-integrations/connectors/source-google-directory/integration_tests/acceptance.py

@@ -0,0 +1,14 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+
+
+import pytest
+
+pytest_plugins = ("source_acceptance_test.plugin",)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def connector_setup():
+    """ This fixture is a placeholder for external resources that acceptance test might require."""
+    yield

airbyte-integrations/connectors/source-google-directory/integration_tests/configured_catalog.json

@@ -0,0 +1,34 @@
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "users",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": false
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "groups",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": false
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "group_members",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": false
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    }
+  ]
+}

airbyte-integrations/connectors/source-google-directory/integration_tests/invalid_config.json

@@ -0,0 +1,4 @@
+{
+  "credentials_json": "{}",
+  "email": "test@test.test"
+}

airbyte-integrations/connectors/source-google-directory/integration_tests/sample_config.json

@@ -0,0 +1,4 @@
+{
+  "credentials_json": "<The contents of the JSON service account key>",
+  "email": "test@test.test"
+}

airbyte-integrations/connectors/source-google-directory/main_dev.py → airbyte-integrations/connectors/source-google-directory/main.py

@@ -5,7 +5,7 @@
 
 import sys
 
-from base_python.entrypoint import launch
+from airbyte_cdk.entrypoint import launch
 from source_google_directory import SourceGoogleDirectory
 
 if __name__ == "__main__":

airbyte-integrations/connectors/source-google-directory/requirements.txt

@@ -1,4 +1,2 @@
--e ../../bases/airbyte-protocol
--e ../../bases/base-python
--e ../../bases/base-python-test
+-e ../../bases/source-acceptance-test
 -e .

airbyte-integrations/connectors/source-google-directory/setup.py

@@ -5,27 +5,29 @@
 
 from setuptools import find_packages, setup
 
+MAIN_REQUIREMENTS = [
+    "airbyte-cdk~=0.1",
+    "google-api-python-client==1.12.8",
+    "google-auth-httplib2==0.0.4",
+    "google-auth-oauthlib==0.4.2",
+    "backoff==1.10.0",
+]
+
+TEST_REQUIREMENTS = [
+    "pytest~=6.1",
+    "pytest-mock~=3.6.1",
+    "source-acceptance-test",
+]
+
 setup(
     name="source_google_directory",
     description="Source implementation for Google Directory.",
     author="Airbyte",
     author_email="contact@airbyte.io",
     packages=find_packages(),
-    install_requires=[
-        "airbyte-protocol",
-        "base-python",
-        "google-api-python-client==1.12.8",
-        "google-auth-httplib2==0.0.4",
-        "google-auth-oauthlib==0.4.2",
-        "backoff==1.10.0",
-    ],
-    package_data={"": ["*.json", "schemas/*.json"]},
-    setup_requires=["pytest-runner"],
-    tests_require=["pytest"],
+    install_requires=MAIN_REQUIREMENTS,
+    package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]},
     extras_require={
-        # Dependencies required by the main package but not integration tests should go in main. Deps required by
-        # integration tests but not the main package go in tests. Deps required by both should go in
-        # install_requires.
-        "tests": ["airbyte-python-test", "pytest"],
+        "tests": TEST_REQUIREMENTS,
     },
 )

airbyte-integrations/connectors/source-google-directory/source_google_directory/api.py

@@ -10,7 +10,7 @@
 
 import backoff
 from google.oauth2 import service_account
-from googleapiclient.discovery import Resource, build
+from googleapiclient.discovery import build
 from googleapiclient.errors import HttpError as GoogleApiHttpError
 
 from .utils import rate_limit_handling
@@ -23,25 +23,26 @@ def __init__(self, credentials_json: str, email: str):
         self._creds = None
         self._credentials_json = credentials_json
         self._admin_email = email
+        self._service = None
 
     def _load_account_info(self) -> Dict:
         account_info = json.loads(self._credentials_json)
         return account_info
 
-    def _obtain_creds(self) -> service_account.Credentials:
+    def _obtain_creds(self):
         account_info = self._load_account_info()
         creds = service_account.Credentials.from_service_account_info(account_info, scopes=SCOPES)
         self._creds = creds.with_subject(self._admin_email)
 
-    def _construct_resource(self) -> Resource:
+    def _construct_resource(self):
         if not self._creds:
             self._obtain_creds()
-        service = build("admin", "directory_v1", credentials=self._creds)
-        return service
+        if not self._service:
+            self._service = build("admin", "directory_v1", credentials=self._creds)
 
     def _get_resource(self, name: str):
-        service = self._construct_resource()
-        return getattr(service, name)
+        self._construct_resource()
+        return getattr(self._service, name)
 
     @backoff.on_exception(backoff.expo, GoogleApiHttpError, max_tries=7, giveup=rate_limit_handling)
     def get(self, name: str, params: Dict = None) -> Dict:

airbyte-integrations/connectors/source-google-directory/source_google_directory/client.py

@@ -5,7 +5,7 @@
 
 from typing import Any, Mapping, Tuple
 
-from base_python import BaseClient
+from airbyte_cdk.sources.deprecated.client import BaseClient
 
 from .api import API, GroupMembersAPI, GroupsAPI, UsersAPI
 

airbyte-integrations/connectors/source-google-directory/source_google_directory/schemas/group_members.json

@@ -3,19 +3,19 @@
   "type": "object",
   "properties": {
     "kind": {
-      "type": "string"
+      "type": ["null", "string"]
     },
     "id": {
-      "type": "string"
+      "type": ["null", "string"]
     },
     "email": {
-      "type": "string"
+      "type": ["null", "string"]
     },
     "role": {
-      "type": "string"
+      "type": ["null", "string"]
     },
     "type": {
-      "type": "string"
+      "type": ["null", "string"]
     }
   }
 }

airbyte-integrations/connectors/source-google-directory/source_google_directory/schemas/groups.json

@@ -3,28 +3,28 @@
   "type": "object",
   "properties": {
     "kind": {
-      "type": "string"
+      "type": ["null", "string"]
     },
     "id": {
-      "type": "string"
+      "type": ["null", "string"]
     },
     "etag": {
-      "type": "string"
+      "type": ["null", "string"]
     },
     "email": {
-      "type": "string"
+      "type": ["null", "string"]
     },
     "name": {
-      "type": "string"
+      "type": ["null", "string"]
     },
     "directMembersCount": {
-      "type": "string"
+      "type": ["null", "string"]
     },
     "description": {
-      "type": "string"
+      "type": ["null", "string"]
     },
     "adminCreated": {
-      "type": "boolean"
+      "type": ["null", "boolean"]
     }
   }
 }