🎉 Source Google Workspace Admin Reports: migrate to the CDK

* Source Google Workspace Admin Reports #7531 - migrate to the CDK * Source Google Workspace Admin Reports #7531 - skip full refresh test * Source Google Workspace Admin Reports #7531 - bump version and update changelog
airbytehq · Nov 8, 2021 · 3cbfb99 · 3cbfb99
1 parent 8a19833
commit 3cbfb99
Show file tree

Hide file tree

Showing 13 changed files with 56 additions and 37 deletions.
diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/ed9dfefa-1bbc-419d-8c5e-4d78f0ef6734.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/ed9dfefa-1bbc-419d-8c5e-4d78f0ef6734.json
@@ -2,6 +2,6 @@
   "sourceDefinitionId": "ed9dfefa-1bbc-419d-8c5e-4d78f0ef6734",
   "name": "Google Workspace Admin Reports",
   "dockerRepository": "airbyte/source-google-workspace-admin-reports",
-  "dockerImageTag": "0.1.5",
+  "dockerImageTag": "0.1.6",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-workspace-admin-reports"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -217,7 +217,7 @@
 - name: Google Workspace Admin Reports
   sourceDefinitionId: ed9dfefa-1bbc-419d-8c5e-4d78f0ef6734
   dockerRepository: airbyte/source-google-workspace-admin-reports
-  dockerImageTag: 0.1.5
+  dockerImageTag: 0.1.6
   documentationUrl: https://docs.airbyte.io/integrations/sources/google-workspace-admin-reports
   sourceType: api
 - name: Greenhouse

diff --git a/airbyte-integrations/connectors/source-google-workspace-admin-reports/.dockerignore b/airbyte-integrations/connectors/source-google-workspace-admin-reports/.dockerignore
@@ -1,6 +1,6 @@
 *
 !Dockerfile
-!Dockerfile.test
+!main.py
 !source_google_workspace_admin_reports
 !setup.py
 !secrets
diff --git a/airbyte-integrations/connectors/source-google-workspace-admin-reports/Dockerfile b/airbyte-integrations/connectors/source-google-workspace-admin-reports/Dockerfile
@@ -1,18 +1,38 @@
-FROM airbyte/integration-base-python:0.1.1
+FROM python:3.7.11-alpine3.14 as base
 
-# Bash is installed for more convenient debugging.
-RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/*
+# build and load all requirements
+FROM base as builder
+WORKDIR /airbyte/integration_code
+
+# upgrade pip to the latest version
+RUN apk --no-cache upgrade \
+    && pip install --upgrade pip \
+    && apk --no-cache add tzdata build-base
 
-ENV CODE_PATH="source_google_workspace_admin_reports"
-ENV AIRBYTE_IMPL_MODULE="source_google_workspace_admin_reports"
-ENV AIRBYTE_IMPL_PATH="SourceGoogleWorkspaceAdminReports"
 
-WORKDIR /airbyte/integration_code
-COPY $CODE_PATH ./$CODE_PATH
 COPY setup.py ./
-RUN pip install .
+# install necessary packages to a temporary folder
+RUN pip install --prefix=/install .
+
+# build a clean environment
+FROM base
+WORKDIR /airbyte/integration_code
+
+# copy all loaded and built libraries to a pure basic image
+COPY --from=builder /install /usr/local
+# add default timezone settings
+COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime
+RUN echo "Etc/UTC" > /etc/timezone
+
+# bash is installed for more convenient debugging.
+RUN apk --no-cache add bash
+
+# copy payload code only
+COPY main.py ./
+COPY source_google_workspace_admin_reports ./source_google_workspace_admin_reports
 
-ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh"
+ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
+ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.5
+LABEL io.airbyte.version=0.1.6
 LABEL io.airbyte.name=airbyte/source-google-workspace-admin-reports
diff --git a/...-integrations/connectors/source-google-workspace-admin-reports/acceptance-test-config.yml b/...-integrations/connectors/source-google-workspace-admin-reports/acceptance-test-config.yml
@@ -15,9 +15,11 @@ tests:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
       empty_streams: ["admin"]
-  full_refresh:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
+#  We have active test account. New records in reports appear frequently.
+#  Therefore, second activity differs from first, and it brakes test.
+#  full_refresh:
+#    - config_path: "secrets/config.json"
+#      configured_catalog_path: "integration_tests/configured_catalog.json"
   incremental:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"

diff --git a/airbyte-integrations/connectors/source-google-workspace-admin-reports/build.gradle b/airbyte-integrations/connectors/source-google-workspace-admin-reports/build.gradle
@@ -8,7 +8,3 @@ airbytePython {
     moduleDirectory 'source_google_workspace_admin_reports'
 }
 
-dependencies {
-    implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs)
-    implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs)
-}
diff --git a/...oogle-workspace-admin-reports/main_dev.py → ...ce-google-workspace-admin-reports/main.py b/...oogle-workspace-admin-reports/main_dev.py → ...ce-google-workspace-admin-reports/main.py
@@ -5,7 +5,7 @@
 
 import sys
 
-from base_python.entrypoint import launch
+from airbyte_cdk.entrypoint import launch
 from source_google_workspace_admin_reports import SourceGoogleWorkspaceAdminReports
 
 if __name__ == "__main__":

diff --git a/airbyte-integrations/connectors/source-google-workspace-admin-reports/requirements.txt b/airbyte-integrations/connectors/source-google-workspace-admin-reports/requirements.txt
@@ -1,5 +1,3 @@
 # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies.
--e ../../bases/airbyte-protocol
--e ../../bases/base-python
 -e ../../bases/source-acceptance-test
 -e .
diff --git a/airbyte-integrations/connectors/source-google-workspace-admin-reports/setup.py b/airbyte-integrations/connectors/source-google-workspace-admin-reports/setup.py
@@ -5,8 +5,18 @@
 
 from setuptools import find_packages, setup
 
+MAIN_REQUIREMENTS = [
+    "airbyte-cdk~=0.1",
+    "google-api-python-client==2.0.2",
+    "google-auth-httplib2==0.1.0",
+    "google-auth-oauthlib==0.4.3",
+    "backoff==1.10.0",
+    "pendulum==2.1.2",
+]
+
 TEST_REQUIREMENTS = [
     "pytest~=6.1",
+    "pytest-mock~=3.6.1",
     "source-acceptance-test",
 ]
 
@@ -16,16 +26,8 @@
     author="Airbyte",
     author_email="contact@airbyte.io",
     packages=find_packages(),
-    install_requires=[
-        "airbyte-protocol",
-        "base-python",
-        "google-api-python-client==2.0.2",
-        "google-auth-httplib2==0.1.0",
-        "google-auth-oauthlib==0.4.3",
-        "backoff==1.10.0",
-        "pendulum==2.1.2",
-    ],
-    package_data={"": ["*.json", "schemas/*.json"]},
+    install_requires=MAIN_REQUIREMENTS,
+    package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]},
     extras_require={
         "tests": TEST_REQUIREMENTS,
     },

diff --git a/...ectors/source-google-workspace-admin-reports/source_google_workspace_admin_reports/api.py b/...ectors/source-google-workspace-admin-reports/source_google_workspace_admin_reports/api.py
@@ -13,7 +13,7 @@
 import backoff
 import pendulum
 import pytz
-from base_python.entrypoint import logger
+from airbyte_cdk.entrypoint import logger
 from google.oauth2 import service_account
 from googleapiclient.discovery import Resource, build
 from googleapiclient.errors import HttpError as GoogleApiHttpError

diff --git a/...ors/source-google-workspace-admin-reports/source_google_workspace_admin_reports/client.py b/...ors/source-google-workspace-admin-reports/source_google_workspace_admin_reports/client.py
@@ -5,7 +5,7 @@
 
 from typing import Any, Mapping, Optional, Tuple
 
-from base_python import BaseClient
+from airbyte_cdk.sources.deprecated.client import BaseClient
 
 from .api import API, AdminAPI, DriveAPI, IncrementalStreamAPI, LoginsAPI, MobileAPI, OAuthTokensAPI
 

diff --git a/...ors/source-google-workspace-admin-reports/source_google_workspace_admin_reports/source.py b/...ors/source-google-workspace-admin-reports/source_google_workspace_admin_reports/source.py
@@ -3,7 +3,7 @@
 #
 
 
-from base_python import BaseSource
+from airbyte_cdk.sources.deprecated.base_source import BaseSource
 
 from .client import Client
 

diff --git a/docs/integrations/sources/google-workspace-admin-reports.md b/docs/integrations/sources/google-workspace-admin-reports.md
@@ -58,4 +58,5 @@ You should now be ready to use the Google Workspace Admin Reports API connector
 
 | Version | Date       | Pull Request | Subject |
 | :------ | :--------  | :-----       | :------ |
+| 0.1.6  | 2021-11-02 | [7623](https://github.com/airbytehq/airbyte/pull/7623) | Migrate to the CDK |
 | 0.1.5   | 2021-10-07 | [6878](https://github.com/airbytehq/airbyte/pull/6878) | Improve testing & output schemas |