🎉 Source Looker: Migrate Looker source to CDK structure, add SAT test…

…ing (#7284)

* Migrate Looker source to CDK structure, add SAT testing

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c.json

@@ -2,7 +2,7 @@
   "sourceDefinitionId": "00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c",
   "name": "Looker",
   "dockerRepository": "airbyte/source-looker",
-  "dockerImageTag": "0.2.4",
+  "dockerImageTag": "0.2.5",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/looker",
   "icon": "looker.svg"
 }

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -292,7 +292,7 @@
 - name: Looker
   sourceDefinitionId: 00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c
   dockerRepository: airbyte/source-looker
-  dockerImageTag: 0.2.4
+  dockerImageTag: 0.2.5
   documentationUrl: https://docs.airbyte.io/integrations/sources/looker
   icon: looker.svg
   sourceType: api

airbyte-integrations/connectors/source-looker/.dockerignore

@@ -1,7 +1,6 @@
 *
 !Dockerfile
-!Dockerfile.test
+!main.py
 !source_looker
 !setup.py
-!integration_tests
 !secrets

airbyte-integrations/connectors/source-looker/Dockerfile

@@ -1,18 +1,37 @@
-FROM airbyte/integration-base-python:0.1.1
+FROM python:3.7.11-alpine3.14 as base
 
-# Bash is installed for more convenient debugging.
-RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/*
+# build and load all requirements
+FROM base as builder
+WORKDIR /airbyte/integration_code
 
-ENV CODE_PATH="source_looker"
-ENV AIRBYTE_IMPL_MODULE="source_looker"
-ENV AIRBYTE_IMPL_PATH="SourceLooker"
+# upgrade pip to the latest version
+RUN apk --no-cache upgrade \
+    && pip install --upgrade pip \
+    && apk --no-cache add tzdata build-base
 
-WORKDIR /airbyte/integration_code
-COPY $CODE_PATH ./$CODE_PATH
 COPY setup.py ./
-RUN pip install .
+# install necessary packages to a temporary folder
+RUN pip install --prefix=/install .
+
+# build a clean environment
+FROM base
+WORKDIR /airbyte/integration_code
+
+# copy all loaded and built libraries to a pure basic image
+COPY --from=builder /install /usr/local
+# add default timezone settings
+COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime
+RUN echo "Etc/UTC" > /etc/timezone
+
+# bash is installed for more convenient debugging.
+RUN apk --no-cache add bash
+
+# copy payload code only
+COPY main.py ./
+COPY source_looker ./source_looker
 
-ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh"
+ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
+ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.4
+LABEL io.airbyte.version=0.2.5
 LABEL io.airbyte.name=airbyte/source-looker

airbyte-integrations/connectors/source-looker/README.md

@@ -30,7 +30,9 @@ If this is mumbo jumbo to you, don't worry about it, just put your deps in `setu
 should work as you expect.
 
 #### Building via Gradle
-From the Airbyte repository root, run:
+You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow.
+
+To build using Gradle, from the Airbyte repository root, run:
 ```
 ./gradlew :airbyte-integrations:connectors:source-looker:build
 ```
@@ -53,12 +55,6 @@ python main_dev.py discover --config secrets/config.json
 python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json
 ```
 
-### Unit Tests
-To run unit tests locally, from the connector directory run:
-```
-python -m pytest unit_tests
-```
-
 ### Locally running the connector docker image
 
 #### Build
@@ -83,10 +79,44 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-looker:dev discover --
 docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-looker:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json
 ```
 
+## Testing
+Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named.
+First install test dependencies into your virtual environment:
+```
+pip install .[tests]
+```
+### Unit Tests
+To run unit tests locally, from the connector directory run:
+```
+python -m pytest unit_tests
+```
+
 ### Integration Tests
-1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-looker:integrationTest` to run the standard integration test suite.
-1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`.
-   Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named.
+There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector).
+#### Custom Integration tests
+Place custom tests inside `integration_tests/` folder, then, from the connector root, run
+```
+python -m pytest integration_tests
+```
+#### Acceptance Tests
+Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information.
+If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
+To run your integration tests with acceptance tests, from the connector root, run
+```
+python -m pytest integration_tests -p integration_tests.acceptance
+```
+To run your integration tests with docker
+
+### Using gradle to run tests
+All commands should be run from airbyte project root.
+To run unit tests:
+```
+./gradlew :airbyte-integrations:connectors:source-looker:unitTest
+```
+To run acceptance and custom integration tests:
+```
+./gradlew :airbyte-integrations:connectors:source-looker:integrationTest
+```
 
 ## Dependency Management
 All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.

airbyte-integrations/connectors/source-looker/acceptance-test-config.yml

@@ -0,0 +1,29 @@
+# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference)
+# for more information about how to configure these tests
+connector_image: airbyte/source-looker:dev
+tests:
+  spec:
+    - spec_path: "source_looker/spec.json"
+  connection:
+    - config_path: "secrets/config.json"
+      status: "succeed"
+    - config_path: "integration_tests/invalid_config.json"
+      status: "failed"
+  discovery:
+    - config_path: "secrets/config.json"
+  basic_read:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
+      empty_streams:
+        [
+          "homepages",
+          "looks",
+          "run_looks",
+          "scheduled_plans",
+          "user_attribute_group_values",
+          "user_login_lockouts",
+          "user_sessions",
+        ]
+  full_refresh:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "integration_tests/configured_catalog_without_dynamic_streams.json"

airbyte-integrations/connectors/source-looker/acceptance-test-docker.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env sh
+
+# Build latest connector image
+docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2)
+
+# Pull latest acctest image
+docker pull airbyte/source-acceptance-test:latest
+
+# Run
+docker run --rm -it \
+    -v /var/run/docker.sock:/var/run/docker.sock \
+    -v /tmp:/tmp \
+    -v $(pwd):/test_input \
+    airbyte/source-acceptance-test \
+    --acceptance-test-config /test_input
+

airbyte-integrations/connectors/source-looker/build.gradle

@@ -1,21 +1,9 @@
 plugins {
     id 'airbyte-python'
     id 'airbyte-docker'
-    id 'airbyte-standard-source-test-file'
+    id 'airbyte-source-acceptance-test'
 }
 
 airbytePython {
     moduleDirectory 'source_looker'
 }
-
-airbyteStandardSourceTestFile {
-    specPath = "source_looker/spec.json"
-    configPath = "secrets/config.json"
-    configuredCatalogPath = "integration_tests/configured_catalog.json"
-}
-
-
-dependencies {
-    implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs)
-    implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs)
-}

airbyte-integrations/connectors/source-looker/integration_tests/__init__.py

@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#

airbyte-integrations/connectors/source-looker/integration_tests/acceptance.py

@@ -0,0 +1,16 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+
+
+import pytest
+
+pytest_plugins = ("source_acceptance_test.plugin",)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def connector_setup():
+    """ This fixture is a placeholder for external resources that acceptance test might require."""
+    # TODO: setup test dependencies if needed. otherwise remove the TODO comments
+    yield
+    # TODO: clean up test dependencies