Add Singer Facebook Source

.github/workflows/gradle.yml

@@ -52,6 +52,7 @@ jobs:
           HUBSPOT_INTEGRATION_TESTS_CREDS: ${{ secrets.HUBSPOT_INTEGRATION_TESTS_CREDS }}
           GSHEETS_INTEGRATION_TESTS_CREDS: ${{ secrets.GSHEETS_INTEGRATION_TESTS_CREDS }}
           SNOWFLAKE_INTEGRATION_TEST_CREDS: ${{ secrets.SNOWFLAKE_INTEGRATION_TEST_CREDS }}
+          FACEBOOK_MARKETING_API_TEST_INTEGRATION_CREDS: ${{ secrets.FACEBOOK_MARKETING_API_TEST_INTEGRATION_CREDS }}
 
       - name: Build
         run: ./gradlew --no-daemon build --scan

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/74d47f79-8d01-44ac-9755-f5eb0d7caacb.json

@@ -0,0 +1,7 @@
+{
+  "sourceDefinitionId": "74d47f79-8d01-44ac-9755-f5eb0d7caacb",
+  "name": "Facebook Marketing APIs",
+  "dockerRepository": "airbyte/source-facebook-marketing-api-singer",
+  "dockerImageTag": "0.1.0",
+  "documentationUrl": "https://hub.docker.com/r/airbyte/source-facebook-marketing-api-singer"
+}

airbyte-integrations/bases/base-singer/base_singer/singer_helpers.py

@@ -28,7 +28,7 @@
 import subprocess
 from dataclasses import dataclass
 from datetime import datetime
-from typing import DefaultDict, Generator
+from typing import DefaultDict, Dict, Generator
 
 from airbyte_protocol import AirbyteCatalog, AirbyteMessage, AirbyteRecordMessage, AirbyteStateMessage, AirbyteStream
 
@@ -60,6 +60,15 @@ def _transform_types(stream_properties: DefaultDict):
             field_object = stream_properties[field_name]
             field_object["type"] = SingerHelper._parse_type(field_object["type"])
 
+    @staticmethod
+    def singer_catalog_to_airbyte_catalog(singer_catalog: Dict[str, any]) -> AirbyteCatalog:
+        airbyte_streams = []
+        for stream in singer_catalog.get("streams"):
+            name = stream.get("stream")
+            schema = stream.get("schema")
+            airbyte_streams += [AirbyteStream(name=name, json_schema=schema)]
+        return AirbyteCatalog(streams=airbyte_streams)
+
     @staticmethod
     def get_catalogs(
         logger, shell_command, singer_transform=(lambda catalog: catalog), airbyte_transform=(lambda catalog: catalog)
@@ -71,15 +80,8 @@ def get_catalogs(
         for line in completed_process.stderr.splitlines():
             logger.log_by_prefix(line, "ERROR")
 
-        airbyte_streams = []
         singer_catalog = singer_transform(json.loads(completed_process.stdout))
-
-        for stream in singer_catalog.get("streams"):
-            name = stream.get("stream")
-            schema = stream.get("schema")
-            airbyte_streams += [AirbyteStream(name=name, json_schema=schema)]
-
-        airbyte_catalog = airbyte_transform(AirbyteCatalog(streams=airbyte_streams))
+        airbyte_catalog = airbyte_transform(SingerHelper.singer_catalog_to_airbyte_catalog(singer_catalog))
 
         return Catalogs(singer_catalog=singer_catalog, airbyte_catalog=airbyte_catalog)
 
@@ -89,7 +91,6 @@ def read(logger, shell_command, is_message=(lambda x: True), transform=(lambda x
             sel = selectors.DefaultSelector()
             sel.register(p.stdout, selectors.EVENT_READ)
             sel.register(p.stderr, selectors.EVENT_READ)
-
             ok = True
             while ok:
                 for key, val1 in sel.select():

airbyte-integrations/connector-templates/source-python/Dockerfile

@@ -1,6 +1,7 @@
 FROM airbyte/integration-base-python:dev
 
-RUN apt-get update && apt-get install -y jq curl bash && rm -rf /var/lib/apt/lists/*
+# Bash is installed for more convenient debugging.
+RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/*
 
 ENV CODE_PATH="source_{{snakeCase name}}"
 ENV AIRBYTE_IMPL_MODULE="source_{{snakeCase name}}"

airbyte-integrations/connector-templates/source-python/Dockerfile.test.hbs

@@ -1,11 +1,10 @@
 FROM airbyte/base-python-test:dev
 
-RUN apt-get update && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/*
 
 ENV CODE_PATH="integration_tests"
 ENV AIRBYTE_TEST_MODULE="integration_tests"
 ENV AIRBYTE_TEST_PATH="Source{{properCase name}}StandardTest"
-ENV AIRBYTE_TEST_CASE=true
 
 LABEL io.airbyte.version=0.1.0
 LABEL io.airbyte.name=airbyte/source-{{dashCase name}}-standard-test

airbyte-integrations/connector-templates/source-python/NEW_SOURCE_CHECKLIST.md.hbs

@@ -2,14 +2,18 @@
 
 This is an autogenerated file describing the high-level steps you need to take to implement a new Airbyte source in Python.
 
+1. First, build the module by running the following from the `airbyte` project root directory:
+   ```
+   ./gradlew :airbyte-integrations:connectors:source-{dashCase name}:build
+   ```
 1. Define the specification for this source connector by modifying `source_{{snakeCase name}}/spec.json`.
    A specification is a JSON file which uses JSONSchema to declare all the inputs needed for your integration to function 
    correctly. For example, if you were configuring a Postgres source, your specification might declare that you need a 
    `username` field which a string, a `password` field which is a string, a `host_url` which is a URI-formatted string, 
    and a `port` which is a `number`. The Airbyte UI will generate configurations that match the specification (by asking 
    the user to input them) and pass those configs to your source connector as input when it is being run.
    You can also manually generate config files and pass them to the integration CLI.
-1. Implement your integration by editing `source_{{snakeCase}}/source.py` (and creating additional files as necessary).
+1. Implement your integration by editing `source_{{snakeCase name}}/source.py` (and creating additional files as necessary).
     1. Implement the `Source` interface. For more information see the docstrings in `Source` and any parent classes.
     1. All logging should be done through the `logger` object passed into each method. Otherwise, logs will not be shown
        in the Airbyte UI.

airbyte-integrations/connector-templates/source-python/README.md.hbs

@@ -59,4 +59,4 @@ Follow the instructions in the [documentation](https://docs.airbyte.io/integrati
 in `secrets/credentials.json`.
 
 ### Airbyte Employee
-Credentials are available in RPass under the secret name `source-{{dashCase name}}-integration-test-creds}}`.
+Credentials are available in RPass under the secret name `source-{{dashCase name}}-integration-test-creds`.

airbyte-integrations/connector-templates/source-singer/Dockerfile

@@ -1,6 +1,7 @@
 FROM airbyte/integration-base-singer:dev
 
-RUN apt-get update && apt-get install -y jq curl bash && rm -rf /var/lib/apt/lists/*
+# Bash is installed for more convenient debugging.
+RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/*
 
 ENV CODE_PATH="source_{{snakeCase name}}_singer"
 ENV AIRBYTE_IMPL_MODULE="source_{{snakeCase name}}_singer"

airbyte-integrations/connector-templates/source-singer/Dockerfile.test.hbs

@@ -1,12 +1,12 @@
 FROM airbyte/base-python-test:dev
 
-RUN apt-get update && rm -rf /var/lib/apt/lists/*
+# Bash is installed for convenient debugging.
+RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/*
 
 ENV MODULE_NAME="source_{{snakeCase name}}_singer"
 ENV CODE_PATH="integration_tests"
 ENV AIRBYTE_TEST_MODULE="integration_tests"
 ENV AIRBYTE_TEST_PATH="Source{{properCase name}}SingerStandardTest"
-ENV AIRBYTE_TEST_CASE=true
 
 LABEL io.airbyte.version=0.1.0
 LABEL io.airbyte.name=airbyte/source-{{dashCase name}}-singer-standard-test

airbyte-integrations/connector-templates/source-singer/NEW_SOURCE_CHECKLIST.md.hbs

@@ -2,6 +2,10 @@
 
 This is an autogenerated file describing the steps needed to implement a new Airbyte source based on a Singer Tap.
 
+1. First, build the module by running the following from the `airbyte` project root directory:
+   ```
+   ./gradlew :airbyte-integrations:connectors:source-{dashCase name}-singer:build
+   ```
 1. Include the Singer Tap you'd like to build this Source on top by including it in `setup.py` as a dependency.
 1. Define the specification for this source connector by modifying `source_{{snakeCase name}}_singer/spec.json`.
    A specification is a JSON file which uses JSONSchema to declare all the inputs needed for your integration to function 
@@ -10,7 +14,7 @@ This is an autogenerated file describing the steps needed to implement a new Air
    and a `port` which is a `number`. The Airbyte UI will generate configurations that match the specification (by asking 
    the user to input them) and pass those configs to your source connector as input when it is being run.
    You can also manually generate config files and pass them to the integration CLI.
-1. Implement your integration by editing `source_{{snakeCase}}_singer/source.py` (and creating additional files as necessary).
+1. Implement your integration by editing `source_{{snakeCase name}}_singer/source.py` (and creating additional files as necessary).
     1. Implement the `Source` interface. For more information see the docstrings in `Source` and any parent classes.
     1. All logging should be done through the `logger` object passed into each method. Otherwise, logs will not be shown
     in the Airbyte UI.

airbyte-integrations/connector-templates/source-singer/README.md.hbs

@@ -60,4 +60,4 @@ Follow the instructions in the [documentation](https://docs.airbyte.io/integrati
 in `secrets/credentials.json`.
 
 ### Airbyte Employee
-Credentials are available in RPass under the secret name `source-{{dashCase name}}-singer-integration-test-creds}}`.
+Credentials are available in RPass under the secret name `source-{{dashCase name}}-singer-integration-test-creds`.

airbyte-integrations/connectors/source-facebook-marketing-api-singer/Dockerfile

@@ -0,0 +1,17 @@
+FROM airbyte/integration-base-singer:dev
+
+# GCC is needed by the ciso8601, a dependency of tap-facebook.
+# See https://github.com/closeio/ciso8601/issues/98 for more information.
+RUN apt-get update && apt-get install -y gcc bash && rm -rf /var/lib/apt/lists/*
+
+ENV CODE_PATH="source_facebook_marketing_api_singer"
+ENV AIRBYTE_IMPL_MODULE="source_facebook_marketing_api_singer"
+ENV AIRBYTE_IMPL_PATH="SourceFacebookMarketingApiSinger"
+
+WORKDIR /airbyte/integration_code
+COPY $CODE_PATH ./$CODE_PATH
+COPY setup.py ./
+RUN pip install ".[main]"
+
+LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.name=airbyte/source-facebook-marketing-api-singer

airbyte-integrations/connectors/source-facebook-marketing-api-singer/Dockerfile.test

@@ -0,0 +1,22 @@
+FROM airbyte/base-python-test:dev
+
+RUN apt-get update && rm -rf /var/lib/apt/lists/*
+
+ENV MODULE_NAME="source_facebook_marketing_api_singer"
+ENV CODE_PATH="integration_tests"
+ENV AIRBYTE_TEST_MODULE="integration_tests"
+ENV AIRBYTE_TEST_PATH="SourceFacebookMarketingApiSingerStandardTest"
+
+LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.name=airbyte/source-facebook-marketing-api-singer-standard-test
+
+WORKDIR /airbyte/integration_code
+COPY $MODULE_NAME $MODULE_NAME
+COPY $CODE_PATH $CODE_PATH
+COPY secrets/* $CODE_PATH
+COPY $MODULE_NAME/*.json $CODE_PATH
+COPY setup.py ./
+
+RUN pip install ".[tests]"
+
+WORKDIR /airbyte

airbyte-integrations/connectors/source-facebook-marketing-api-singer/README.md

@@ -0,0 +1,63 @@
+# Source Facebook Marketing Api Singer
+
+This is the repository for the Facebook Marketing Api source connector, based on a Singer tap.
+For information about how to use this connector within Airbyte, see [the User Documentation](https://docs.airbyte.io/integrations/sources/facebook-marketing-api).
+
+## Local development
+### Prerequisites
+**To iterate on this connector, make sure to complete this prerequisites section.**
+
+#### Build & Activate Virtual Environment
+First, build the module by running the following from the `airbyte` project root directory: 
+```
+./gradlew :airbyte-integrations:connectors:source-facebook-marketing-api-singer:build
+```
+
+This will generate a virtualenv for this module in `source-facebook-marketing-api-singer/.venv`. Make sure this venv is active in your
+development environment of choice. If you are on the terminal, run the following
+```
+cd airbyte-integrations/connectors/source-facebook-marketing-api # cd into the connector directory
+source .venv/bin/activate
+```
+If you are in an IDE, follow your IDE's instructions to activate the virtualenv. 
+
+#### Create credentials
+If you are an Airbyte core member, copy the credentials in RPass under the secret name `source-facebook-marketing-api-singer-integration-test-creds`
+and place them into `secrets/config.json`.
+
+If you are a contributor, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/facebook-marketing-api) to generate an access token and  obtain your ad account ID. 
+Then create a file `secrets/config.json` conforming to the `spec.json` file. See `sample_files/sample_config.json` for a sample config file.
+
+### Locally running the connector
+```
+python main_dev.py spec
+python main_dev.py check --config secrets/config.json
+python main_dev.py discover --config secrets/config.json
+python main_dev.py read --config secrets/config.json --catalog sample_files/sample_catalog.json
+```
+
+### Unit Tests
+To run unit tests locally, from the connector root run:
+```
+pytest unit_tests
+```
+
+### Locally running the connector docker image
+
+```
+# in airbyte root directory
+./gradlew :airbyte-integrations:connectors:source-facebook-marketing-api-singer:buildImage
+docker run --rm airbyte/source-facebook-marketing-api-singer:dev spec
+docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-facebook-marketing-api-singer/secrets:/secrets airbyte/source-facebook-marketing-api-singer:dev check --config /secrets/config.json
+docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-facebook-marketing-api-singer/secrets:/secrets airbyte/source-facebook-marketing-api-singer:dev discover --config /secrets/config.json
+docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-facebook-marketing-api-singer/secrets:/secrets -v $(pwd)/airbyte-integrations/connectors/source-facebook-marketing-api-singer/sample_files:/sample_files airbyte/source-facebook-marketing-api-singer:dev read --config /secrets/config.json --catalog /sample_files/sample_catalog.json
+```
+
+### Integration Tests 
+1. Configure credentials as appropriate, described below.
+1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-facebook-marketing-api-singer:standardSourceTestPython` to run the standard integration test suite.
+1. To run additional integration tests, place your integration tests in the `integration_tests` directory and run them with `pytest integration_tests`.
+   Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named.
+
+## Dependency Management
+All dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.

airbyte-integrations/connectors/source-facebook-marketing-api-singer/airbyte_protocol

@@ -0,0 +1 @@
+../../bases/airbyte-protocol/airbyte_protocol

airbyte-integrations/connectors/source-facebook-marketing-api-singer/base_python

@@ -0,0 +1 @@
+../../bases/base-python/base_python

airbyte-integrations/connectors/source-facebook-marketing-api-singer/base_singer

@@ -0,0 +1 @@
+../../bases/base-singer/base_singer

airbyte-integrations/connectors/source-facebook-marketing-api-singer/build.gradle

@@ -0,0 +1,34 @@
+project.ext.pyModule = 'source_facebook_marketing_api_singer'
+apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle')
+apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
+apply from: rootProject.file('tools/gradle/commons/integrations/test-image.gradle')
+apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle')
+apply from: rootProject.file('tools/gradle/commons/integrations/standard-source-test-python.gradle')
+
+
+standardSourceTestPython {
+    ext {
+        imageName = "${extractImageName(project.file('Dockerfile'))}:dev"
+        pythonContainerName = "${extractImageName(project.file('Dockerfile.test'))}:dev"
+    }
+}
+
+task installTestDeps(type: PythonTask){
+    module = "pip"
+    command = "install .[tests]"
+}
+
+task unitTest(type: PythonTask){
+    module = "pytest"
+    command = "unit_tests"
+}
+
+unitTest.dependsOn(installTestDeps)
+build.dependsOn(unitTest)
+build.dependsOn ':airbyte-integrations:bases:base-python-test:build'
+
+buildImage.dependsOn ':airbyte-integrations:bases:base-singer:buildImage'
+buildTestImage.dependsOn ':airbyte-integrations:bases:base-singer:buildImage'
+
+integrationTest.dependsOn(buildImage)
+standardSourceTestPython.dependsOn(buildTestImage)

airbyte-integrations/connectors/source-facebook-marketing-api-singer/integration_tests/__init__.py

@@ -0,0 +1,27 @@
+"""
+MIT License
+
+Copyright (c) 2020 Airbyte
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+
+from .standard_source_test import SourceFacebookMarketingApiSingerStandardTest
+
+__all__ = ["SourceFacebookMarketingApiSingerStandardTest"]