diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/3052c77e-8b91-47e2-97a0-a29a22794b4b.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/3052c77e-8b91-47e2-97a0-a29a22794b4b.json new file mode 100644 index 0000000000000..4f8e9e308887d --- /dev/null +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/3052c77e-8b91-47e2-97a0-a29a22794b4b.json @@ -0,0 +1,8 @@ +{ + "sourceDefinitionId": "3052c77e-8b91-47e2-97a0-a29a22794b4b", + "name": "PersistIq", + "dockerRepository": "airbyte/source-persistiq", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://docs.airbyte.io/integrations/sources/persistiq", + "icon": "persistiq.svg" +} diff --git a/airbyte-config/init/src/main/resources/icons/persistiq.svg b/airbyte-config/init/src/main/resources/icons/persistiq.svg new file mode 100644 index 0000000000000..e10a9374a771b --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/persistiq.svg @@ -0,0 +1,8 @@ + + + + + + diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 1302890f6e688..7024b2dfbfa07 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -499,6 +499,13 @@ documentationUrl: https://docs.airbyte.io/integrations/sources/paystack icon: paystack.svg sourceType: api +- name: PersistIq + sourceDefinitionId: 3052c77e-8b91-47e2-97a0-a29a22794b4b + dockerRepository: airbyte/source-persistiq + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/persistiq + icon: persistiq.svg + sourceType: api - name: Pinterest sourceDefinitionId: 5cb7e5fe-38c2-11ec-8d3d-0242ac130003 dockerRepository: airbyte/source-pinterest diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index ed9459b0c5bbc..b4ef2cbc6d58f 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -5277,6 +5277,25 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-persistiq:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.io/integrations/sources/persistiq" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Persistiq Spec" + type: "object" + required: + - "api_key" + additionalProperties: false + properties: + api_key: + type: "string" + description: "PersistIq API Key. See the docs for more information on where to find that key." + airbyte_secret: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-pinterest:0.1.1" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/pinterest" diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 001ae80fd3248..da3de49450972 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -68,6 +68,7 @@ | Oracle DB | [![source-oracle](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-oracle%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-oracle) | | Paypal Transaction | [![paypal-transaction](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-paypal-transaction%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-paypal-transaction) | | Paystack | [![source-paystack](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-paystack%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-paystack) | +| PersistIq | [![source-persistiq](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-persistiq%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-persistiq/) | | Pipedrive | [![source-pipedrive](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-pipedrive%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-pipedrive) | | Plaid | [![source-plaid](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-plaid%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-plaid) | | Postgres | [![source-postgres](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-postgres%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-postgres) | diff --git a/airbyte-integrations/connectors/source-persistiq/.dockerignore b/airbyte-integrations/connectors/source-persistiq/.dockerignore new file mode 100644 index 0000000000000..7754a0a84ebcd --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_persistiq +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-persistiq/Dockerfile b/airbyte-integrations/connectors/source-persistiq/Dockerfile new file mode 100644 index 0000000000000..e533a190826d1 --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.7.11-alpine3.14 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_persistiq ./source_persistiq + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-persistiq diff --git a/airbyte-integrations/connectors/source-persistiq/README.md b/airbyte-integrations/connectors/source-persistiq/README.md new file mode 100644 index 0000000000000..f36570bb419c2 --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/README.md @@ -0,0 +1,133 @@ +# Persistiq Source + +This is the repository for the Persistiq source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/persistiq). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-persistiq:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/persistiq) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_persistiq/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. +To obtain credentials, create an account on PersistIq and follow the [documentation](https://apidocs.persistiq.com/#authentication) + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source persistiq test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-persistiq:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-persistiq:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-persistiq:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-persistiq:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-persistiq:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-persistiq:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-persistiq:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-persistiq:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-persistiq/acceptance-test-config.yml b/airbyte-integrations/connectors/source-persistiq/acceptance-test-config.yml new file mode 100644 index 0000000000000..4446ede08e8ec --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/acceptance-test-config.yml @@ -0,0 +1,23 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-persistiq:dev +tests: + spec: + - spec_path: "source_persistiq/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + expect_records: + path: "integration_tests/expected_campaigns_stream.txt" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + diff --git a/airbyte-integrations/connectors/source-persistiq/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-persistiq/acceptance-test-docker.sh new file mode 100644 index 0000000000000..c51577d10690c --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-persistiq/bootstrap.md b/airbyte-integrations/connectors/source-persistiq/bootstrap.md new file mode 100644 index 0000000000000..c4f0a41ca68a7 --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/bootstrap.md @@ -0,0 +1,18 @@ +# PersistIq +PersistIq is an outbound automation tool designed for small teams to find, reach, and organize customers all in one simple platform. + +## Streams + +This Source is capable of syncing the following streams: +* [Users](https://apidocs.persistiq.com/#users) +* [Leads](https://apidocs.persistiq.com/#leads) +* [Campaigns](https://apidocs.persistiq.com/#campaigns) + +### Incremental streams +Incremental streams were not implemented in the initial version. + +### Next steps +Implement incremental sync and additional streams (`Lead status`, `Lead fields`, `Events`). + +### Rate limits +The API rate limit is at 100 requests/minutes. Read [Rate Limits](https://apidocs.persistiq.com/#error-codes) for more informations. diff --git a/airbyte-integrations/connectors/source-persistiq/build.gradle b/airbyte-integrations/connectors/source-persistiq/build.gradle new file mode 100644 index 0000000000000..a7700be96ab8e --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_persistiq' +} diff --git a/airbyte-integrations/connectors/source-persistiq/integration_tests/__init__.py b/airbyte-integrations/connectors/source-persistiq/integration_tests/__init__.py new file mode 100644 index 0000000000000..46b7376756ec6 --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-persistiq/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-persistiq/integration_tests/acceptance.py new file mode 100644 index 0000000000000..0347f2a0b143d --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-persistiq/integration_tests/catalog.json b/airbyte-integrations/connectors/source-persistiq/integration_tests/catalog.json new file mode 100644 index 0000000000000..3a8f681f82c78 --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/integration_tests/catalog.json @@ -0,0 +1,195 @@ +{ + "streams": [ + { + "stream": { + "name": "leads", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "bounced": { + "type": ["null", "boolean"] + }, + "owner_id": { + "type": ["null", "string"] + }, + "optedout": { + "type": ["null", "boolean"] + }, + "sent_count": { + "type": ["null", "integer"] + }, + "replied_count": { + "type": ["null", "integer"] + }, + "last_sent_at": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "data": { + "company_name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"], + "format": "email" + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "industry": { + "type": ["null", "string"] + }, + "snippet": { + "type": ["null", "string"] + }, + "snippet1": { + "type": ["null", "string"] + }, + "snippet2": { + "type": ["null", "string"] + }, + "snippet3": { + "type": ["null", "string"] + }, + "snippet4": { + "type": ["null", "string"] + }, + "twitch_name": { + "type": ["null", "string"] + }, + "linkedin": { + "type": ["null", "string"] + }, + "twitter": { + "type": ["null", "string"] + }, + "facebook": { + "type": ["null", "string"] + }, + "salesforce_id": { + "type": ["null", "string"] + } + } + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "campaigns", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "creator": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + } + }, + "stats": { + "prospects_contacted": { + "type": ["null", "integer"] + }, + "prospects_reached": { + "type": ["null", "integer"] + }, + "prospects_opened": { + "type": ["null", "integer"] + }, + "prospects_replied": { + "type": ["null", "integer"] + }, + "prospects_bounced": { + "type": ["null", "integer"] + }, + "prospects_optedout": { + "type": ["null", "integer"] + }, + "total_contacted": { + "type": ["null", "integer"] + } + } + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "users", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"], + "format": "email" + }, + "activated": { + "type": ["null", "boolean"] + }, + "default_mailbox_id": { + "type": ["null", "string"] + }, + "salesforce_id": { + "type": ["null", "string"] + } + } + } + }, + "supported_sync_modes": ["full_refresh"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "source_defined_primary_key": [["id"]] + } + ] +} diff --git a/airbyte-integrations/connectors/source-persistiq/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-persistiq/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..6719f174b0bbf --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/integration_tests/configured_catalog.json @@ -0,0 +1,31 @@ +{ + "streams": [ + { + "stream": { + "name": "campaigns", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "leads", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-persistiq/integration_tests/expected_campaigns_stream.txt b/airbyte-integrations/connectors/source-persistiq/integration_tests/expected_campaigns_stream.txt new file mode 100644 index 0000000000000..47c985a587c9b --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/integration_tests/expected_campaigns_stream.txt @@ -0,0 +1,3 @@ +{"stream": "campaigns", "data": {"id": "c_3kqMZZ","name": "Schedule Meetings with Reps (sample campaign)","creator": {"id": "u_pdKWk3", "name": "Sherif Nada", "email": "integration-test@airbyte.io"},"stats": {"prospects_contacted": 0, "prospects_reached": 0, "prospects_opened": 0, "prospects_replied": 0, "prospects_bounced": 0, "prospects_optedout": 0, "total_prospects": 3}}, "emitted_at": 1629119628000} +{"stream": "campaigns", "data": {"id": "c_ljDgZB","name": "Schedule Meetings with Managers (sample campaign)","creator": {"id": "u_pdKWk3", "name": "Sherif Nada", "email": "integration-test@airbyte.io"},"stats": {"prospects_contacted": 0, "prospects_reached": 0, "prospects_opened": 0, "prospects_replied": 0, "prospects_bounced": 0, "prospects_optedout": 0, "total_prospects": 3}}, "emitted_at": 1629119628000} +{"stream": "campaigns", "data": {"id": "c_3e01Kb","name": "Schedule Meetings with CEOs (sample campaign)","creator": {"id": "u_pdKWk3", "name": "Sherif Nada", "email": "integration-test@airbyte.io"},"stats": {"prospects_contacted": 0, "prospects_reached": 0, "prospects_opened": 0, "prospects_replied": 0, "prospects_bounced": 0, "prospects_optedout": 0, "total_prospects": 3}}, "emitted_at": 1629119628000} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-persistiq/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-persistiq/integration_tests/invalid_config.json new file mode 100644 index 0000000000000..dc521ade7acf9 --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "api_key": "" +} diff --git a/airbyte-integrations/connectors/source-persistiq/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-persistiq/integration_tests/sample_config.json new file mode 100644 index 0000000000000..c16cd6d40596a --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "api_key": "" +} diff --git a/airbyte-integrations/connectors/source-persistiq/main.py b/airbyte-integrations/connectors/source-persistiq/main.py new file mode 100644 index 0000000000000..3f871b1110903 --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_persistiq import SourcePersistiq + +if __name__ == "__main__": + source = SourcePersistiq() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-persistiq/requirements.txt b/airbyte-integrations/connectors/source-persistiq/requirements.txt new file mode 100644 index 0000000000000..0411042aa0911 --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-persistiq/setup.py b/airbyte-integrations/connectors/source-persistiq/setup.py new file mode 100644 index 0000000000000..af1420f9687cc --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/setup.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "requests_mock==1.8.0", + "source-acceptance-test", +] + +setup( + name="source_persistiq", + description="Source implementation for Persistiq.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-persistiq/source_persistiq/__init__.py b/airbyte-integrations/connectors/source-persistiq/source_persistiq/__init__.py new file mode 100644 index 0000000000000..a3923d95bbe87 --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/source_persistiq/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from .source import SourcePersistiq + +__all__ = ["SourcePersistiq"] diff --git a/airbyte-integrations/connectors/source-persistiq/source_persistiq/schemas/campaigns.json b/airbyte-integrations/connectors/source-persistiq/source_persistiq/schemas/campaigns.json new file mode 100644 index 0000000000000..d602148216150 --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/source_persistiq/schemas/campaigns.json @@ -0,0 +1,52 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["string"] + }, + "name": { + "type": ["null", "string"] + }, + "creator": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + } + } + }, + "stats": { + "type": ["null", "object"], + "properties": { + "prospects_contacted": { + "type": ["null", "integer"] + }, + "prospects_reached": { + "type": ["null", "integer"] + }, + "prospects_opened": { + "type": ["null", "integer"] + }, + "prospects_replied": { + "type": ["null", "integer"] + }, + "prospects_bounced": { + "type": ["null", "integer"] + }, + "prospects_optedout": { + "type": ["null", "integer"] + }, + "total_contacted": { + "type": ["null", "integer"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-persistiq/source_persistiq/schemas/leads.json b/airbyte-integrations/connectors/source-persistiq/source_persistiq/schemas/leads.json new file mode 100644 index 0000000000000..cb6ba052a8828 --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/source_persistiq/schemas/leads.json @@ -0,0 +1,93 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["string"] + }, + "bounced": { + "type": ["null", "boolean"] + }, + "owner_id": { + "type": ["string"] + }, + "optedout": { + "type": ["null", "boolean"] + }, + "sent_count": { + "type": ["null", "integer"] + }, + "replied_count": { + "type": ["null", "integer"] + }, + "last_sent_at": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "data": { + "company_name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"], + "format": "email" + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "industry": { + "type": ["null", "string"] + }, + "snippet": { + "type": ["null", "string"] + }, + "snippet1": { + "type": ["null", "string"] + }, + "snippet2": { + "type": ["null", "string"] + }, + "snippet3": { + "type": ["null", "string"] + }, + "snippet4": { + "type": ["null", "string"] + }, + "twitch_name": { + "type": ["null", "string"] + }, + "linkedin": { + "type": ["null", "string"] + }, + "twitter": { + "type": ["null", "string"] + }, + "facebook": { + "type": ["null", "string"] + }, + "salesforce_id": { + "type": ["null", "string"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-persistiq/source_persistiq/schemas/users.json b/airbyte-integrations/connectors/source-persistiq/source_persistiq/schemas/users.json new file mode 100644 index 0000000000000..8a4d6b307689e --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/source_persistiq/schemas/users.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["string"] + }, + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["string"], + "format": "email" + }, + "activated": { + "type": ["null", "boolean"] + }, + "default_mailbox_id": { + "type": ["null", "string"] + }, + "salesforce_id": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-persistiq/source_persistiq/source.py b/airbyte-integrations/connectors/source-persistiq/source_persistiq/source.py new file mode 100644 index 0000000000000..b4dc493d4e57f --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/source_persistiq/source.py @@ -0,0 +1,96 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import NoAuth + +# Basic full refresh stream + + +class PersistiqStream(HttpStream, ABC): + def __init__(self, api_key: str, **kwargs): + super().__init__(**kwargs) + self.api_key = api_key + + url_base = "https://api.persistiq.com/v1/" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + json_response = response.json() + if not json_response.get("has_more", False): + return None + + return {"page": json_response.get("next_page")[-1]} + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return {"page": 1 if not next_page_token else next_page_token["page"]} + + def request_headers(self, **kwargs) -> MutableMapping[str, Any]: + return {"x-api-key": self.api_key} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + yield response.json() + + +class Users(PersistiqStream): + primary_key = "id" + + def path(self, **kwargs) -> str: + return "users" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + json = response.json() + yield from json["users"] + + +class Leads(PersistiqStream): + primary_key = "id" + + def path(self, **kwargs) -> str: + return "leads" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + json = response.json() + yield from json["leads"] + + +class Campaigns(PersistiqStream): + primary_key = "id" + + def path(self, **kwargs) -> str: + return "campaigns" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + json = response.json() + yield from json["campaigns"] + + +# Source + + +class SourcePersistiq(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + headers = {"x-api-key": config["api_key"]} + url = "https://api.persistiq.com/v1/users" + try: + response = requests.get(url, headers=headers) + response.raise_for_status() + return True, None + except requests.exceptions.RequestException as e: + return False, e + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + auth = NoAuth() + return [ + Users(authenticator=auth, api_key=config["api_key"]), + Leads(authenticator=auth, api_key=config["api_key"]), + Campaigns(authenticator=auth, api_key=config["api_key"]), + ] diff --git a/airbyte-integrations/connectors/source-persistiq/source_persistiq/spec.json b/airbyte-integrations/connectors/source-persistiq/source_persistiq/spec.json new file mode 100644 index 0000000000000..bf3260203357e --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/source_persistiq/spec.json @@ -0,0 +1,17 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/persistiq", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Persistiq Spec", + "type": "object", + "required": ["api_key"], + "additionalProperties": false, + "properties": { + "api_key": { + "type": "string", + "description": "PersistIq API Key. See the docs for more information on where to find that key.", + "airbyte_secret": true + } + } + } +} diff --git a/airbyte-integrations/connectors/source-persistiq/unit_tests/__init__.py b/airbyte-integrations/connectors/source-persistiq/unit_tests/__init__.py new file mode 100644 index 0000000000000..46b7376756ec6 --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-persistiq/unit_tests/test_source.py b/airbyte-integrations/connectors/source-persistiq/unit_tests/test_source.py new file mode 100644 index 0000000000000..b230eb6c9fd32 --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/unit_tests/test_source.py @@ -0,0 +1,38 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import requests +from source_persistiq.source import SourcePersistiq + + +def test_check_connection(mocker, requests_mock): + source = SourcePersistiq() + mock_logger = mocker.Mock() + test_config = {"api_key": "mybeautifulkey"} + # success + requests_mock.get( + "https://api.persistiq.com/v1/users", + json={ + "id": "u_3an2Jp", + "name": "Gabriel Rossmann", + "email": "gabriel@punctual.cc", + "activated": "true", + "default_mailbox_id": "mbox_38ymEp", + "salesforce_id": "", + }, + ) + assert source.check_connection(mock_logger, test_config) == (True, None) + + # failure + requests_mock.get("https://api.persistiq.com/v1/users", status_code=500) + connection_success, connection_failure = source.check_connection(mock_logger, test_config) + assert not connection_success + assert isinstance(connection_failure, requests.exceptions.HTTPError) + + +def test_streams(): + source = SourcePersistiq() + config = {"api_key": "my-api-key"} + streams = source.streams(config) + assert len(streams) == 3 diff --git a/airbyte-integrations/connectors/source-persistiq/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-persistiq/unit_tests/test_streams.py new file mode 100644 index 0000000000000..32f5fe3153909 --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/unit_tests/test_streams.py @@ -0,0 +1,64 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import pytest +from source_persistiq.source import PersistiqStream + + +def mocked_requests_get(*args, **kwargs): + class MockResponse: + def __init__(self, json_data, status_code): + self.json_data = json_data + self.status_code = status_code + + def json(self): + return self.json_data + + return MockResponse(json_data=kwargs["json_data"], status_code=kwargs["status_code"]) + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(PersistiqStream, "path", "v0/example_endpoint") + mocker.patch.object(PersistiqStream, "primary_key", "test_primary_key") + mocker.patch.object(PersistiqStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = PersistiqStream(api_key="mybeautifulkey") + inputs = {"next_page_token": {"page": 1}} + expected_params = {"page": 1} + assert stream.request_params(stream_state=None, **inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = PersistiqStream(api_key="mybeautifulkey") + # With next page + response = mocked_requests_get(json_data={"has_more": True, "next_page": "https://api.persistiq.com/v1/users?page=2"}, status_code=200) + expected_token = "2" + assert stream.next_page_token(response=response) == {"page": expected_token} + # Without next page + response = mocked_requests_get(json_data={}, status_code=200) + expected_token = None + assert stream.next_page_token(response=response) == expected_token + + +def test_parse_response(patch_base_class): + stream = PersistiqStream(api_key="mybeautifulkey") + response = mocked_requests_get(json_data={"users": [{"id": 1, "name": "John Doe"}]}, status_code=200) + expected_parsed_object = {"users": [{"id": 1, "name": "John Doe"}]} + assert next(stream.parse_response(response=response)) == expected_parsed_object + + +def test_request_headers(patch_base_class): + stream = PersistiqStream(api_key="mybeautifulkey") + expected_headers = {"x-api-key": "mybeautifulkey"} + assert stream.request_headers() == expected_headers + + +def test_http_method(patch_base_class): + stream = PersistiqStream(api_key="mybeautifulkey") + expected_method = "GET" + assert stream.http_method == expected_method diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 56c36d6dec8e2..8ce89a412a469 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -114,6 +114,7 @@ * [Outreach](integrations/sources/outreach.md) * [Paypal Transaction](integrations/sources/paypal-transaction.md) * [Paystack](integrations/sources/paystack.md) + * [Persistiq](integrations/sources/persistiq.md) * [Plaid](integrations/sources/plaid.md) * [Pinterest](integrations/sources/pinterest.md) * [Pipedrive](integrations/sources/pipedrive.md) diff --git a/docs/integrations/README.md b/docs/integrations/README.md index f91687ce224ae..49b06df810bf9 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -87,6 +87,7 @@ Airbyte uses a grading system for connectors to help users understand what to ex | [Outreach](./sources/outreach.md)| Alpha | | [PayPal Transaction](sources/paypal-transaction.md) | Beta | | [Paystack](sources/paystack.md) | Alpha | +| [PersistIq](sources/persistiq.md) | Alpha | | [Pinterest](sources/pinterest.md) | Alpha | | [Pipedrive](sources/pipedrive.md) | Alpha | | [Plaid](sources/plaid.md) | Alpha | diff --git a/docs/integrations/sources/persistiq.md b/docs/integrations/sources/persistiq.md new file mode 100644 index 0000000000000..fa466116718ed --- /dev/null +++ b/docs/integrations/sources/persistiq.md @@ -0,0 +1,38 @@ +# PersistIq + +## Sync overview + +The PersistIq source supports Full Refresh syncs only. + +This source syncs data for the [PersistIq API](https://apidocs.persistiq.com/#introduction). + + +### Output schema + +This Source is capable of syncing the following streams: + +* [Users](https://apidocs.persistiq.com/#users) +* [Leads](https://apidocs.persistiq.com/#leads) +* [Campaigns](https://apidocs.persistiq.com/#campaigns) + +### Features + +| Feature | Supported?\(Yes/No\) +| :--- | :--- | +| Full Refresh Sync | Yes | +| Incremental - Append Sync | No | +| Namespaces | No | + +### Performance considerations + +The PersistIq connector should not run into PersistIq API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. + +## Getting started + +### Requirements + +* PersistIq API Key + +### Setup guide + +Please read [How to find your API key](https://apidocs.persistiq.com/#introduction).