From d6fb18bdaa38e7ddd8f152c0d217e96888f06782 Mon Sep 17 00:00:00 2001 From: Cirdes Henrique Date: Sun, 30 Oct 2022 16:25:14 -0300 Subject: [PATCH 1/8] New source Mailersend --- .../source-mailersend/.dockerignore | 6 ++ .../connectors/source-mailersend/Dockerfile | 38 +++++++++ .../connectors/source-mailersend/README.md | 79 +++++++++++++++++++ .../connectors/source-mailersend/__init__.py | 3 + .../acceptance-test-config.yml | 38 +++++++++ .../acceptance-test-docker.sh | 16 ++++ .../connectors/source-mailersend/build.gradle | 9 +++ .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 5 ++ .../integration_tests/acceptance.py | 16 ++++ .../integration_tests/configured_catalog.json | 13 +++ .../integration_tests/invalid_config.json | 6 ++ .../integration_tests/sample_config.json | 6 ++ .../integration_tests/sample_state.json | 26 ++++++ .../connectors/source-mailersend/main.py | 13 +++ .../source-mailersend/requirements.txt | 2 + .../connectors/source-mailersend/setup.py | 29 +++++++ .../source_mailersend/__init__.py | 8 ++ .../source_mailersend/mailersend.yaml | 50 ++++++++++++ .../source_mailersend/schemas/activity.json | 76 ++++++++++++++++++ .../source_mailersend/source.py | 18 +++++ .../source_mailersend/spec.yaml | 29 +++++++ docs/integrations/sources/mailersend.md | 27 +++++++ 23 files changed, 516 insertions(+) create mode 100644 airbyte-integrations/connectors/source-mailersend/.dockerignore create mode 100644 airbyte-integrations/connectors/source-mailersend/Dockerfile create mode 100644 airbyte-integrations/connectors/source-mailersend/README.md create mode 100644 airbyte-integrations/connectors/source-mailersend/__init__.py create mode 100644 airbyte-integrations/connectors/source-mailersend/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-mailersend/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-mailersend/build.gradle create mode 100644 airbyte-integrations/connectors/source-mailersend/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-mailersend/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-mailersend/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-mailersend/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-mailersend/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-mailersend/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-mailersend/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-mailersend/main.py create mode 100644 airbyte-integrations/connectors/source-mailersend/requirements.txt create mode 100644 airbyte-integrations/connectors/source-mailersend/setup.py create mode 100644 airbyte-integrations/connectors/source-mailersend/source_mailersend/__init__.py create mode 100644 airbyte-integrations/connectors/source-mailersend/source_mailersend/mailersend.yaml create mode 100644 airbyte-integrations/connectors/source-mailersend/source_mailersend/schemas/activity.json create mode 100644 airbyte-integrations/connectors/source-mailersend/source_mailersend/source.py create mode 100644 airbyte-integrations/connectors/source-mailersend/source_mailersend/spec.yaml create mode 100644 docs/integrations/sources/mailersend.md diff --git a/airbyte-integrations/connectors/source-mailersend/.dockerignore b/airbyte-integrations/connectors/source-mailersend/.dockerignore new file mode 100644 index 0000000000000..bdcf3c857308a --- /dev/null +++ b/airbyte-integrations/connectors/source-mailersend/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_mailersend +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-mailersend/Dockerfile b/airbyte-integrations/connectors/source-mailersend/Dockerfile new file mode 100644 index 0000000000000..9eacdb43b4273 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailersend/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_mailersend ./source_mailersend + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-mailersend diff --git a/airbyte-integrations/connectors/source-mailersend/README.md b/airbyte-integrations/connectors/source-mailersend/README.md new file mode 100644 index 0000000000000..53f11774addf6 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailersend/README.md @@ -0,0 +1,79 @@ +# Mailersend Source + +This is the repository for the Mailersend configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/mailersend). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-mailersend:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/mailersend) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_mailersend/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source mailersend test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-mailersend:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-mailersend:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-mailersend:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailersend:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailersend:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-mailersend:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-mailersend:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-mailersend:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-mailersend/__init__.py b/airbyte-integrations/connectors/source-mailersend/__init__.py new file mode 100644 index 0000000000000..1100c1c58cf51 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailersend/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-mailersend/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mailersend/acceptance-test-config.yml new file mode 100644 index 0000000000000..46533de7f7447 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailersend/acceptance-test-config.yml @@ -0,0 +1,38 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-mailersend:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_mailersend/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.txt" + # extra_fields: no + # exact_order: no + # extra_recods: yes + incremental: + bypass_reason: "This connector does not implement incremental sync" + # TODO uncomment this block this block if your connector implements incremental sync: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-mailersend/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-mailersend/acceptance-test-docker.sh new file mode 100644 index 0000000000000..c51577d10690c --- /dev/null +++ b/airbyte-integrations/connectors/source-mailersend/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-mailersend/build.gradle b/airbyte-integrations/connectors/source-mailersend/build.gradle new file mode 100644 index 0000000000000..15a2083d97183 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailersend/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_mailersend' +} diff --git a/airbyte-integrations/connectors/source-mailersend/integration_tests/__init__.py b/airbyte-integrations/connectors/source-mailersend/integration_tests/__init__.py new file mode 100644 index 0000000000000..1100c1c58cf51 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailersend/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-mailersend/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-mailersend/integration_tests/abnormal_state.json new file mode 100644 index 0000000000000..251c9d6a0bf89 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailersend/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "activity": { + "date": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-mailersend/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mailersend/integration_tests/acceptance.py new file mode 100644 index 0000000000000..1302b2f57e10e --- /dev/null +++ b/airbyte-integrations/connectors/source-mailersend/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-mailersend/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-mailersend/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..b151cb448eeb7 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailersend/integration_tests/configured_catalog.json @@ -0,0 +1,13 @@ +{ + "streams": [ + { + "stream": { + "name": "activity", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-mailersend/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-mailersend/integration_tests/invalid_config.json new file mode 100644 index 0000000000000..d1f3ad6152729 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailersend/integration_tests/invalid_config.json @@ -0,0 +1,6 @@ +{ + "api_token": "invalid", + "domain_id": "0000000000", + "date_from": "1666963299", + "date_to": "1667049682" +} diff --git a/airbyte-integrations/connectors/source-mailersend/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-mailersend/integration_tests/sample_config.json new file mode 100644 index 0000000000000..03c78f7829b97 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailersend/integration_tests/sample_config.json @@ -0,0 +1,6 @@ +{ + "api_token": "", + "domain_id": "0p7kx4xn7vg9yjree", + "date_from": "1666963299", + "date_to": "1667049682" +} diff --git a/airbyte-integrations/connectors/source-mailersend/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-mailersend/integration_tests/sample_state.json new file mode 100644 index 0000000000000..d4bd418eba241 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailersend/integration_tests/sample_state.json @@ -0,0 +1,26 @@ +{ + "activity": { + "id": "635d289a9d503fb0760382ba", + "created_at": "2022-10-29T13:20:26.020000Z", + "updated_at": "2022-10-29T13:20:26.020000Z", + "type": "soft_bounced", + "email": { + "id": "635d1a39e8e985ebb60bb41e", + "from": "no-reply@exemple.com", + "subject": "subject", + "text": null, + "html": null, + "status": "rejected", + "tags": null, + "created_at": "2022-10-29T12:19:05.181000Z", + "updated_at": "2022-10-29T13:20:25.998000Z", + "recipient": { + "id": "62c34cacaf1b3db0da0f6ee9", + "email": "test@exemple.com", + "created_at": "2022-07-04T20:25:16.846000Z", + "updated_at": "2022-09-19T22:47:26.486000Z", + "deleted_at": "2022-09-19T22:47:26.486000Z" + } + } + } +} diff --git a/airbyte-integrations/connectors/source-mailersend/main.py b/airbyte-integrations/connectors/source-mailersend/main.py new file mode 100644 index 0000000000000..aff885f4889b3 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailersend/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_mailersend import SourceMailersend + +if __name__ == "__main__": + source = SourceMailersend() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-mailersend/requirements.txt b/airbyte-integrations/connectors/source-mailersend/requirements.txt new file mode 100644 index 0000000000000..0411042aa0911 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailersend/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-mailersend/setup.py b/airbyte-integrations/connectors/source-mailersend/setup.py new file mode 100644 index 0000000000000..2ac82c32b78c9 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailersend/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_mailersend", + description="Source implementation for Mailersend.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-mailersend/source_mailersend/__init__.py b/airbyte-integrations/connectors/source-mailersend/source_mailersend/__init__.py new file mode 100644 index 0000000000000..3ba312859ad86 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailersend/source_mailersend/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceMailersend + +__all__ = ["SourceMailersend"] diff --git a/airbyte-integrations/connectors/source-mailersend/source_mailersend/mailersend.yaml b/airbyte-integrations/connectors/source-mailersend/source_mailersend/mailersend.yaml new file mode 100644 index 0000000000000..e7b4af2684ba7 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailersend/source_mailersend/mailersend.yaml @@ -0,0 +1,50 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + field_pointer: ["data"] + requester: + url_base: "https://api.mailersend.com/v1" + http_method: "GET" + authenticator: + type: BearerAuthenticator + api_token: "{{ config['api_token'] }}" + request_options_provider: + request_parameters: + limit: "100" + date_from: "{{ config['date_from'] }}" + date_to: "{{ config['date_to'] }}" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: "DefaultPaginator" + page_size_option: + inject_into: "request_parameter" + field_name: "page_size" + pagination_strategy: + type: "PageIncrement" + page_size: 100 + page_token_option: + inject_into: "request_parameter" + field_name: "page" + url_base: "https://api.mailersend.com/v1" + requester: + $ref: "*ref(definitions.requester)" + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + activity_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "activity" + primary_key: "id" + path: "/activity/{{ config['domain_id'] }}" + +streams: + - "*ref(definitions.activity_stream)" + +check: + stream_names: + - "activity" diff --git a/airbyte-integrations/connectors/source-mailersend/source_mailersend/schemas/activity.json b/airbyte-integrations/connectors/source-mailersend/source_mailersend/schemas/activity.json new file mode 100644 index 0000000000000..7831357119793 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailersend/source_mailersend/schemas/activity.json @@ -0,0 +1,76 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["string"] + }, + "created_at": { + "type": ["string"], + "format": "date-time" + }, + "updated_at": { + "type": ["string"], + "format": "date-time" + }, + "type": { + "type": ["string"] + }, + "email": { + "type": ["object"], + "properties": { + "id": { + "type": ["string"] + }, + "from": { + "type": ["string"] + }, + "subject": { + "type": ["string"] + }, + "text": { + "type": ["null", "string"] + }, + "html": { + "type": ["null", "string"] + }, + "status": { + "type": ["string"] + }, + "tags": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["string"], + "format": "date-time" + }, + "updated_at": { + "type": ["string"], + "format": "date-time" + }, + "recipient": { + "type": ["object"], + "properties": { + "id": { + "type": ["string"] + }, + "email": { + "type": ["string"] + }, + "created_at": { + "type": ["string"], + "format": "date-time" + }, + "updated_at": { + "type": ["string"], + "format": "date-time" + }, + "deleted_at": { + "type": ["string"] + } + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-mailersend/source_mailersend/source.py b/airbyte-integrations/connectors/source-mailersend/source_mailersend/source.py new file mode 100644 index 0000000000000..b0b111a3883cb --- /dev/null +++ b/airbyte-integrations/connectors/source-mailersend/source_mailersend/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceMailersend(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "mailersend.yaml"}) diff --git a/airbyte-integrations/connectors/source-mailersend/source_mailersend/spec.yaml b/airbyte-integrations/connectors/source-mailersend/source_mailersend/spec.yaml new file mode 100644 index 0000000000000..591963eca12cd --- /dev/null +++ b/airbyte-integrations/connectors/source-mailersend/source_mailersend/spec.yaml @@ -0,0 +1,29 @@ +documentationUrl: https://docsurl.com +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Mailersend Spec + type: object + required: + - api_token + - domain_id + additionalProperties: true + properties: + api_token: + type: string + description: Your API Token. See here. + domain_id: + type: string + description: The domain entity in mailersend + examples: + - airbyte.com + - linkana.com + date_from: + type: string + description: Timestamp is assumed to be UTC. Must be lower than date_to. + examples: + - "1443651141" + date_to: + type: string + description: Timestamp is assumed to be UTC. Must be higher than date_from. + examples: + - "1443651141" diff --git a/docs/integrations/sources/mailersend.md b/docs/integrations/sources/mailersend.md new file mode 100644 index 0000000000000..5d4601745480a --- /dev/null +++ b/docs/integrations/sources/mailersend.md @@ -0,0 +1,27 @@ +# Mailersend + +## Sync overview + +This source can sync data from the [Mailersend](https://developers.mailersend.com/#mailersend-api). At present this connector only supports full refresh syncs meaning that each time you use the connector it will sync all available records from scratch. + +## This Source Supports the Following Streams + +- [activity](https://developers.mailersend.com/api/v1/activity.html#get-a-list-of-activities) + +### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :---------------- | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental Sync | No | | + +### Performance considerations + +MailerSend has a default [rate limit](https://developers.mailersend.com/general.html#api-response) of 60 requests per minute on general API endpoints. + +## Getting started + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--- | :----------- | :------ | From aba40d2d485beb0329759a62f82ff18b3d576a76 Mon Sep 17 00:00:00 2001 From: Cirdes Henrique Date: Mon, 31 Oct 2022 19:52:05 -0300 Subject: [PATCH 2/8] first review --- .../source-mailersend/source_mailersend/mailersend.yaml | 2 +- .../connectors/source-mailersend/source_mailersend/spec.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-mailersend/source_mailersend/mailersend.yaml b/airbyte-integrations/connectors/source-mailersend/source_mailersend/mailersend.yaml index e7b4af2684ba7..5d564dafab869 100644 --- a/airbyte-integrations/connectors/source-mailersend/source_mailersend/mailersend.yaml +++ b/airbyte-integrations/connectors/source-mailersend/source_mailersend/mailersend.yaml @@ -29,7 +29,7 @@ definitions: page_token_option: inject_into: "request_parameter" field_name: "page" - url_base: "https://api.mailersend.com/v1" + url_base: "*ref(definitions.requester.url_base)" requester: $ref: "*ref(definitions.requester)" base_stream: diff --git a/airbyte-integrations/connectors/source-mailersend/source_mailersend/spec.yaml b/airbyte-integrations/connectors/source-mailersend/source_mailersend/spec.yaml index 591963eca12cd..9766edb1d5940 100644 --- a/airbyte-integrations/connectors/source-mailersend/source_mailersend/spec.yaml +++ b/airbyte-integrations/connectors/source-mailersend/source_mailersend/spec.yaml @@ -1,4 +1,4 @@ -documentationUrl: https://docsurl.com +documentationUrl: https://docs.airbyte.com/integrations/sources/mailersend connectionSpecification: $schema: http://json-schema.org/draft-07/schema# title: Mailersend Spec From 6a4d66148d363973c380375235165fd9a20b1990 Mon Sep 17 00:00:00 2001 From: Cirdes Henrique Date: Sun, 6 Nov 2022 13:21:47 -0300 Subject: [PATCH 3/8] trying to add stream_slicer --- .../integration_tests/sample_config.json | 2 +- .../source_mailersend/mailersend.yaml | 46 +++++++++++++------ .../source_mailersend/spec.yaml | 7 +-- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/airbyte-integrations/connectors/source-mailersend/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-mailersend/integration_tests/sample_config.json index 03c78f7829b97..35f141620e8b0 100644 --- a/airbyte-integrations/connectors/source-mailersend/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-mailersend/integration_tests/sample_config.json @@ -1,6 +1,6 @@ { "api_token": "", "domain_id": "0p7kx4xn7vg9yjree", - "date_from": "1666963299", + "date_from": "1667746836", "date_to": "1667049682" } diff --git a/airbyte-integrations/connectors/source-mailersend/source_mailersend/mailersend.yaml b/airbyte-integrations/connectors/source-mailersend/source_mailersend/mailersend.yaml index 5d564dafab869..18a0f7d1036c0 100644 --- a/airbyte-integrations/connectors/source-mailersend/source_mailersend/mailersend.yaml +++ b/airbyte-integrations/connectors/source-mailersend/source_mailersend/mailersend.yaml @@ -13,25 +13,44 @@ definitions: request_options_provider: request_parameters: limit: "100" - date_from: "{{ config['date_from'] }}" - date_to: "{{ config['date_to'] }}" + stream_slicer: + type: "DatetimeStreamSlicer" + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%s" + end_datetime: + datetime: "{{ now_utc() }}" + datetime_format: "%Y-%m-%d %H:%M:%S.%f%z" + step: "1d" + cursor_field: "{{ options['stream_cursor_field'] }}" + start_time_option: + field_name: "date_from" + inject_into: "request_parameter" + end_time_option: + field_name: "date_to" + inject_into: "request_parameter" + datetime_format: "%s" + paginator: + type: "DefaultPaginator" + page_size_option: + inject_into: "request_parameter" + field_name: "page_size" + pagination_strategy: + type: "PageIncrement" + page_size: 100 + page_token_option: + inject_into: "request_parameter" + field_name: "page" + url_base: "*ref(definitions.requester.url_base)" retriever: record_selector: $ref: "*ref(definitions.selector)" paginator: - type: "DefaultPaginator" - page_size_option: - inject_into: "request_parameter" - field_name: "page_size" - pagination_strategy: - type: "PageIncrement" - page_size: 100 - page_token_option: - inject_into: "request_parameter" - field_name: "page" - url_base: "*ref(definitions.requester.url_base)" + $ref: "*ref(definitions.paginator)" requester: $ref: "*ref(definitions.requester)" + stream_slicer: + $ref: "*ref(definitions.stream_slicer)" base_stream: retriever: $ref: "*ref(definitions.retriever)" @@ -41,6 +60,7 @@ definitions: name: "activity" primary_key: "id" path: "/activity/{{ config['domain_id'] }}" + stream_cursor_field: "created_at" streams: - "*ref(definitions.activity_stream)" diff --git a/airbyte-integrations/connectors/source-mailersend/source_mailersend/spec.yaml b/airbyte-integrations/connectors/source-mailersend/source_mailersend/spec.yaml index 9766edb1d5940..3f3d05b6db1ac 100644 --- a/airbyte-integrations/connectors/source-mailersend/source_mailersend/spec.yaml +++ b/airbyte-integrations/connectors/source-mailersend/source_mailersend/spec.yaml @@ -17,13 +17,8 @@ connectionSpecification: examples: - airbyte.com - linkana.com - date_from: + start_date: type: string description: Timestamp is assumed to be UTC. Must be lower than date_to. examples: - "1443651141" - date_to: - type: string - description: Timestamp is assumed to be UTC. Must be higher than date_from. - examples: - - "1443651141" From e7ccdee24dc8a5feec1df54e1198b6e29bb21f14 Mon Sep 17 00:00:00 2001 From: Cirdes Henrique Date: Sun, 6 Nov 2022 13:27:33 -0300 Subject: [PATCH 4/8] change spec.yml --- .../connectors/source-mailersend/source_mailersend/spec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-mailersend/source_mailersend/spec.yaml b/airbyte-integrations/connectors/source-mailersend/source_mailersend/spec.yaml index 3f3d05b6db1ac..978322fe38d8b 100644 --- a/airbyte-integrations/connectors/source-mailersend/source_mailersend/spec.yaml +++ b/airbyte-integrations/connectors/source-mailersend/source_mailersend/spec.yaml @@ -21,4 +21,4 @@ connectionSpecification: type: string description: Timestamp is assumed to be UTC. Must be lower than date_to. examples: - - "1443651141" + - "2022-11-06" From 541bcaa327ad302b7cf294b098ffc45822cfe980 Mon Sep 17 00:00:00 2001 From: Cirdes Henrique Date: Fri, 11 Nov 2022 10:40:30 -0300 Subject: [PATCH 5/8] add streamslicer --- airbyte-integrations/connectors/source-mailersend/setup.py | 2 +- .../source-mailersend/source_mailersend/mailersend.yaml | 7 ++----- .../source-mailersend/source_mailersend/spec.yaml | 5 +++-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/connectors/source-mailersend/setup.py b/airbyte-integrations/connectors/source-mailersend/setup.py index 2ac82c32b78c9..91769700bd11c 100644 --- a/airbyte-integrations/connectors/source-mailersend/setup.py +++ b/airbyte-integrations/connectors/source-mailersend/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk~=0.4", ] TEST_REQUIREMENTS = [ diff --git a/airbyte-integrations/connectors/source-mailersend/source_mailersend/mailersend.yaml b/airbyte-integrations/connectors/source-mailersend/source_mailersend/mailersend.yaml index 18a0f7d1036c0..11dd18976de75 100644 --- a/airbyte-integrations/connectors/source-mailersend/source_mailersend/mailersend.yaml +++ b/airbyte-integrations/connectors/source-mailersend/source_mailersend/mailersend.yaml @@ -19,16 +19,13 @@ definitions: datetime: "{{ config['start_date'] }}" datetime_format: "%s" end_datetime: - datetime: "{{ now_utc() }}" - datetime_format: "%Y-%m-%d %H:%M:%S.%f%z" + datetime: "{{ now_utc().strftime('%s') }}" + datetime_format: "%s" step: "1d" cursor_field: "{{ options['stream_cursor_field'] }}" start_time_option: field_name: "date_from" inject_into: "request_parameter" - end_time_option: - field_name: "date_to" - inject_into: "request_parameter" datetime_format: "%s" paginator: type: "DefaultPaginator" diff --git a/airbyte-integrations/connectors/source-mailersend/source_mailersend/spec.yaml b/airbyte-integrations/connectors/source-mailersend/source_mailersend/spec.yaml index 978322fe38d8b..ed823f41877ca 100644 --- a/airbyte-integrations/connectors/source-mailersend/source_mailersend/spec.yaml +++ b/airbyte-integrations/connectors/source-mailersend/source_mailersend/spec.yaml @@ -18,7 +18,8 @@ connectionSpecification: - airbyte.com - linkana.com start_date: - type: string + type: number description: Timestamp is assumed to be UTC. Must be lower than date_to. examples: - - "2022-11-06" + # - "2022-11-06" + - 123131321 From 3b5566a013eea0feb428e82d6556d1d5ce913c73 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Fri, 11 Nov 2022 17:00:04 -0300 Subject: [PATCH 6/8] Update spec.yaml --- .../connectors/source-mailersend/source_mailersend/spec.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-mailersend/source_mailersend/spec.yaml b/airbyte-integrations/connectors/source-mailersend/source_mailersend/spec.yaml index ed823f41877ca..96d6bb18f9964 100644 --- a/airbyte-integrations/connectors/source-mailersend/source_mailersend/spec.yaml +++ b/airbyte-integrations/connectors/source-mailersend/source_mailersend/spec.yaml @@ -11,6 +11,7 @@ connectionSpecification: api_token: type: string description: Your API Token. See here. + airbyte_secret: true domain_id: type: string description: The domain entity in mailersend @@ -19,7 +20,6 @@ connectionSpecification: - linkana.com start_date: type: number - description: Timestamp is assumed to be UTC. Must be lower than date_to. + description: Timestamp is assumed to be UTC. examples: - # - "2022-11-06" - 123131321 From 306c9aaabc28ad2f7cc7f6949a64532f1685b229 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Fri, 11 Nov 2022 18:11:22 -0300 Subject: [PATCH 7/8] add mailersend to source def --- .../init/src/main/resources/seed/source_definitions.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 4cb1349174bb6..dc851980a507c 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -823,6 +823,13 @@ documentationUrl: https://docs.airbyte.com/integrations/sources/mailerlite sourceType: api releaseStage: alpha +- name: MailerSend + sourceDefinitionId: 2707d529-3c04-46eb-9c7e-40d4038df6f7 + dockerRepository: airbyte/source-mailersend + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/mailersend + sourceType: api + releaseStage: alpha - name: Mailgun sourceDefinitionId: 5b9cb09e-1003-4f9c-983d-5779d1b2cd51 dockerRepository: airbyte/source-mailgun From defb916a6ddd0b9d6a0c582cc73c3862378b1095 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Sun, 13 Nov 2022 22:57:15 +0000 Subject: [PATCH 8/8] auto-bump connector version --- .../src/main/resources/seed/source_specs.yaml | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 6a3ff98507f42..f3baa8f468f0b 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -7305,6 +7305,37 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-mailersend:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/mailersend" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Mailersend Spec" + type: "object" + required: + - "api_token" + - "domain_id" + additionalProperties: true + properties: + api_token: + type: "string" + description: "Your API Token. See here." + airbyte_secret: true + domain_id: + type: "string" + description: "The domain entity in mailersend" + examples: + - "airbyte.com" + - "linkana.com" + start_date: + type: "number" + description: "Timestamp is assumed to be UTC." + examples: + - 123131321 + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-mailgun:0.1.0" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/mailgun"