diff --git a/airbyte-config-oss/init-oss/src/main/resources/icons/yotpo.svg b/airbyte-config-oss/init-oss/src/main/resources/icons/yotpo.svg new file mode 100644 index 00000000000000..7c095613e53582 --- /dev/null +++ b/airbyte-config-oss/init-oss/src/main/resources/icons/yotpo.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json b/airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json index c4b78be97f49ae..36aafc390887fb 100644 --- a/airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json +++ b/airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json @@ -28965,6 +28965,58 @@ "allowedHosts": { "hosts": [ "api-metrica.yandex.net" ] } + }, { + "sourceDefinitionId": "18139f00-b1ba-4971-8f80-8387b617cfd8", + "name": "Yotpo", + "dockerRepository": "airbyte/source-yotpo", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://docs.airbyte.com/integrations/sources/yotpo", + "icon": "yotpo.svg", + "sourceType": "api", + "spec": { + "documentationUrl": "https://docs.airbyte.com/integrations/sources/yotpo", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Yotpo Spec", + "type": "object", + "required": [ "access_token", "app_key", "start_date", "email" ], + "additionalProperties": true, + "properties": { + "access_token": { + "title": "Access Token", + "type": "string", + "description": "Access token recieved as a result of API call to https://api.yotpo.com/oauth/token (Ref- https://apidocs.yotpo.com/reference/yotpo-authentication)", + "airbyte_secret": true + }, + "app_key": { + "title": "App Key", + "type": "string", + "description": "App key found at settings (Ref- https://settings.yotpo.com/#/general_settings)" + }, + "start_date": { + "title": "Date-From Filter", + "type": "string", + "description": "Date time filter for incremental filter, Specify which date to extract from.", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}Z$", + "examples": [ "2022-03-01T00:00:00.000Z" ], + "format": "date-time" + }, + "email": { + "title": "Registered email address", + "type": "string", + "description": "Email address registered with yotpo.", + "default": "example@gmail.com" + } + } + }, + "supportsNormalization": false, + "supportsDBT": false, + "supported_destination_sync_modes": [ ] + }, + "tombstone": false, + "public": true, + "custom": false, + "releaseStage": "alpha" }, { "sourceDefinitionId": "9c74c2d7-531a-4ebf-b6d8-6181f805ecdc", "name": "Younium", diff --git a/airbyte-config-oss/init-oss/src/main/resources/seed/source_definitions.yaml b/airbyte-config-oss/init-oss/src/main/resources/seed/source_definitions.yaml index 680d161fae8706..8bf7dd32f466cb 100644 --- a/airbyte-config-oss/init-oss/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config-oss/init-oss/src/main/resources/seed/source_definitions.yaml @@ -2557,6 +2557,14 @@ allowedHosts: hosts: - api-metrica.yandex.net +- name: Yotpo + sourceDefinitionId: 18139f00-b1ba-4971-8f80-8387b617cfd8 + dockerRepository: airbyte/source-yotpo + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/yotpo + icon: yotpo.svg + sourceType: api + releaseStage: alpha - name: Younium sourceDefinitionId: 9c74c2d7-531a-4ebf-b6d8-6181f805ecdc dockerRepository: airbyte/source-younium diff --git a/airbyte-config-oss/init-oss/src/main/resources/seed/source_specs.yaml b/airbyte-config-oss/init-oss/src/main/resources/seed/source_specs.yaml index d18d9b09f55054..8607e7dc789f15 100644 --- a/airbyte-config-oss/init-oss/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config-oss/init-oss/src/main/resources/seed/source_specs.yaml @@ -18661,6 +18661,47 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-yotpo:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/yotpo" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Yotpo Spec" + type: "object" + required: + - "access_token" + - "app_key" + - "start_date" + - "email" + additionalProperties: true + properties: + access_token: + title: "Access Token" + type: "string" + description: "Access token recieved as a result of API call to https://api.yotpo.com/oauth/token\ + \ (Ref- https://apidocs.yotpo.com/reference/yotpo-authentication)" + airbyte_secret: true + app_key: + title: "App Key" + type: "string" + description: "App key found at settings (Ref- https://settings.yotpo.com/#/general_settings)" + start_date: + title: "Date-From Filter" + type: "string" + description: "Date time filter for incremental filter, Specify which date\ + \ to extract from." + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}Z$" + examples: + - "2022-03-01T00:00:00.000Z" + format: "date-time" + email: + title: "Registered email address" + type: "string" + description: "Email address registered with yotpo." + default: "example@gmail.com" + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-younium:0.1.0" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/younium" diff --git a/airbyte-integrations/connectors/source-yotpo/.dockerignore b/airbyte-integrations/connectors/source-yotpo/.dockerignore new file mode 100644 index 00000000000000..7b7c00a7be1ed4 --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_yotpo +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-yotpo/Dockerfile b/airbyte-integrations/connectors/source-yotpo/Dockerfile new file mode 100644 index 00000000000000..2616324339ad39 --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_yotpo ./source_yotpo + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-yotpo diff --git a/airbyte-integrations/connectors/source-yotpo/README.md b/airbyte-integrations/connectors/source-yotpo/README.md new file mode 100644 index 00000000000000..d0ba27b612ed67 --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/README.md @@ -0,0 +1,82 @@ +# Yotpo Source + +This is the repository for the Yotpo configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/yotpo). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-yotpo:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/yotpo) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_yotpo/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source yotpo test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-yotpo:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-yotpo:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-yotpo:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-yotpo:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-yotpo:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-yotpo:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with Docker, run: +``` +./acceptance-test-docker.sh +``` + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-yotpo:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-yotpo:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-yotpo/__init__.py b/airbyte-integrations/connectors/source-yotpo/__init__.py new file mode 100644 index 00000000000000..c941b30457953b --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-yotpo/acceptance-test-config.yml b/airbyte-integrations/connectors/source-yotpo/acceptance-test-config.yml new file mode 100644 index 00000000000000..1d6d15b8872519 --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/acceptance-test-config.yml @@ -0,0 +1,44 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-yotpo:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_yotpo/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: "raw_data" + bypass_reason: "Sandbox account cannot seed the endpoint" + - name: "unsubscribers" + bypass_reason: "Sandbox account cannot seed the endpoint" + - name: "webhooks" + bypass_reason: "Sandbox account cannot seed the endpoint" + - name: "webhook_events" + bypass_reason: "Sandbox account cannot seed the endpoint" + expect_records: + path: "integration_tests/expected_records.jsonl" + extra_fields: no + exact_order: no + extra_records: yes + incremental: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-yotpo/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-yotpo/acceptance-test-docker.sh new file mode 100755 index 00000000000000..b6d65deeccb436 --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/acceptance-test-docker.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env sh + +source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-yotpo/build.gradle b/airbyte-integrations/connectors/source-yotpo/build.gradle new file mode 100644 index 00000000000000..83058c6777f4fb --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-connector-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_yotpo' +} diff --git a/airbyte-integrations/connectors/source-yotpo/integration_tests/__init__.py b/airbyte-integrations/connectors/source-yotpo/integration_tests/__init__.py new file mode 100644 index 00000000000000..c941b30457953b --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-yotpo/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-yotpo/integration_tests/abnormal_state.json new file mode 100644 index 00000000000000..c044edae232610 --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/integration_tests/abnormal_state.json @@ -0,0 +1,9 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "created_at": "2099-04-12T18:13:36.000Z" }, + "stream_descriptor": { "name": "reviews" } + } + } +] diff --git a/airbyte-integrations/connectors/source-yotpo/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-yotpo/integration_tests/acceptance.py new file mode 100644 index 00000000000000..9e6409236281ff --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("connector_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-yotpo/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-yotpo/integration_tests/configured_catalog.json new file mode 100644 index 00000000000000..4f5228e40e1dc7 --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/integration_tests/configured_catalog.json @@ -0,0 +1,58 @@ +{ + "streams": [ + { + "stream": { + "name": "email_analytics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "raw_data", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "reviews", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "unsubscribers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "webhooks", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "webhook_events", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-yotpo/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-yotpo/integration_tests/expected_records.jsonl new file mode 100644 index 00000000000000..84f5c0a95f2f9c --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/integration_tests/expected_records.jsonl @@ -0,0 +1 @@ +{"stream": "reviews", "data":{"id":465176213,"title":"Great Phone","content":"It’s really good","score":5,"votes_up":0,"votes_down":0,"created_at":"2023-04-27T18:23:01.000Z","updated_at":"2023-04-27T18:24:42.000Z","sentiment":0.960069,"sku":"100","name":"John S.","email":"moles57397@snowlash.com","reviewer_type":"verified_reviewer","deleted":false,"archived":false,"escalated":false,"is_incentivized":false}, "emitted_at": 1679051186823} diff --git a/airbyte-integrations/connectors/source-yotpo/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-yotpo/integration_tests/invalid_config.json new file mode 100644 index 00000000000000..c266c8bd0766b8 --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/integration_tests/invalid_config.json @@ -0,0 +1,6 @@ +{ + "app_key": "", + "access_token": "", + "start_date": "2099-04-12T18:42:50.000Z", + "email": "" +} diff --git a/airbyte-integrations/connectors/source-yotpo/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-yotpo/integration_tests/sample_config.json new file mode 100644 index 00000000000000..d654106695ac01 --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/integration_tests/sample_config.json @@ -0,0 +1,5 @@ +{ + "app_key": "XXXXXXXXXXXXXXXXXXX", + "access_token": "YYYYYYYYYYYYYYYY", + "email": "example@gmail.com" +} diff --git a/airbyte-integrations/connectors/source-yotpo/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-yotpo/integration_tests/sample_state.json new file mode 100644 index 00000000000000..0c018bd01cfccc --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/integration_tests/sample_state.json @@ -0,0 +1,9 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "created_at": "2023-04-12T18:13:36.000Z" }, + "stream_descriptor": { "name": "reviews" } + } + } +] diff --git a/airbyte-integrations/connectors/source-yotpo/main.py b/airbyte-integrations/connectors/source-yotpo/main.py new file mode 100644 index 00000000000000..d456ca2abab3de --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_yotpo import SourceYotpo + +if __name__ == "__main__": + source = SourceYotpo() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-yotpo/requirements.txt b/airbyte-integrations/connectors/source-yotpo/requirements.txt new file mode 100644 index 00000000000000..cc57334ef619a3 --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/connector-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-yotpo/setup.py b/airbyte-integrations/connectors/source-yotpo/setup.py new file mode 100644 index 00000000000000..4714ade53c354e --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.2", + "pytest-mock~=3.6.1", + "connector-acceptance-test", +] + +setup( + name="source_yotpo", + description="Source implementation for Yotpo.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-yotpo/source_yotpo/__init__.py b/airbyte-integrations/connectors/source-yotpo/source_yotpo/__init__.py new file mode 100644 index 00000000000000..ccbbe46ad976fe --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/source_yotpo/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceYotpo + +__all__ = ["SourceYotpo"] diff --git a/airbyte-integrations/connectors/source-yotpo/source_yotpo/manifest.yaml b/airbyte-integrations/connectors/source-yotpo/source_yotpo/manifest.yaml new file mode 100644 index 00000000000000..518ebaf4bbca1f --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/source_yotpo/manifest.yaml @@ -0,0 +1,136 @@ +version: "0.29.0" + +definitions: + yotpo_extractor: + type: DpathExtractor + field_path: ["{{ parameters['extractorPath'] }}"] + + yotpo_stream_requester: + type: HttpRequester + url_base: "{{ parameters['requesterUrl'] }}" + http_method: "GET" + request_parameters: + utoken: "{{ config['access_token'] }}" + + yotpo_paginator: + type: "DefaultPaginator" + page_size_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "count" + pagination_strategy: + type: "OffsetIncrement" + page_size: 1 + page_token_option: + type: "RequestOption" + field_name: "page" + inject_into: "request_parameter" + + yotpo_base: + type: DeclarativeStream + retriever: + type: SimpleRetriever + record_selector: + type: RecordSelector + extractor: + $ref: "#/definitions/yotpo_extractor" + paginator: + $ref: "#/definitions/yotpo_paginator" + requester: + $ref: "#/definitions/yotpo_stream_requester" + + yotpo_base_without_pagination: + type: DeclarativeStream + retriever: + type: SimpleRetriever + record_selector: + type: RecordSelector + extractor: + $ref: "#/definitions/yotpo_extractor" + paginator: + type: NoPagination + requester: + $ref: "#/definitions/yotpo_stream_requester" + + reviews_stream: + $parameters: + extractorPath: "reviews" + path: "/apps/{{ config['app_key'] }}/reviews" + requesterUrl: "https://api.yotpo.com/v1" + $ref: "#/definitions/yotpo_base" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: "created_at" + datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z" + cursor_granularity: "PT0.000001S" + lookback_window: "P31D" + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z" + end_datetime: + datetime: "{{ today_utc() }}" + datetime_format: "%Y-%m-%d" + step: "P1M" + name: "reviews" + primary_key: "id" + + email_analytics_stream: + $parameters: + extractorPath: "date_series_points" + path: "/emails/{{ config['app_key'] }}/emails_sent" + requesterUrl: "https://api.yotpo.com/analytics/v1" + $ref: "#/definitions/yotpo_base_without_pagination" + name: "email_analytics" + + raw_data_stream: + $parameters: + extractorPath: "records" + path: "/emails/{{ config['app_key'] }}/export/raw_data" + requesterUrl: "https://api.yotpo.com/analytics/v1" + $ref: "#/definitions/yotpo_base_without_pagination" + name: "raw_data" + + unsubscribers_stream: + $parameters: + extractorPath: | + "response", "unsubscribers" + requesterUrl: "https://api.yotpo.com" + path: "/apps/{{ config['app_key'] }}/unsubscribers" + $ref: "#/definitions/yotpo_base" + name: "unsubscribers" + + webhook_events_stream: + $parameters: + extractorPath: | + "response", "webhook_events" + requesterUrl: "https://api.yotpo.com" + path: "/webhook_events" + $ref: "#/definitions/yotpo_base_without_pagination" + name: "webhook_events" + + webhooks_stream: + $parameters: + extractorPath: | + "response", "webhooks" + requesterUrl: "https://api.yotpo.com" + path: "/apps/{{ config['app_key'] }}/webhooks" + $ref: "#/definitions/yotpo_base_without_pagination" + name: "webhooks" + +streams: + - "#/definitions/email_analytics_stream" + - "#/definitions/raw_data_stream" + - "#/definitions/reviews_stream" + - "#/definitions/webhooks_stream" + - "#/definitions/unsubscribers_stream" + - "#/definitions/webhook_events_stream" + +check: + type: CheckStream + stream_names: + - "email_analytics" + - "raw_data" + - "reviews" + - "unsubscribers" + - "webhooks" + - "webhook_events" diff --git a/airbyte-integrations/connectors/source-yotpo/source_yotpo/schemas/email_analytics.json b/airbyte-integrations/connectors/source-yotpo/source_yotpo/schemas/email_analytics.json new file mode 100644 index 00000000000000..22f32bf9e14097 --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/source_yotpo/schemas/email_analytics.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Email Analytics", + "additionalProperties": true, + "type": "object", + "properties": { + "since": { + "type": ["null", "string"] + }, + "until": { + "type": ["null", "string"] + }, + "values": { + "type": ["null", "object"], + "properties": { + "done": { + "type": ["null", "boolean"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-yotpo/source_yotpo/schemas/raw_data.json b/airbyte-integrations/connectors/source-yotpo/source_yotpo/schemas/raw_data.json new file mode 100644 index 00000000000000..ff2feeac9183b9 --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/source_yotpo/schemas/raw_data.json @@ -0,0 +1,71 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Raw Data Schema", + "additionalProperties": true, + "type": "object", + "properties": { + "email_address": { + "type": ["null", "string"] + }, + "order_id": { + "type": ["null", "string"] + }, + "order_timestamp": { + "type": ["null", "string"] + }, + "product_id": { + "type": ["null", "string"] + }, + "sku": { + "type": ["null", "number"] + }, + "email_type": { + "type": ["null", "string"] + }, + "reminder_num": { + "type": ["null", "number"] + }, + "trr_bundle_id": { + "type": ["null", "string"] + }, + "trr_bundle_subject": { + "type": ["null", "string"] + }, + "review_type": { + "type": ["null", "string"] + }, + "coupon_code": { + "type": ["null", "number"] + }, + "email_sent_timestamp": { + "type": ["null", "string"] + }, + "opened_timestamp": { + "type": ["null", "string"] + }, + "clicked_through_timestamp": { + "type": ["null", "string"] + }, + "content_creation_timestamp": { + "type": ["null", "string"] + }, + "platform": { + "type": ["null", "string"] + }, + "invalid_address_timestamp": { + "type": ["null", "string"] + }, + "failed_timestamp": { + "type": ["null", "string"] + }, + "unsubscribed_timestamp": { + "type": ["null", "string"] + }, + "marked_spam_timestamp": { + "type": ["null", "string"] + }, + "arrived_early_timestamp": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-yotpo/source_yotpo/schemas/reviews.json b/airbyte-integrations/connectors/source-yotpo/source_yotpo/schemas/reviews.json new file mode 100644 index 00000000000000..3710a009987c0c --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/source_yotpo/schemas/reviews.json @@ -0,0 +1,64 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Reviews Schema", + "additionalProperties": true, + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "title": { + "type": ["null", "string"] + }, + "incentive_type": { + "type": ["null", "string"] + }, + "content": { + "type": ["null", "string"] + }, + "score": { + "type": ["null", "number"] + }, + "votes_up": { + "type": ["null", "number"] + }, + "votes_down": { + "type": ["null", "number"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "sentiment": { + "type": ["null", "number"] + }, + "sku": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "reviewer_type": { + "type": ["null", "string"] + }, + "deleted": { + "type": ["null", "boolean"] + }, + "archived": { + "type": ["null", "boolean"] + }, + "escalated": { + "type": ["null", "boolean"] + }, + "is_incentivized": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-yotpo/source_yotpo/schemas/unsubscribers.json b/airbyte-integrations/connectors/source-yotpo/source_yotpo/schemas/unsubscribers.json new file mode 100644 index 00000000000000..e3426d28a71fcf --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/source_yotpo/schemas/unsubscribers.json @@ -0,0 +1,20 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Unsubscribers Schema", + "additionalProperties": true, + "type": "object", + "properties": { + "id": { + "type": ["null", "number"] + }, + "user_email": { + "type": ["null", "string"] + }, + "email_type_id": { + "type": ["null", "number"] + }, + "unsubscribed_by_name": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-yotpo/source_yotpo/schemas/webhook_events.json b/airbyte-integrations/connectors/source-yotpo/source_yotpo/schemas/webhook_events.json new file mode 100644 index 00000000000000..58ff83078426c5 --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/source_yotpo/schemas/webhook_events.json @@ -0,0 +1,14 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Webhook events Schema", + "additionalProperties": true, + "type": "object", + "properties": { + "name": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-yotpo/source_yotpo/schemas/webhooks.json b/airbyte-integrations/connectors/source-yotpo/source_yotpo/schemas/webhooks.json new file mode 100644 index 00000000000000..b93f4e074aea66 --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/source_yotpo/schemas/webhooks.json @@ -0,0 +1,23 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Webhooks Schema", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "number"] + }, + "webhook_event_id": { + "type": ["null", "number"] + }, + "webhook_event_name": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "version": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-yotpo/source_yotpo/source.py b/airbyte-integrations/connectors/source-yotpo/source_yotpo/source.py new file mode 100644 index 00000000000000..dc275c06fdcda2 --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/source_yotpo/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceYotpo(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-yotpo/source_yotpo/spec.yaml b/airbyte-integrations/connectors/source-yotpo/source_yotpo/spec.yaml new file mode 100644 index 00000000000000..6e7d6441711a1e --- /dev/null +++ b/airbyte-integrations/connectors/source-yotpo/source_yotpo/spec.yaml @@ -0,0 +1,34 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/yotpo +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Yotpo Spec + type: object + additionalProperties: true + required: + - access_token + - app_key + - start_date + - email + properties: + access_token: + title: Access Token + type: string + description: Access token recieved as a result of API call to https://api.yotpo.com/oauth/token (Ref- https://apidocs.yotpo.com/reference/yotpo-authentication) + airbyte_secret: true + app_key: + title: App Key + type: string + description: App key found at settings (Ref- https://settings.yotpo.com/#/general_settings) + start_date: + title: Date-From Filter + type: string + description: Date time filter for incremental filter, Specify which date to extract from. + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}Z$" + examples: + - "2022-03-01T00:00:00.000Z" + format: "date-time" + email: + title: Registered email address + type: string + description: Email address registered with yotpo. + default: example@gmail.com diff --git a/docs/integrations/sources/yotpo.md b/docs/integrations/sources/yotpo.md new file mode 100644 index 00000000000000..c419b4b4442532 --- /dev/null +++ b/docs/integrations/sources/yotpo.md @@ -0,0 +1,71 @@ +# Yotpo + +This page contains the setup guide and reference information for the [Yotpo](https://apidocs.yotpo.com/reference/welcome) source + +## Prerequisites + +Access Token (which acts as bearer token) is mandate for this connector to work, It could be generated from the auth token call (ref - https://apidocs.yotpo.com/reference/yotpo-authentication). + +## Setup guide + +### Step 1: Set up Yotpo connection + +- Generate an Yotpo access token via auth endpoint (ref - https://apidocs.yotpo.com/reference/yotpo-authentication) +- Setup params (All params are required) +- Available params + - access_token: The generated access token + - app_key: Seen at the yotpo settings (ref - https://settings.yotpo.com/#/general_settings) + - start_date: Date filter for eligible streams, enter + - email: Registered email address + +## Step 2: Set up the Yotpo connector in Airbyte + +### For Airbyte Cloud: + +1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**. +3. On the Set up the source page, enter the name for the Yotpo connector and select **Yotpo** from the Source type dropdown. +4. Enter your `access_token, app_key, start_date and email`. +5. Click **Set up source**. + +### For Airbyte OSS: + +1. Navigate to the Airbyte Open Source dashboard. +2. Set the name for your source. +3. Enter your `access_token, app_key, start_date and email`. +5. Click **Set up source**. + +## Supported sync modes + +The Yotpo source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): + +| Feature | Supported? | +| :---------------------------- | :--------- | +| Full Refresh Sync | Yes | +| Incremental Sync | Yes | +| Replicate Incremental Deletes | No | +| SSL connection | Yes | +| Namespaces | No | + +## Supported Streams + +- email_analytics +- raw_data +- reviews +- unsubscribers +- webhooks +- webhook_events + +## API method example + +GET https://api.yotpo.com/v1/apps/APPAAAAAATTTTTTDDDDDD/reviews?utoken=abcdefghikjlimls + +## Performance considerations + +Yotpo [API reference](https://api.yotpo.com/v1/) has v1 at present. The connector as default uses v1 and changed according to different endpoints. + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :----------------------------------------------------- | :------------- | +| 0.1.0 | 2023-04-14 | [Init](https://github.com/airbytehq/airbyte/pull/25532)| Initial commit |