diff --git a/airbyte-config/init/src/main/resources/icons/coingeckocoins.svg b/airbyte-config/init/src/main/resources/icons/coingeckocoins.svg new file mode 100644 index 0000000000000..f64a48e2e5349 --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/coingeckocoins.svg @@ -0,0 +1 @@ +CoinGecko \ No newline at end of file diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 942173cc2398e..843fa36d9a78f 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -246,6 +246,14 @@ icon: close.svg sourceType: api releaseStage: alpha +- name: CoinGecko Coins + sourceDefinitionId: 9cdd4183-d0ba-40c3-aad3-6f46d4103974 + dockerRepository: airbyte/source-coingecko-coins + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/coingecko-coins + icon: coingeckocoins.svg + sourceType: api + releaseStage: alpha - name: Cockroachdb sourceDefinitionId: 9fa5862c-da7c-11eb-8d19-0242ac130003 dockerRepository: airbyte/source-cockroachdb diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 9ac655fdf4ead..b81c8bd0d1c9f 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -2353,6 +2353,65 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-coingecko-coins:0.1.0" + spec: + documentationUrl: "https://docsurl.com" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "CoinGecko Coins Spec" + type: "object" + required: + - "coin_id" + - "vs_currency" + - "days" + - "start_date" + additionalProperties: true + properties: + api_key: + type: "string" + description: "API Key (for pro users)" + airbyte_secret: true + order: 0 + coin_id: + type: "string" + description: "CoinGecko coin ID (e.g. bitcoin). Can be retrieved from the\n\ + `/coins/list` endpoint.\n" + order: 1 + vs_currency: + type: "string" + description: "The target currency of market data (e.g. usd, eur, jpy, etc.)\n" + order: 2 + days: + type: "string" + description: "The number of days of data for market chart.\n" + enum: + - "1" + - "7" + - "14" + - "30" + - "90" + - "180" + - "365" + - "max" + default: "30" + order: 3 + start_date: + type: "string" + description: "The start date for the historical data stream in dd-mm-yyyy\ + \ format.\n" + format: "date" + pattern: "^[0-9]{2}-[0-9]{2}-[0-9]{4}$" + order: 4 + end_date: + type: "string" + description: "The end date for the historical data stream in dd-mm-yyyy\ + \ format.\n" + format: "date" + pattern: "^[0-9]{2}-[0-9]{2}-[0-9]{4}$" + order: 5 + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-cockroachdb:0.1.18" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/cockroachdb" diff --git a/airbyte-integrations/connectors/source-coingecko-coins/.dockerignore b/airbyte-integrations/connectors/source-coingecko-coins/.dockerignore new file mode 100644 index 0000000000000..6fef53fe07d7e --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_coingecko_coins +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-coingecko-coins/Dockerfile b/airbyte-integrations/connectors/source-coingecko-coins/Dockerfile new file mode 100644 index 0000000000000..df43451220b87 --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_coingecko_coins ./source_coingecko_coins + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-coingecko-coins diff --git a/airbyte-integrations/connectors/source-coingecko-coins/README.md b/airbyte-integrations/connectors/source-coingecko-coins/README.md new file mode 100644 index 0000000000000..3b79c356b80b0 --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/README.md @@ -0,0 +1,79 @@ +# Coingecko Coins Source + +This is the repository for the Coingecko Coins configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/coingecko-coins). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-coingecko-coins:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/coingecko-coins) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_coingecko_coins/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source coingecko-coins test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-coingecko-coins:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-coingecko-coins:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-coingecko-coins:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-coingecko-coins:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-coingecko-coins:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-coingecko-coins:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-coingecko-coins:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-coingecko-coins:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-coingecko-coins/__init__.py b/airbyte-integrations/connectors/source-coingecko-coins/__init__.py new file mode 100644 index 0000000000000..1100c1c58cf51 --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-coingecko-coins/acceptance-test-config.yml b/airbyte-integrations/connectors/source-coingecko-coins/acceptance-test-config.yml new file mode 100644 index 0000000000000..02f8962dd2585 --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/acceptance-test-config.yml @@ -0,0 +1,20 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-coingecko-coins:dev +tests: + spec: + - spec_path: "source_coingecko_coins/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-coingecko-coins/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-coingecko-coins/acceptance-test-docker.sh new file mode 100644 index 0000000000000..c51577d10690c --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-coingecko-coins/build.gradle b/airbyte-integrations/connectors/source-coingecko-coins/build.gradle new file mode 100644 index 0000000000000..3f201f518dc6f --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_coingecko_coins' +} diff --git a/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/__init__.py b/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/__init__.py new file mode 100644 index 0000000000000..1100c1c58cf51 --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/abnormal_state.json new file mode 100644 index 0000000000000..52b0f2c2118f4 --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/acceptance.py new file mode 100644 index 0000000000000..1302b2f57e10e --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/catalog.json b/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/catalog.json new file mode 100644 index 0000000000000..2b5311cc2a311 --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/catalog.json @@ -0,0 +1,17 @@ +{ + "streams": [ + { + "name": "market_chart", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, + "json_schema": {} + }, + { + "name": "history", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": "date", + "json_schema": {} + } + ] +} diff --git a/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..5d190e52a5851 --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/configured_catalog.json @@ -0,0 +1,22 @@ +{ + "streams": [ + { + "stream": { + "name": "market_chart", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "history", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/invalid_config.json new file mode 100644 index 0000000000000..fcb72cfdc8540 --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/invalid_config.json @@ -0,0 +1,6 @@ +{ + "coin_id": "bitcoin", + "vs_currency": "usd", + "days": "1", + "start_date": "40-10-2022" +} diff --git a/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/sample_config.json new file mode 100644 index 0000000000000..3300694916604 --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/sample_config.json @@ -0,0 +1,6 @@ +{ + "coin_id": "bitcoin", + "vs_currency": "usd", + "days": "1", + "start_date": "10-10-2022" +} diff --git a/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/sample_state.json new file mode 100644 index 0000000000000..3587e579822d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-coingecko-coins/main.py b/airbyte-integrations/connectors/source-coingecko-coins/main.py new file mode 100644 index 0000000000000..b41f77ca37488 --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_coingecko_coins import SourceCoingeckoCoins + +if __name__ == "__main__": + source = SourceCoingeckoCoins() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-coingecko-coins/requirements.txt b/airbyte-integrations/connectors/source-coingecko-coins/requirements.txt new file mode 100644 index 0000000000000..0411042aa0911 --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-coingecko-coins/setup.py b/airbyte-integrations/connectors/source-coingecko-coins/setup.py new file mode 100644 index 0000000000000..56d30b19c39f0 --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_coingecko_coins", + description="Source implementation for Coingecko Coins.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/__init__.py b/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/__init__.py new file mode 100644 index 0000000000000..ec62bdac21408 --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceCoingeckoCoins + +__all__ = ["SourceCoingeckoCoins"] diff --git a/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/coingecko_coins.yaml b/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/coingecko_coins.yaml new file mode 100644 index 0000000000000..31a622c19add4 --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/coingecko_coins.yaml @@ -0,0 +1,70 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + field_pointer: [ ] + requester: + url_base: "https://{{ 'api' if not config['api_key'] else 'pro-api' }}.coingecko.com/api/v3/coins/{{ config['coin_id'] }}" + http_method: "GET" + request_options_provider: + request_parameters: + x_cg_pro_api_key: "{{ config['api_key'] }}" + vs_currency: "{{ config['vs_currency'] }}" + days: "{{ config['days'] }}" + stream_slicer: + type: DatetimeStreamSlicer + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%d-%m-%Y" + end_datetime: + datetime: "{{ config['end_date'] or now_utc().strftime('%d-%m-%Y') }}" + datetime_format: "%d-%m-%Y" + step: 1d + datetime_format: "%d-%m-%Y" + cursor_field: "date" + start_time_option: + field_name: "date" + inject_into: "request_parameter" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: NoPagination + requester: + $ref: "*ref(definitions.requester)" + sliced_retriever: + $ref: "*ref(definitions.retriever)" + stream_slicer: + $ref: "*ref(definitions.stream_slicer)" + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + sliced_stream: + retriever: + $ref: "*ref(definitions.sliced_retriever)" + market_chart_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "market_chart" + path: "/market_chart" + primary_key: "prices" + history_stream: + $ref: "*ref(definitions.sliced_stream)" + $options: + name: "history" + path: "/history" + transformations: + - type: AddFields + fields: + - path: [ "date" ] + value: "{{ stream_slice['start_time'] }}" + +streams: + - "*ref(definitions.market_chart_stream)" + - "*ref(definitions.history_stream)" + +check: + stream_names: + - "market_chart" + - "history" diff --git a/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/schemas/history.json b/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/schemas/history.json new file mode 100644 index 0000000000000..06fcf9b71eed0 --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/schemas/history.json @@ -0,0 +1,768 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "symbol": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "localization": { + "type": "object", + "properties": { + "en": { + "type": ["string", "null"] + }, + "de": { + "type": ["string", "null"] + }, + "es": { + "type": ["string", "null"] + }, + "fr": { + "type": ["string", "null"] + }, + "it": { + "type": ["string", "null"] + }, + "pl": { + "type": ["string", "null"] + }, + "ro": { + "type": ["string", "null"] + }, + "hu": { + "type": ["string", "null"] + }, + "nl": { + "type": ["string", "null"] + }, + "pt": { + "type": ["string", "null"] + }, + "sv": { + "type": ["string", "null"] + }, + "vi": { + "type": ["string", "null"] + }, + "tr": { + "type": ["string", "null"] + }, + "ru": { + "type": ["string", "null"] + }, + "ja": { + "type": ["string", "null"] + }, + "zh": { + "type": ["string", "null"] + }, + "zh-tw": { + "type": ["string", "null"] + }, + "ko": { + "type": ["string", "null"] + }, + "ar": { + "type": ["string", "null"] + }, + "th": { + "type": ["string", "null"] + }, + "id": { + "type": ["string", "null"] + }, + "cs": { + "type": ["string", "null"] + }, + "da": { + "type": ["string", "null"] + }, + "el": { + "type": ["string", "null"] + }, + "hi": { + "type": ["string", "null"] + }, + "no": { + "type": ["string", "null"] + }, + "sk": { + "type": ["string", "null"] + }, + "uk": { + "type": ["string", "null"] + }, + "he": { + "type": ["string", "null"] + }, + "fi": { + "type": ["string", "null"] + }, + "bg": { + "type": ["string", "null"] + }, + "hr": { + "type": ["string", "null"] + }, + "lt": { + "type": ["string", "null"] + }, + "sl": { + "type": ["string", "null"] + } + } + }, + "image": { + "type": "object", + "properties": { + "thumb": { + "type": ["string", "null"] + }, + "small": { + "type": ["string", "null"] + } + } + }, + "market_data": { + "type": "object", + "properties": { + "current_price": { + "type": "object", + "properties": { + "aed": { + "type": ["number", "null"] + }, + "ars": { + "type": ["number", "null"] + }, + "aud": { + "type": ["number", "null"] + }, + "bch": { + "type": ["number", "null"] + }, + "bdt": { + "type": ["number", "null"] + }, + "bhd": { + "type": ["number", "null"] + }, + "bmd": { + "type": ["number", "null"] + }, + "bnb": { + "type": ["number", "null"] + }, + "brl": { + "type": ["number", "null"] + }, + "btc": { + "type": ["number", "null"] + }, + "cad": { + "type": ["number", "null"] + }, + "chf": { + "type": ["number", "null"] + }, + "clp": { + "type": ["number", "null"] + }, + "cny": { + "type": ["number", "null"] + }, + "czk": { + "type": ["number", "null"] + }, + "dkk": { + "type": ["number", "null"] + }, + "dot": { + "type": ["number", "null"] + }, + "eos": { + "type": ["number", "null"] + }, + "eth": { + "type": ["number", "null"] + }, + "eur": { + "type": ["number", "null"] + }, + "gbp": { + "type": ["number", "null"] + }, + "hkd": { + "type": ["number", "null"] + }, + "huf": { + "type": ["number", "null"] + }, + "idr": { + "type": ["number", "null"] + }, + "ils": { + "type": ["number", "null"] + }, + "inr": { + "type": ["number", "null"] + }, + "jpy": { + "type": ["number", "null"] + }, + "krw": { + "type": ["number", "null"] + }, + "kwd": { + "type": ["number", "null"] + }, + "lkr": { + "type": ["number", "null"] + }, + "ltc": { + "type": ["number", "null"] + }, + "mmk": { + "type": ["number", "null"] + }, + "mxn": { + "type": ["number", "null"] + }, + "myr": { + "type": ["number", "null"] + }, + "ngn": { + "type": ["number", "null"] + }, + "nok": { + "type": ["number", "null"] + }, + "nzd": { + "type": ["number", "null"] + }, + "php": { + "type": ["number", "null"] + }, + "pkr": { + "type": ["number", "null"] + }, + "pln": { + "type": ["number", "null"] + }, + "rub": { + "type": ["number", "null"] + }, + "sar": { + "type": ["number", "null"] + }, + "sek": { + "type": ["number", "null"] + }, + "sgd": { + "type": ["number", "null"] + }, + "thb": { + "type": ["number", "null"] + }, + "try": { + "type": ["number", "null"] + }, + "twd": { + "type": ["number", "null"] + }, + "uah": { + "type": ["number", "null"] + }, + "usd": { + "type": ["number", "null"] + }, + "vef": { + "type": ["number", "null"] + }, + "vnd": { + "type": ["number", "null"] + }, + "xag": { + "type": ["number", "null"] + }, + "xau": { + "type": ["number", "null"] + }, + "xdr": { + "type": ["number", "null"] + }, + "xlm": { + "type": ["number", "null"] + }, + "xrp": { + "type": ["number", "null"] + }, + "yfi": { + "type": ["number", "null"] + }, + "zar": { + "type": ["number", "null"] + }, + "bits": { + "type": ["number", "null"] + }, + "link": { + "type": ["number", "null"] + }, + "sats": { + "type": ["number", "null"] + } + } + }, + "market_cap": { + "type": "object", + "properties": { + "aed": { + "type": ["number", "null"] + }, + "ars": { + "type": ["number", "null"] + }, + "aud": { + "type": ["number", "null"] + }, + "bch": { + "type": ["number", "null"] + }, + "bdt": { + "type": ["number", "null"] + }, + "bhd": { + "type": ["number", "null"] + }, + "bmd": { + "type": ["number", "null"] + }, + "bnb": { + "type": ["number", "null"] + }, + "brl": { + "type": ["number", "null"] + }, + "btc": { + "type": ["number", "null"] + }, + "cad": { + "type": ["number", "null"] + }, + "chf": { + "type": ["number", "null"] + }, + "clp": { + "type": ["number", "null"] + }, + "cny": { + "type": ["number", "null"] + }, + "czk": { + "type": ["number", "null"] + }, + "dkk": { + "type": ["number", "null"] + }, + "dot": { + "type": ["number", "null"] + }, + "eos": { + "type": ["number", "null"] + }, + "eth": { + "type": ["number", "null"] + }, + "eur": { + "type": ["number", "null"] + }, + "gbp": { + "type": ["number", "null"] + }, + "hkd": { + "type": ["number", "null"] + }, + "huf": { + "type": ["number", "null"] + }, + "idr": { + "type": ["number", "null"] + }, + "ils": { + "type": ["number", "null"] + }, + "inr": { + "type": ["number", "null"] + }, + "jpy": { + "type": ["number", "null"] + }, + "krw": { + "type": ["number", "null"] + }, + "kwd": { + "type": ["number", "null"] + }, + "lkr": { + "type": ["number", "null"] + }, + "ltc": { + "type": ["number", "null"] + }, + "mmk": { + "type": ["number", "null"] + }, + "mxn": { + "type": ["number", "null"] + }, + "myr": { + "type": ["number", "null"] + }, + "ngn": { + "type": ["number", "null"] + }, + "nok": { + "type": ["number", "null"] + }, + "nzd": { + "type": ["number", "null"] + }, + "php": { + "type": ["number", "null"] + }, + "pkr": { + "type": ["number", "null"] + }, + "pln": { + "type": ["number", "null"] + }, + "rub": { + "type": ["number", "null"] + }, + "sar": { + "type": ["number", "null"] + }, + "sek": { + "type": ["number", "null"] + }, + "sgd": { + "type": ["number", "null"] + }, + "thb": { + "type": ["number", "null"] + }, + "try": { + "type": ["number", "null"] + }, + "twd": { + "type": ["number", "null"] + }, + "uah": { + "type": ["number", "null"] + }, + "usd": { + "type": ["number", "null"] + }, + "vef": { + "type": ["number", "null"] + }, + "vnd": { + "type": ["number", "null"] + }, + "xag": { + "type": ["number", "null"] + }, + "xau": { + "type": ["number", "null"] + }, + "xdr": { + "type": ["number", "null"] + }, + "xlm": { + "type": ["number", "null"] + }, + "xrp": { + "type": ["number", "null"] + }, + "yfi": { + "type": ["number", "null"] + }, + "zar": { + "type": ["number", "null"] + }, + "bits": { + "type": ["number", "null"] + }, + "link": { + "type": ["number", "null"] + }, + "sats": { + "type": ["number", "null"] + } + } + }, + "total_volume": { + "type": "object", + "properties": { + "aed": { + "type": ["number", "null"] + }, + "ars": { + "type": ["number", "null"] + }, + "aud": { + "type": ["number", "null"] + }, + "bch": { + "type": ["number", "null"] + }, + "bdt": { + "type": ["number", "null"] + }, + "bhd": { + "type": ["number", "null"] + }, + "bmd": { + "type": ["number", "null"] + }, + "bnb": { + "type": ["number", "null"] + }, + "brl": { + "type": ["number", "null"] + }, + "btc": { + "type": ["number", "null"] + }, + "cad": { + "type": ["number", "null"] + }, + "chf": { + "type": ["number", "null"] + }, + "clp": { + "type": ["number", "null"] + }, + "cny": { + "type": ["number", "null"] + }, + "czk": { + "type": ["number", "null"] + }, + "dkk": { + "type": ["number", "null"] + }, + "dot": { + "type": ["number", "null"] + }, + "eos": { + "type": ["number", "null"] + }, + "eth": { + "type": ["number", "null"] + }, + "eur": { + "type": ["number", "null"] + }, + "gbp": { + "type": ["number", "null"] + }, + "hkd": { + "type": ["number", "null"] + }, + "huf": { + "type": ["number", "null"] + }, + "idr": { + "type": ["number", "null"] + }, + "ils": { + "type": ["number", "null"] + }, + "inr": { + "type": ["number", "null"] + }, + "jpy": { + "type": ["number", "null"] + }, + "krw": { + "type": ["number", "null"] + }, + "kwd": { + "type": ["number", "null"] + }, + "lkr": { + "type": ["number", "null"] + }, + "ltc": { + "type": ["number", "null"] + }, + "mmk": { + "type": ["number", "null"] + }, + "mxn": { + "type": ["number", "null"] + }, + "myr": { + "type": ["number", "null"] + }, + "ngn": { + "type": ["number", "null"] + }, + "nok": { + "type": ["number", "null"] + }, + "nzd": { + "type": ["number", "null"] + }, + "php": { + "type": ["number", "null"] + }, + "pkr": { + "type": ["number", "null"] + }, + "pln": { + "type": ["number", "null"] + }, + "rub": { + "type": ["number", "null"] + }, + "sar": { + "type": ["number", "null"] + }, + "sek": { + "type": ["number", "null"] + }, + "sgd": { + "type": ["number", "null"] + }, + "thb": { + "type": ["number", "null"] + }, + "try": { + "type": ["number", "null"] + }, + "twd": { + "type": ["number", "null"] + }, + "uah": { + "type": ["number", "null"] + }, + "usd": { + "type": ["number", "null"] + }, + "vef": { + "type": ["number", "null"] + }, + "vnd": { + "type": ["number", "null"] + }, + "xag": { + "type": ["number", "null"] + }, + "xau": { + "type": ["number", "null"] + }, + "xdr": { + "type": ["number", "null"] + }, + "xlm": { + "type": ["number", "null"] + }, + "xrp": { + "type": ["number", "null"] + }, + "yfi": { + "type": ["number", "null"] + }, + "zar": { + "type": ["number", "null"] + }, + "bits": { + "type": ["number", "null"] + }, + "link": { + "type": ["number", "null"] + }, + "sats": { + "type": ["number", "null"] + } + } + } + } + }, + "community_data": { + "type": "object", + "properties": { + "facebook_likes": {}, + "twitter_followers": {}, + "reddit_average_posts_48h": { + "type": ["number", "null"] + }, + "reddit_average_comments_48h": { + "type": ["number", "null"] + }, + "reddit_subscribers": { + "type": ["number", "null"] + }, + "reddit_accounts_active_48h": { + "type": ["string", "null"] + } + } + }, + "developer_data": { + "type": "object", + "properties": { + "forks": { + "type": ["number", "null"] + }, + "stars": { + "type": ["number", "null"] + }, + "subscribers": { + "type": ["number", "null"] + }, + "total_issues": { + "type": ["number", "null"] + }, + "closed_issues": { + "type": ["number", "null"] + }, + "pull_requests_merged": { + "type": ["number", "null"] + }, + "pull_request_contributors": { + "type": ["number", "null"] + }, + "code_additions_deletions_4_weeks": { + "type": "object", + "properties": { + "additions": { + "type": ["number", "null"] + }, + "deletions": { + "type": ["number", "null"] + } + } + }, + "commit_count_4_weeks": { + "type": ["number", "null"] + } + } + }, + "public_interest_stats": { + "type": "object", + "properties": { + "alexa_rank": {}, + "bing_matches": {} + } + } + } +} diff --git a/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/schemas/market_chart.json b/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/schemas/market_chart.json new file mode 100644 index 0000000000000..b29d24456bce1 --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/schemas/market_chart.json @@ -0,0 +1,33 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "prices": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + }, + "market_caps": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + }, + "total_volumes": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/source.py b/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/source.py new file mode 100644 index 0000000000000..9ed8e8ca9909b --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceCoingeckoCoins(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "coingecko_coins.yaml"}) diff --git a/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/spec.yaml b/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/spec.yaml new file mode 100644 index 0000000000000..a3e02acb41f37 --- /dev/null +++ b/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/spec.yaml @@ -0,0 +1,57 @@ +documentationUrl: https://docsurl.com +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: CoinGecko Coins Spec + type: object + required: + - coin_id + - vs_currency + - days + - start_date + additionalProperties: true + properties: + api_key: + type: string + description: API Key (for pro users) + airbyte_secret: true + order: 0 + coin_id: + type: string + description: | + CoinGecko coin ID (e.g. bitcoin). Can be retrieved from the + `/coins/list` endpoint. + order: 1 + vs_currency: + type: string + description: | + The target currency of market data (e.g. usd, eur, jpy, etc.) + order: 2 + days: + type: string + description: | + The number of days of data for market chart. + enum: + - "1" + - "7" + - "14" + - "30" + - "90" + - "180" + - "365" + - "max" + default: "30" + order: 3 + start_date: + type: string + description: | + The start date for the historical data stream in dd-mm-yyyy format. + format: date + pattern: "^[0-9]{2}-[0-9]{2}-[0-9]{4}$" + order: 4 + end_date: + type: string + description: | + The end date for the historical data stream in dd-mm-yyyy format. + format: date + pattern: "^[0-9]{2}-[0-9]{2}-[0-9]{4}$" + order: 5 diff --git a/docs/integrations/sources/coingecko-coins.md b/docs/integrations/sources/coingecko-coins.md new file mode 100644 index 0000000000000..d059e39296f2e --- /dev/null +++ b/docs/integrations/sources/coingecko-coins.md @@ -0,0 +1,54 @@ +# CoinGecko Coins + +## Sync overview + +This source can sync market chart and historical data for a single coin listed on the +[CoinGecko](https://coingecko.com) API. It currently supports only Full Refresh syncing. + +### Output schema + +This source is capable of syncing the following streams: + +* `market_chart` +* `history` + +### Features + +| Feature | Supported? \(Yes/No\) | Notes | +|:------------------|:----------------------|:-------------------------------------------------------| +| Full Refresh Sync | Yes | | +| Incremental Sync | No | | +| CoinGecko Pro API | Yes | Will default to free API unless an API key is provided | + +### Performance considerations + +The CoinGecko API has a rate limit of 10-50 requests per minute. The connector should not run into this +under normal operation. + +CoinGecko also request that free users provide attribution when using CoinGecko data. Please read more about +this [here](https://www.coingecko.com/en/branding). + +## Getting started + + +### Requirements + +1. Choose a coin to pull data from. The coin must be listed on CoinGecko, and can be listed via the `/coins/list` endpoint. +2. Choose a `vs_currency` to pull data in. This can be any currency listed on CoinGecko, and can be listed via the `/simple/supported_vs_currencies` endpoint. + +### Setup guide + +The following fields are required fields for the connector to work: + +- `coin_id`: The ID of the coin to pull data for. This can be found via the `/coins/list` endpoint. +- `vs_currency`: The currency to pull data for. This can be found via the `/simple/supported_vs_currencies` endpoint. +- `days`: The number of days to pull `market_chart` data for. +- `start_date`: The start date to pull `history` data from. +- (optional) `end_date`: The end date to pull `history` data until. + +## Changelog + +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:-----------| +| 0.1.0 | 2022-10-20 | [18248](https://github.com/airbytehq/airbyte/pull/18248) | New source | +