From c617d64900d389bf50d954816b4dae088e0f33c6 Mon Sep 17 00:00:00 2001 From: Jared Rhizor Date: Fri, 21 Oct 2022 05:25:40 -0700 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20New=20Source:=20Whisky=20Hunter?= =?UTF-8?q?=20API=20[low-code=20CDK]=20(#17918)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * initial commit for source-whisky-hunter * better logs * finish up connector * fix catalog * fix bug in connector_runner.py that doesn't allow falsey configs * add bootstrap.md * add docs * fix typo * fix source-accept test * auto-bump connector version Co-authored-by: marcosmarxm Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 7 + .../src/main/resources/seed/source_specs.yaml | 12 ++ .../source_acceptance_test/tests/test_core.py | 5 +- .../utils/connector_runner.py | 5 +- .../unit_tests/test_core.py | 2 +- airbyte-integrations/builds.md | 1 + .../source-whisky-hunter/.dockerignore | 6 + .../source-whisky-hunter/Dockerfile | 38 +++++ .../connectors/source-whisky-hunter/README.md | 79 +++++++++++ .../source-whisky-hunter/__init__.py | 3 + .../acceptance-test-config.yml | 18 +++ .../acceptance-test-docker.sh | 16 +++ .../source-whisky-hunter/bootstrap.md | 20 +++ .../source-whisky-hunter/build.gradle | 9 ++ .../integration_tests/__init__.py | 3 + .../integration_tests/acceptance.py | 14 ++ .../integration_tests/catalog.json | 112 +++++++++++++++ .../integration_tests/configured_catalog.json | 130 ++++++++++++++++++ .../integration_tests/sample_config.json | 1 + .../integration_tests/sample_state.json | 1 + .../connectors/source-whisky-hunter/main.py | 13 ++ .../source-whisky-hunter/requirements.txt | 2 + .../connectors/source-whisky-hunter/setup.py | 29 ++++ .../source_whisky_hunter/__init__.py | 8 ++ .../schemas/auctions_data.json | 33 +++++ .../schemas/auctions_info.json | 30 ++++ .../schemas/distilleries_info.json | 24 ++++ .../source_whisky_hunter/source.py | 18 +++ .../source_whisky_hunter/spec.yaml | 7 + .../source_whisky_hunter/whisky_hunter.yaml | 46 +++++++ docs/integrations/README.md | 1 + docs/integrations/sources/whisky-hunter.md | 37 +++++ 32 files changed, 726 insertions(+), 4 deletions(-) create mode 100644 airbyte-integrations/connectors/source-whisky-hunter/.dockerignore create mode 100644 airbyte-integrations/connectors/source-whisky-hunter/Dockerfile create mode 100644 airbyte-integrations/connectors/source-whisky-hunter/README.md create mode 100644 airbyte-integrations/connectors/source-whisky-hunter/__init__.py create mode 100644 airbyte-integrations/connectors/source-whisky-hunter/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-whisky-hunter/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-whisky-hunter/bootstrap.md create mode 100644 airbyte-integrations/connectors/source-whisky-hunter/build.gradle create mode 100644 airbyte-integrations/connectors/source-whisky-hunter/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-whisky-hunter/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-whisky-hunter/integration_tests/catalog.json create mode 100644 airbyte-integrations/connectors/source-whisky-hunter/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-whisky-hunter/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-whisky-hunter/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-whisky-hunter/main.py create mode 100644 airbyte-integrations/connectors/source-whisky-hunter/requirements.txt create mode 100644 airbyte-integrations/connectors/source-whisky-hunter/setup.py create mode 100644 airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/__init__.py create mode 100644 airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/schemas/auctions_data.json create mode 100644 airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/schemas/auctions_info.json create mode 100644 airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/schemas/distilleries_info.json create mode 100644 airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/source.py create mode 100644 airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/spec.yaml create mode 100644 airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/whisky_hunter.yaml create mode 100644 docs/integrations/sources/whisky-hunter.md diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index eb41f515d4b003..75aadbd817d002 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1156,6 +1156,13 @@ icon: webflow.svg sourceType: api releaseStage: alpha +- name: Whisky Hunter + sourceDefinitionId: e65f84c0-7598-458a-bfac-f770c381ff5d + dockerRepository: airbyte/source-whisky-hunter + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/whisky-hunter + sourceType: api + releaseStage: alpha - name: WooCommerce sourceDefinitionId: 2a2552ca-9f78-4c1c-9eb7-4d0dc66d72df dockerRepository: airbyte/source-woocommerce diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index a1d450de0f6a31..71503972f7cc73 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -11844,6 +11844,18 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-whisky-hunter:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.io/integrations/sources/whisky-hunter" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Whisky Hunter Spec" + type: "object" + additionalProperties: true + properties: {} + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-woocommerce:0.1.2" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/woocommerce" diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py index d53594855323a7..3cdfc2987d7f33 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py @@ -360,7 +360,10 @@ def _validate_records_structure(records: List[AirbyteRecordMessage], configured_ continue record_fields = set(get_object_structure(record.data)) common_fields = set.intersection(record_fields, schema_pathes) - assert common_fields, f" Record from {record.stream} stream should have some fields mentioned by json schema, {schema_pathes}" + + assert ( + common_fields + ), f" Record {record} from {record.stream} stream with fields {record_fields} should have some fields mentioned by json schema: {schema_pathes}" @staticmethod def _validate_schema(records: List[AirbyteRecordMessage], configured_catalog: ConfiguredAirbyteCatalog): diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/connector_runner.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/connector_runner.py index 533eba42eeb4b6..0d7aa48045a3fa 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/connector_runner.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/connector_runner.py @@ -39,7 +39,8 @@ def _prepare_volumes(self, config: Optional[Mapping], state: Optional[Mapping], self.input_folder.mkdir(parents=True) self.output_folder.mkdir(parents=True) - if config: + # using "is not None" to allow falsey config objects like {} to still write + if config is not None: with open(str(self.input_folder / "tap_config.json"), "w") as outfile: json.dump(dict(config), outfile) @@ -149,7 +150,7 @@ def read(cls, container: Container, command: str = None, with_ext: bool = True) raise if exit_status["StatusCode"]: error = exit_status["Error"] or exception or line - logging.error(f"Docker container was failed, " f'code {exit_status["StatusCode"]}, error:\n{error}') + logging.error(f"Docker container failed, " f'code {exit_status["StatusCode"]}, error:\n{error}') if with_ext: raise ContainerError( container=container, diff --git a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_core.py b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_core.py index 6db6125eb4c86c..eec175467915bd 100644 --- a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_core.py +++ b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_core.py @@ -268,7 +268,7 @@ def test_read(schema, record, should_fail): ] t = _TestBasicRead() if should_fail: - with pytest.raises(AssertionError, match="stream should have some fields mentioned by json schema"): + with pytest.raises(AssertionError, match="should have some fields mentioned by json schema"): t.test_read(None, catalog, input_config, [], docker_runner_mock, MagicMock()) else: t.test_read(None, catalog, input_config, [], docker_runner_mock, MagicMock()) diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 0444a363c60b97..1385df6ad8fb34 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -118,6 +118,7 @@ | Twilio | [![source-twilio](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-twilio%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-twilio) | | Typeform | [![source-typeform](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-typeform%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-typeform) | | US Census | [![source-us-census](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-us-census%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-us-census) | +| Whisky Hunter | [![source-whisky-hunter](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-whisky-hunter%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-whisky-hunter) | | Wrike | [![source-wrike](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-wrike%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-wrike) | | YouTube Analytics | [![source-youtube-analytics](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-youtube-analytics%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-youtube-analytics) | | Zendesk Chat | [![source-zendesk-chat](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-zendesk-chat%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-zendesk-chat) | diff --git a/airbyte-integrations/connectors/source-whisky-hunter/.dockerignore b/airbyte-integrations/connectors/source-whisky-hunter/.dockerignore new file mode 100644 index 00000000000000..26206a22c8cca9 --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_whisky_hunter +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-whisky-hunter/Dockerfile b/airbyte-integrations/connectors/source-whisky-hunter/Dockerfile new file mode 100644 index 00000000000000..51864abfbf859b --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_whisky_hunter ./source_whisky_hunter + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-whisky-hunter diff --git a/airbyte-integrations/connectors/source-whisky-hunter/README.md b/airbyte-integrations/connectors/source-whisky-hunter/README.md new file mode 100644 index 00000000000000..232fe8b3abe8ae --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/README.md @@ -0,0 +1,79 @@ +# Whisky Hunter Source + +This is the repository for the Whisky Hunter configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/whisky-hunter). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-whisky-hunter:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/whisky-hunter) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_whisky_hunter/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source whisky-hunter test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-whisky-hunter:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-whisky-hunter:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-whisky-hunter:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-whisky-hunter:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-whisky-hunter:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-whisky-hunter:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-whisky-hunter:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-whisky-hunter:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-whisky-hunter/__init__.py b/airbyte-integrations/connectors/source-whisky-hunter/__init__.py new file mode 100644 index 00000000000000..1100c1c58cf510 --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-whisky-hunter/acceptance-test-config.yml b/airbyte-integrations/connectors/source-whisky-hunter/acceptance-test-config.yml new file mode 100644 index 00000000000000..e4537545ec4769 --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/acceptance-test-config.yml @@ -0,0 +1,18 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-whisky-hunter:dev +tests: + spec: + - spec_path: "source_whisky_hunter/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-whisky-hunter/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-whisky-hunter/acceptance-test-docker.sh new file mode 100644 index 00000000000000..c51577d10690c1 --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-whisky-hunter/bootstrap.md b/airbyte-integrations/connectors/source-whisky-hunter/bootstrap.md new file mode 100644 index 00000000000000..472c1c2709ef94 --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/bootstrap.md @@ -0,0 +1,20 @@ +## Core streams + +[Whisky Hunter](https://whiskyhunter.net/api/) is an API. Connector is implemented with the [Airbyte Low-Code CDK](https://docs.airbyte.com/connector-development/config-based/low-code-cdk-overview). + +Connector supports the following three streams: +* `auctions_data` + * Provides stats about specific auctions. +* `auctions_info` + * Provides information and metadata about recurring and one-off auctions. +* `distilleries_info` + * Provides information about distilleries. + +Rate Limiting: +* No published rate limit. + +Authentication and Permissions: +* No authentication. + + +See [this](https://docs.airbyte.io/integrations/sources/whisky-hunter) link for the connector docs. diff --git a/airbyte-integrations/connectors/source-whisky-hunter/build.gradle b/airbyte-integrations/connectors/source-whisky-hunter/build.gradle new file mode 100644 index 00000000000000..6fb1398539bf5c --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_whisky_hunter' +} diff --git a/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/__init__.py b/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/__init__.py new file mode 100644 index 00000000000000..1100c1c58cf510 --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/acceptance.py new file mode 100644 index 00000000000000..950b53b59d4166 --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/catalog.json b/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/catalog.json new file mode 100644 index 00000000000000..b5acb2a9ab7695 --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/catalog.json @@ -0,0 +1,112 @@ +{ + "type": "CATALOG", + "catalog": { + "streams": [ + { + "name": "auctions_data", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "dt": { + "type": "string" + }, + "all_auctions_lots_count": { + "type": "integer" + }, + "auction_slug": { + "type": "string" + }, + "auction_lots_count": { + "type": "integer" + }, + "auction_name": { + "type": "string" + }, + "winning_bid_min": { + "type": "number" + }, + "auction_trading_volume": { + "type": "number" + }, + "winning_bid_max": { + "type": "number" + }, + "winning_bid_mean": { + "type": "number" + } + } + }, + "supported_sync_modes": [ + "full_refresh" + ] + }, + { + "name": "auctions_info", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "base_currency": { + "type": "string" + }, + "buyers_fee": { + "type": "number" + }, + "reserve_fee": { + "type": "number" + }, + "listing_fee": { + "type": "number" + }, + "name": { + "type": "string" + }, + "sellers_fee": { + "type": "number" + }, + "slug": { + "type": "string" + }, + "url": { + "type": "string" + } + } + }, + "supported_sync_modes": [ + "full_refresh" + ] + }, + { + "name": "distilleries_info", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "whiskybase_votes": { + "type": "string" + }, + "country": { + "type": "string" + }, + "whiskybase_whiskies": { + "type": "string" + }, + "name": { + "type": "string" + }, + "slug": { + "type": "string" + }, + "whiskybase_rating": { + "type": "string" + } + } + }, + "supported_sync_modes": [ + "full_refresh" + ] + } + ] + } +} diff --git a/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/configured_catalog.json new file mode 100644 index 00000000000000..74fdfedae11cc9 --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/configured_catalog.json @@ -0,0 +1,130 @@ +{ + "streams": [ + { + "stream": { + "name": "auctions_data", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "array", + "items": { + "type": "object", + "properties": { + "dt": { + "type": "string" + }, + "all_auctions_lots_count": { + "type": "integer" + }, + "auction_slug": { + "type": "string" + }, + "auction_lots_count": { + "type": "integer" + }, + "auction_name": { + "type": "string" + }, + "winning_bid_min": { + "type": "number" + }, + "auction_trading_volume": { + "type": "number" + }, + "winning_bid_max": { + "type": "number" + }, + "winning_bid_mean": { + "type": "number" + } + } + } + }, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "auctions_info", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "array", + "items": { + "type": "object", + "properties": { + "base_currency": { + "type": "string" + }, + "buyers_fee": { + "type": "number" + }, + "reserve_fee": { + "type": "number" + }, + "listing_fee": { + "type": "number" + }, + "name": { + "type": "string" + }, + "sellers_fee": { + "type": "number" + }, + "slug": { + "type": "string" + }, + "url": { + "type": "string" + } + } + } + }, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "distilleries_info", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "array", + "items": { + "type": "object", + "properties": { + "whiskybase_votes": { + "type": "string" + }, + "country": { + "type": "string" + }, + "whiskybase_whiskies": { + "type": "string" + }, + "name": { + "type": "string" + }, + "slug": { + "type": "string" + }, + "whiskybase_rating": { + "type": "string" + } + } + } + }, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/sample_config.json new file mode 100644 index 00000000000000..0967ef424bce67 --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/sample_config.json @@ -0,0 +1 @@ +{} diff --git a/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/sample_state.json new file mode 100644 index 00000000000000..0967ef424bce67 --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/sample_state.json @@ -0,0 +1 @@ +{} diff --git a/airbyte-integrations/connectors/source-whisky-hunter/main.py b/airbyte-integrations/connectors/source-whisky-hunter/main.py new file mode 100644 index 00000000000000..1f7a4dba114a59 --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_whisky_hunter import SourceWhiskyHunter + +if __name__ == "__main__": + source = SourceWhiskyHunter() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-whisky-hunter/requirements.txt b/airbyte-integrations/connectors/source-whisky-hunter/requirements.txt new file mode 100644 index 00000000000000..0411042aa0911f --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-whisky-hunter/setup.py b/airbyte-integrations/connectors/source-whisky-hunter/setup.py new file mode 100644 index 00000000000000..fbeae664a051d4 --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.2", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_whisky_hunter", + description="Source implementation for Whisky Hunter.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/__init__.py b/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/__init__.py new file mode 100644 index 00000000000000..b6a52f86e1cfb1 --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceWhiskyHunter + +__all__ = ["SourceWhiskyHunter"] diff --git a/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/schemas/auctions_data.json b/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/schemas/auctions_data.json new file mode 100644 index 00000000000000..afd2e3be83018b --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/schemas/auctions_data.json @@ -0,0 +1,33 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "dt": { + "type": "string" + }, + "all_auctions_lots_count": { + "type": "integer" + }, + "auction_slug": { + "type": "string" + }, + "auction_lots_count": { + "type": "integer" + }, + "auction_name": { + "type": "string" + }, + "winning_bid_min": { + "type": "number" + }, + "auction_trading_volume": { + "type": "number" + }, + "winning_bid_max": { + "type": "number" + }, + "winning_bid_mean": { + "type": "number" + } + } +} diff --git a/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/schemas/auctions_info.json b/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/schemas/auctions_info.json new file mode 100644 index 00000000000000..8a8bd673b8e228 --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/schemas/auctions_info.json @@ -0,0 +1,30 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "base_currency": { + "type": "string" + }, + "buyers_fee": { + "type": "number" + }, + "reserve_fee": { + "type": "number" + }, + "listing_fee": { + "type": "number" + }, + "name": { + "type": "string" + }, + "sellers_fee": { + "type": "number" + }, + "slug": { + "type": "string" + }, + "url": { + "type": "string" + } + } +} diff --git a/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/schemas/distilleries_info.json b/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/schemas/distilleries_info.json new file mode 100644 index 00000000000000..d94228e12ce5d1 --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/schemas/distilleries_info.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "whiskybase_votes": { + "type": "string" + }, + "country": { + "type": "string" + }, + "whiskybase_whiskies": { + "type": "string" + }, + "name": { + "type": "string" + }, + "slug": { + "type": "string" + }, + "whiskybase_rating": { + "type": "string" + } + } +} diff --git a/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/source.py b/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/source.py new file mode 100644 index 00000000000000..c0d22e785848a5 --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceWhiskyHunter(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "whisky_hunter.yaml"}) diff --git a/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/spec.yaml b/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/spec.yaml new file mode 100644 index 00000000000000..37450e46c38a73 --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/spec.yaml @@ -0,0 +1,7 @@ +documentationUrl: https://docs.airbyte.io/integrations/sources/whisky-hunter +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Whisky Hunter Spec + type: object + additionalProperties: true + properties: {} diff --git a/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/whisky_hunter.yaml b/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/whisky_hunter.yaml new file mode 100644 index 00000000000000..26c9ee9a6060a5 --- /dev/null +++ b/airbyte-integrations/connectors/source-whisky-hunter/source_whisky_hunter/whisky_hunter.yaml @@ -0,0 +1,46 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + field_pointer: [] + requester: + url_base: "https://whiskyhunter.net/api" + http_method: "GET" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: NoPagination + requester: + $ref: "*ref(definitions.requester)" + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + auctions_data_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "auctions_data" + path: "/auctions_data/?format=json" + auctions_info_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "auctions_info" + path: "/auctions_info?format=json" + distilleries_info_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "distilleries_info" + path: "/distilleries_info/?format=json" + + +streams: + - "*ref(definitions.auctions_data_stream)" + - "*ref(definitions.auctions_info_stream)" + - "*ref(definitions.distilleries_info_stream)" + +check: + stream_names: + - "auctions_data" + - "auctions_info" + - "distilleries_info" diff --git a/docs/integrations/README.md b/docs/integrations/README.md index 95659449e4833e..f21e8a5b294603 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -170,6 +170,7 @@ For more information about the grading system, see [Product Release Stages](http | [US Census](sources/us-census.md) | Alpha | Yes | | [VictorOps](sources/victorops.md) | Alpha | No | | [Webflow](sources/webflow.md ) | Alpha | Yes | +| [Whisky Hunter](sources/whisky-hunter.md ) | Alpha | No | | [WooCommerce](sources/woocommerce.md) | Alpha | No | | [Wordpress](sources/wordpress.md) | Alpha | No | | [Wrike](sources/wrike.md) | Alpha | No | diff --git a/docs/integrations/sources/whisky-hunter.md b/docs/integrations/sources/whisky-hunter.md new file mode 100644 index 00000000000000..cc999a84156077 --- /dev/null +++ b/docs/integrations/sources/whisky-hunter.md @@ -0,0 +1,37 @@ +# Whisky Hunter + +## Overview + +The Whisky Hunter source can sync data from the [Whisky Hunter API](https://whiskyhunter.net/api/) + +#### Output schema + +This source is capable of syncing the following streams: +* `auctions_data` + * Provides stats about specific auctions. +* `auctions_info` + * Provides information and metadata about recurring and one-off auctions. +* `distilleries_info` + * Provides information about distilleries. + +#### Features + +| Feature | Supported? | +| :--- | :--- | +| Full Refresh Sync | Yes | +| Incremental - Append Sync | No | +| Namespaces | No | + +### Requirements / Setup Guide + +No config is required. + +## Performance considerations + +There is no published rate limit. However, since this data updates infrequently, it is recommended to set the update cadence to 24hr or higher. + +## Changelog + +| Version | Date | Pull Request | Subject | +| :--- | :--- | :--- | :--- | +| 0.1.0 | 2022-10-12 | [17918](https://github.com/airbytehq/airbyte/pull/17918) | Initial release supporting the Whisky Hunter API |