From 79ce6d597edb4183941c5427e7c7b03601f1305d Mon Sep 17 00:00:00 2001 From: Aviraj Gour Date: Thu, 5 Oct 2023 00:37:59 +0530 Subject: [PATCH 1/2] migrated to low code --- .../connectors/source-qualaroo/.dockerignore | 1 - .../connectors/source-qualaroo/Dockerfile | 11 +- .../connectors/source-qualaroo/README.md | 65 ++------- .../connectors/source-qualaroo/__init__.py | 3 + .../acceptance-test-config.yml | 31 +++-- .../source-qualaroo/acceptance-test-docker.sh | 1 + .../connectors/source-qualaroo/build.gradle | 8 ++ .../connectors/source-qualaroo/icon.svg | 1 - .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 4 +- .../integration_tests/acceptance.py | 3 + .../integration_tests/catalog.json | 24 ---- .../integration_tests/configured_catalog.json | 6 +- .../integration_tests/sample_config.json | 5 + .../integration_tests/sample_state.json | 5 + .../connectors/source-qualaroo/metadata.yaml | 25 ++-- .../source-qualaroo/requirements.txt | 1 + .../connectors/source-qualaroo/setup.py | 11 +- .../source_qualaroo/__init__.py | 22 +--- .../source_qualaroo/components.py | 34 +++++ .../source_qualaroo/manifest.yaml | 123 ++++++++++++++++++ .../source-qualaroo/source_qualaroo/source.py | 83 ++---------- .../source-qualaroo/source_qualaroo/spec.json | 40 ------ .../source_qualaroo/streams.py | 102 --------------- .../source-qualaroo/unit_tests/__init__.py | 23 ---- .../source-qualaroo/unit_tests/conftest.py | 10 -- .../source-qualaroo/unit_tests/helpers.py | 20 --- .../unit_tests/test_components.py | 32 +++++ .../unit_tests/test_streams.py | 102 --------------- 29 files changed, 287 insertions(+), 512 deletions(-) create mode 100644 airbyte-integrations/connectors/source-qualaroo/__init__.py mode change 100644 => 100755 airbyte-integrations/connectors/source-qualaroo/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-qualaroo/build.gradle delete mode 100644 airbyte-integrations/connectors/source-qualaroo/icon.svg delete mode 100644 airbyte-integrations/connectors/source-qualaroo/integration_tests/catalog.json create mode 100644 airbyte-integrations/connectors/source-qualaroo/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-qualaroo/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-qualaroo/source_qualaroo/components.py create mode 100644 airbyte-integrations/connectors/source-qualaroo/source_qualaroo/manifest.yaml delete mode 100644 airbyte-integrations/connectors/source-qualaroo/source_qualaroo/spec.json delete mode 100644 airbyte-integrations/connectors/source-qualaroo/source_qualaroo/streams.py delete mode 100644 airbyte-integrations/connectors/source-qualaroo/unit_tests/__init__.py delete mode 100644 airbyte-integrations/connectors/source-qualaroo/unit_tests/conftest.py delete mode 100644 airbyte-integrations/connectors/source-qualaroo/unit_tests/helpers.py create mode 100644 airbyte-integrations/connectors/source-qualaroo/unit_tests/test_components.py delete mode 100644 airbyte-integrations/connectors/source-qualaroo/unit_tests/test_streams.py diff --git a/airbyte-integrations/connectors/source-qualaroo/.dockerignore b/airbyte-integrations/connectors/source-qualaroo/.dockerignore index a224999d042b8..72a4047972a56 100644 --- a/airbyte-integrations/connectors/source-qualaroo/.dockerignore +++ b/airbyte-integrations/connectors/source-qualaroo/.dockerignore @@ -1,6 +1,5 @@ * !Dockerfile -!Dockerfile.test !main.py !source_qualaroo !setup.py diff --git a/airbyte-integrations/connectors/source-qualaroo/Dockerfile b/airbyte-integrations/connectors/source-qualaroo/Dockerfile index 94c4c48ff6de8..889137e3a673c 100644 --- a/airbyte-integrations/connectors/source-qualaroo/Dockerfile +++ b/airbyte-integrations/connectors/source-qualaroo/Dockerfile @@ -6,7 +6,9 @@ WORKDIR /airbyte/integration_code # upgrade pip to the latest version RUN apk --no-cache upgrade \ - && pip install --upgrade pip + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + COPY setup.py ./ # install necessary packages to a temporary folder @@ -18,8 +20,11 @@ WORKDIR /airbyte/integration_code # copy all loaded and built libraries to a pure basic image COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone -# Bash is installed for more convenient debugging. +# bash is installed for more convenient debugging. RUN apk --no-cache add bash # copy payload code only @@ -29,5 +34,5 @@ COPY source_qualaroo ./source_qualaroo ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.0 +LABEL io.airbyte.version=0.3.0 LABEL io.airbyte.name=airbyte/source-qualaroo diff --git a/airbyte-integrations/connectors/source-qualaroo/README.md b/airbyte-integrations/connectors/source-qualaroo/README.md index 0d7d94544593c..ecb93fce3c6c8 100644 --- a/airbyte-integrations/connectors/source-qualaroo/README.md +++ b/airbyte-integrations/connectors/source-qualaroo/README.md @@ -1,34 +1,10 @@ # Qualaroo Source -This is the repository for the Qualaroo source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/qualaroo). +This is the repository for the Qualaroo configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/qualaroo). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. @@ -38,22 +14,14 @@ To build using Gradle, from the Airbyte repository root, run: ``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/qualaroo) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_qualaroo/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/qualaroo) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_qualaroo/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source qualaroo test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image #### Build @@ -78,32 +46,15 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-qualaroo:dev discover docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-qualaroo:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` #### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run + +To run your integration tests with Docker, run: ``` -python -m pytest integration_tests -p integration_tests.acceptance +./acceptance-test-docker.sh ``` -To run your integration tests with docker ### Using gradle to run tests All commands should be run from airbyte project root. diff --git a/airbyte-integrations/connectors/source-qualaroo/__init__.py b/airbyte-integrations/connectors/source-qualaroo/__init__.py new file mode 100644 index 0000000000000..c941b30457953 --- /dev/null +++ b/airbyte-integrations/connectors/source-qualaroo/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-qualaroo/acceptance-test-config.yml b/airbyte-integrations/connectors/source-qualaroo/acceptance-test-config.yml index 3d77add761e5c..788f44eb71b42 100644 --- a/airbyte-integrations/connectors/source-qualaroo/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-qualaroo/acceptance-test-config.yml @@ -1,20 +1,27 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-qualaroo:dev -tests: +acceptance_tests: spec: - - spec_path: "source_qualaroo/spec.json" + tests: + - spec_path: "source_qualaroo/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + incremental: + bypass_reason: "This connector does not implement incremental sync" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-qualaroo/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-qualaroo/acceptance-test-docker.sh old mode 100644 new mode 100755 index 5797d20fe9a78..b6d65deeccb43 --- a/airbyte-integrations/connectors/source-qualaroo/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-qualaroo/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-qualaroo/build.gradle b/airbyte-integrations/connectors/source-qualaroo/build.gradle new file mode 100644 index 0000000000000..396638f891910 --- /dev/null +++ b/airbyte-integrations/connectors/source-qualaroo/build.gradle @@ -0,0 +1,8 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' +} + +airbytePython { + moduleDirectory 'source_qualaroo' +} diff --git a/airbyte-integrations/connectors/source-qualaroo/icon.svg b/airbyte-integrations/connectors/source-qualaroo/icon.svg deleted file mode 100644 index bc68344d292ec..0000000000000 --- a/airbyte-integrations/connectors/source-qualaroo/icon.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-qualaroo/integration_tests/__init__.py b/airbyte-integrations/connectors/source-qualaroo/integration_tests/__init__.py index e69de29bb2d1d..c941b30457953 100644 --- a/airbyte-integrations/connectors/source-qualaroo/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-qualaroo/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-qualaroo/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-qualaroo/integration_tests/abnormal_state.json index 4c3b2cd94d487..52b0f2c2118f4 100644 --- a/airbyte-integrations/connectors/source-qualaroo/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-qualaroo/integration_tests/abnormal_state.json @@ -1,5 +1,5 @@ { - "actions": { - "date": "3021-08-18T08:35:49.540Z" + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" } } diff --git a/airbyte-integrations/connectors/source-qualaroo/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-qualaroo/integration_tests/acceptance.py index d49b558823338..9e6409236281f 100644 --- a/airbyte-integrations/connectors/source-qualaroo/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-qualaroo/integration_tests/acceptance.py @@ -10,4 +10,7 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-qualaroo/integration_tests/catalog.json b/airbyte-integrations/connectors/source-qualaroo/integration_tests/catalog.json deleted file mode 100644 index b5706b1ef05e8..0000000000000 --- a/airbyte-integrations/connectors/source-qualaroo/integration_tests/catalog.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "surveys", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "responses", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-qualaroo/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-qualaroo/integration_tests/configured_catalog.json index b5706b1ef05e8..98d2544aae833 100644 --- a/airbyte-integrations/connectors/source-qualaroo/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-qualaroo/integration_tests/configured_catalog.json @@ -4,8 +4,7 @@ "stream": { "name": "surveys", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -14,8 +13,7 @@ "stream": { "name": "responses", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" diff --git a/airbyte-integrations/connectors/source-qualaroo/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-qualaroo/integration_tests/sample_config.json new file mode 100644 index 0000000000000..cf86ff0cf8585 --- /dev/null +++ b/airbyte-integrations/connectors/source-qualaroo/integration_tests/sample_config.json @@ -0,0 +1,5 @@ +{ + "token": "sample_token", + "key": "sample_key", + "start_date": "3021-02-11T08:35:49.540Z" +} diff --git a/airbyte-integrations/connectors/source-qualaroo/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-qualaroo/integration_tests/sample_state.json new file mode 100644 index 0000000000000..3587e579822d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-qualaroo/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-qualaroo/metadata.yaml b/airbyte-integrations/connectors/source-qualaroo/metadata.yaml index 5e8ab3d7a8dc6..7364d8abe5c36 100644 --- a/airbyte-integrations/connectors/source-qualaroo/metadata.yaml +++ b/airbyte-integrations/connectors/source-qualaroo/metadata.yaml @@ -1,24 +1,25 @@ data: - ab_internal: - ql: 200 - sl: 100 + allowedHosts: + hosts: + - "*" # Please change to the hostname of the source. + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source - definitionId: b08e4776-d1de-4e80-ab5c-1e51dad934a2 - dockerImageTag: 0.2.0 + definitionId: eb655362-28a8-4311-8806-4fcc612734a7 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-qualaroo - documentationUrl: https://docs.airbyte.com/integrations/sources/qualaroo githubIssueLabel: source-qualaroo icon: qualaroo.svg license: MIT name: Qualaroo - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: "2021-08-18" releaseStage: alpha supportLevel: community + documentationUrl: https://docs.airbyte.com/integrations/sources/qualaroo tags: - - language:python + - language:lowcode metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-qualaroo/requirements.txt b/airbyte-integrations/connectors/source-qualaroo/requirements.txt index d6e1198b1ab1f..cf563bcab6851 100644 --- a/airbyte-integrations/connectors/source-qualaroo/requirements.txt +++ b/airbyte-integrations/connectors/source-qualaroo/requirements.txt @@ -1 +1,2 @@ -e . +-e ../../bases/connector-acceptance-test \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-qualaroo/setup.py b/airbyte-integrations/connectors/source-qualaroo/setup.py index b2929697645c0..9723bf3062f73 100644 --- a/airbyte-integrations/connectors/source-qualaroo/setup.py +++ b/airbyte-integrations/connectors/source-qualaroo/setup.py @@ -10,19 +10,20 @@ ] TEST_REQUIREMENTS = [ - "pytest~=6.1", + "requests-mock~=1.9.3", + "pytest~=6.2", "pytest-mock~=3.6.1", - "requests-mock", + "connector-acceptance-test", ] setup( name="source_qualaroo", description="Source implementation for Qualaroo.", - author="Daniel Diamond", - author_email="danieldiamond1@gmail.com", + author="Airbyte", + author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/__init__.py b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/__init__.py index b27f605765f09..99c5fb690f899 100644 --- a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/__init__.py +++ b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/__init__.py @@ -1,25 +1,5 @@ # -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/components.py b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/components.py new file mode 100644 index 0000000000000..f7c362950c145 --- /dev/null +++ b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/components.py @@ -0,0 +1,34 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import requests +from typing import Any, List, Mapping +from base64 import b64encode +from dataclasses import dataclass +from airbyte_cdk.sources.declarative.auth.token import BasicHttpAuthenticator +from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor + + +@dataclass +class CustomAuthenticator(BasicHttpAuthenticator): + @property + def token(self): + + key = str(self._username.eval(self.config)).encode("latin1") + token = self._password.eval(self.config).encode("latin1") + encoded_credentials = b64encode(b":".join((key, token))).strip() + token = "Basic " + encoded_credentials.decode("ascii") + return token + + +class CustomExtractor(RecordExtractor): + + def extract_records(self, response: requests.Response, **kwargs) -> List[Mapping[str, Any]]: + + extracted=[] + for rec in response.json(): + if "answered_questions" in rec: + rec["answered_questions"] = list(rec["answered_questions"].values()) + extracted.append(rec) + return extracted diff --git a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/manifest.yaml b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/manifest.yaml new file mode 100644 index 0000000000000..4e2cc51af2736 --- /dev/null +++ b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/manifest.yaml @@ -0,0 +1,123 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + record_filter: + condition: "{{ record['id'] in config['survey_ids'] if config['survey_ids'] else true}}" + requester: + type: HttpRequester + url_base: "https://api.qualaroo.com/api/v1/" + http_method: "GET" + authenticator: + class_name: source_qualaroo.components.CustomAuthenticator + username: "{{config['key']}}" + password: "{{config['token']}}" + request_parameters: + limit: "500" + start_date: "{{config['start_date']}}" + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "OffsetIncrement" + page_size: 500 + page_token_option: + type: "RequestOption" + field_name: "offset" + inject_into: "request_parameter" + requester: + $ref: "#/definitions/requester" + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + surveys_stream: + $ref: "#/definitions/base_stream" + name: "surveys" + primary_key: "id" + $parameters: + path: "nudges" + + responses_stream: + name: "responses" + primary_key: "id" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + path: "nudges/{{ stream_slice.parent_id }}/responses.json" + record_selector: + type: RecordSelector + extractor: + class_name: source_qualaroo.components.CustomExtractor + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/surveys_stream" + parent_key: "id" + partition_field: "parent_id" + +streams: + - "#/definitions/surveys_stream" + - "#/definitions/responses_stream" + +check: + type: CheckStream + stream_names: + - "surveys" + +spec: + type: Spec + documentationUrl: https://docs.airbyte.com/integrations/sources/qualaroo + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + title: Qualaroo Spec + type: object + required: + - token + - key + - start_date + additionalProperties: true + properties: + token: + type: string + title: API token + description: >- + A Qualaroo token. See the docs + for instructions on how to generate it. + airbyte_secret: true + key: + type: string + title: API key + description: >- + A Qualaroo token. See the docs + for instructions on how to generate it. + airbyte_secret: true + start_date: + type: string + title: Start Date + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}Z$ + description: >- + UTC date and time in the format 2017-01-25T00:00:00Z. Any data before + this date will not be replicated. + examples: + - '2021-03-01T00:00:00.000Z' + survey_ids: + type: array + items: + type: string + pattern: ^[0-9]{1,8}$ + title: Qualaroo survey IDs + description: >- + IDs of the surveys from which you'd like to replicate data. If left + empty, data from all surveys to which you have access will be + replicated. diff --git a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/source.py b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/source.py index 1ea0bb6514acb..d9312d834b373 100644 --- a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/source.py +++ b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/source.py @@ -2,80 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from base64 import b64encode -from typing import Any, List, Mapping, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import pendulum -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator +WARNING: Do not modify this file. +""" -from .streams import QualarooStream, Responses, Surveys - -class QualarooAuthenticator(HttpAuthenticator): - """ - Generate auth header for start making requests from API token and API key. - """ - - def __init__( - self, - key: str, - token: str, - auth_header: str = "Authorization", - key_header: str = "oauth_consumer_key", - token_header: str = "oauth_token", - ): - self._key = key - self._token = b64encode(b":".join((key.encode("latin1"), token.encode("latin1")))).strip().decode("ascii") - self.auth_header = auth_header - self.key_header = key_header - self.token_header = token_header - - def get_auth_header(self) -> Mapping[str, Any]: - return {self.auth_header: f"Basic {self._token}"} - - -class SourceQualaroo(AbstractSource): - """ - Source Qualaroo fetch date from web-based, Kanban-style, list-making application. - """ - - @staticmethod - def _get_authenticator(config: dict) -> QualarooAuthenticator: - key, token = config["key"], config["token"] - return QualarooAuthenticator(token=token, key=key) - - def check_connection(self, logger, config) -> Tuple[bool, any]: - """ - Testing connection availability for the connector by granting the credentials. - """ - - try: - url = f"{QualarooStream.url_base}nudges" - - authenticator = self._get_authenticator(config) - - response = requests.get(url, headers=authenticator.get_auth_header()) - - response.raise_for_status() - available_surveys = {row.get("id") for row in response.json()} - for survey_id in config.get("survey_ids", []): - if survey_id not in available_surveys: - return False, f"survey_id {survey_id} not found" - return True, None - except requests.exceptions.RequestException as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - args = {} - # convert start_date to epoch time for qualaroo API - args["start_date"] = pendulum.parse(config["start_date"]).strftime("%s") - args["survey_ids"] = config.get("survey_ids", []) - args["authenticator"] = self._get_authenticator(config) - return [Surveys(**args), Responses(**args)] +# Declarative Source +class SourceQualaroo(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/spec.json b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/spec.json deleted file mode 100644 index 20c61166c40ce..0000000000000 --- a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/spec.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/qualaroo", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Qualaroo Spec", - "type": "object", - "required": ["token", "key", "start_date"], - "additionalProperties": true, - "properties": { - "token": { - "type": "string", - "title": "API token", - "description": "A Qualaroo token. See the docs for instructions on how to generate it.", - "airbyte_secret": true - }, - "key": { - "type": "string", - "title": "API key", - "description": "A Qualaroo token. See the docs for instructions on how to generate it.", - "airbyte_secret": true - }, - "start_date": { - "type": "string", - "title": "Start Date", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}Z$", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "examples": ["2021-03-01T00:00:00.000Z"] - }, - "survey_ids": { - "type": "array", - "items": { - "type": "string", - "pattern": "^[0-9]{1,8}$" - }, - "title": "Qualaroo survey IDs", - "description": "IDs of the surveys from which you'd like to replicate data. If left empty, data from all surveys to which you have access will be replicated." - } - } - } -} diff --git a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/streams.py b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/streams.py deleted file mode 100644 index 6bd5a0cfefb80..0000000000000 --- a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/streams.py +++ /dev/null @@ -1,102 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional - -import pendulum -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http import HttpStream - - -class QualarooStream(HttpStream, ABC): - url_base = "https://api.qualaroo.com/api/v1/" - - # Define primary key as sort key for full_refresh, or very first sync for incremental_refresh - primary_key = "id" - - # Page size - limit = 500 - - extra_params = None - - def __init__(self, start_date: pendulum.datetime, survey_ids: List[str] = [], **kwargs): - super().__init__(**kwargs) - self._start_date = start_date - self._survey_ids = survey_ids - self._offset = 0 - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - resp_json = response.json() - - if len(resp_json) == 500: - self._offset += 500 - return {"offset": self._offset} - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params = {"limit": self.limit, "start_date": self._start_date} - if next_page_token: - params.update(**next_page_token) - if self.extra_params: - params.update(self.extra_params) - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - json_response = response.json() - for record in json_response: - yield record - - -class ChildStreamMixin: - parent_stream_class: Optional[QualarooStream] = None - - def stream_slices(self, sync_mode, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - for item in self.parent_stream_class(config=self.config).read_records(sync_mode=sync_mode): - yield {"id": item["id"]} - - -class Surveys(QualarooStream): - """Return list of all Surveys. - API Docs: https://help.qualaroo.com/hc/en-us/articles/201969438-The-REST-Reporting-API - Endpoint: https://api.qualaroo.com/api/v1/nudges/ - """ - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return "nudges" - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - survey_ids = self._survey_ids - result = super().parse_response(response=response, **kwargs) - for record in result: - if not survey_ids or str(record["id"]) in survey_ids: - yield record - - -class Responses(ChildStreamMixin, QualarooStream): - """Return list of all responses of a survey. - API Docs: hhttps://help.qualaroo.com/hc/en-us/articles/201969438-The-REST-Reporting-API - Endpoint: https://api.qualaroo.com/api/v1/nudges//responses.json - """ - - parent_stream_class = Surveys - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - survey_id = stream_slice["survey_id"] - return f"nudges/{survey_id}/responses.json" - - def stream_slices(self, **kwargs): - survey_stream = Surveys(start_date=self._start_date, survey_ids=self._survey_ids, authenticator=self.authenticator) - for survey in survey_stream.read_records(sync_mode=SyncMode.full_refresh): - yield {"survey_id": survey["id"]} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - response_data = response.json() - # de-nest the answered_questions object if exists - for rec in response_data: - if "answered_questions" in rec: - rec["answered_questions"] = list(rec["answered_questions"].values()) - yield from response_data diff --git a/airbyte-integrations/connectors/source-qualaroo/unit_tests/__init__.py b/airbyte-integrations/connectors/source-qualaroo/unit_tests/__init__.py deleted file mode 100644 index 9db886e0930f0..0000000000000 --- a/airbyte-integrations/connectors/source-qualaroo/unit_tests/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# diff --git a/airbyte-integrations/connectors/source-qualaroo/unit_tests/conftest.py b/airbyte-integrations/connectors/source-qualaroo/unit_tests/conftest.py deleted file mode 100644 index 8d9c18dfc54d1..0000000000000 --- a/airbyte-integrations/connectors/source-qualaroo/unit_tests/conftest.py +++ /dev/null @@ -1,10 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from pytest import fixture - - -@fixture -def config(): - return {"start_date": "start_date", "authenticator": None} diff --git a/airbyte-integrations/connectors/source-qualaroo/unit_tests/helpers.py b/airbyte-integrations/connectors/source-qualaroo/unit_tests/helpers.py deleted file mode 100644 index b4e062d6522e6..0000000000000 --- a/airbyte-integrations/connectors/source-qualaroo/unit_tests/helpers.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -NO_SLEEP_HEADERS = { - "x-rate-limit-api-token-max": "1", - "x-rate-limit-api-token-remaining": "1", - "x-rate-limit-api-key-max": "1", - "x-rate-limit-api-key-remaining": "1", -} - - -def read_all_records(stream): - records = [] - slices = stream.stream_slices(sync_mode=None) - for slice in slices: - for record in stream.read_records(sync_mode=None, stream_slice=slice): - records.append(record) - return records diff --git a/airbyte-integrations/connectors/source-qualaroo/unit_tests/test_components.py b/airbyte-integrations/connectors/source-qualaroo/unit_tests/test_components.py new file mode 100644 index 0000000000000..45c352381df1f --- /dev/null +++ b/airbyte-integrations/connectors/source-qualaroo/unit_tests/test_components.py @@ -0,0 +1,32 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from source_qualaroo.components import CustomAuthenticator, CustomExtractor +import requests +import json + + +def test_token_generation(): + + config = {"key": "4524324", "token": "token"} + authenticator = CustomAuthenticator(config=config, username="example@gmail.com", password="api_key", parameters=None) + token = authenticator.token + expected_token = "Basic ZXhhbXBsZUBnbWFpbC5jb206YXBpX2tleQ==" + assert expected_token == token + +def test_extract_records_with_answered_questions(): + + response_data = [ + {"id": 1, "answered_questions": {"q1": "A1", "q2": "A2"}}, + {"id": 2, "answered_questions": {"q3": "A3"}}, + ] + response = requests.Response() + response._content = json.dumps(response_data).encode('utf-8') + extracted_records = CustomExtractor().extract_records(response) + + assert (len(extracted_records), 2) + assert (extracted_records[0]["id"], 1) + assert (extracted_records[0]["answered_questions"], ["A1", "A2"]) + assert (extracted_records[1]["id"], 2) + assert (extracted_records[1]["answered_questions"], ["A3"]) diff --git a/airbyte-integrations/connectors/source-qualaroo/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-qualaroo/unit_tests/test_streams.py deleted file mode 100644 index 30e0dad7fdc77..0000000000000 --- a/airbyte-integrations/connectors/source-qualaroo/unit_tests/test_streams.py +++ /dev/null @@ -1,102 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from unittest.mock import MagicMock - -from pytest import fixture -from source_qualaroo.source import QualarooStream, Responses, Surveys - -from .helpers import NO_SLEEP_HEADERS, read_all_records - - -@fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(QualarooStream, "path", "v0/example_endpoint") - mocker.patch.object(QualarooStream, "primary_key", "test_primary_key") - mocker.patch.object(QualarooStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class, config): - stream = QualarooStream(**config) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": {"before": "id"}} - expected_params = {"limit": 500, "start_date": "start_date", "before": "id"} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class, config): - stream = QualarooStream(**config) - inputs = {"response": MagicMock()} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_surveys_stream(requests_mock): - mock_surveys_request = requests_mock.get( - "https://api.qualaroo.com/api/v1/nudges?limit=500&start_date=2021-02-11T08%3A35%3A49.540Z", - headers=NO_SLEEP_HEADERS, - json=[{"id": "b11111111111111111111111", "name": "survey_1"}, {"id": "b22222222222222222222222", "name": "survey_2"}], - ) - - args = {"authenticator": None, "start_date": "2021-02-11T08:35:49.540Z", "survey_ids": []} - stream1 = Surveys(**args) - records = read_all_records(stream1) - assert records == [{"id": "b11111111111111111111111", "name": "survey_1"}, {"id": "b22222222222222222222222", "name": "survey_2"}] - - args["survey_ids"] = ["b22222222222222222222222"] - stream2 = Surveys(**args) - records = read_all_records(stream2) - assert records == [{"id": "b22222222222222222222222", "name": "survey_2"}] - - args["survey_ids"] = ["not-found"] - stream3 = Surveys(**args) - records = read_all_records(stream3) - assert records == [] - - assert mock_surveys_request.call_count == 3 - - -def test_responses_stream(requests_mock): - mock_surveys_request = requests_mock.get( - "https://api.qualaroo.com/api/v1/nudges?limit=500&start_date=2021-02-11T08%3A35%3A49.540Z", - headers=NO_SLEEP_HEADERS, - json=[{"id": "b11111111111111111111111", "name": "survey_1"}, {"id": "b22222222222222222222222", "name": "survey_2"}], - ) - - mock_responses_request_1 = requests_mock.get( - "https://api.qualaroo.com/api/v1/nudges/b11111111111111111111111/responses.json", - headers=NO_SLEEP_HEADERS, - json=[{"id": "c11111111111111111111111", "name": "response_1"}, {"id": "c22222222222222222222222", "name": "response_2"}], - ) - - mock_responses_request_2 = requests_mock.get( - "https://api.qualaroo.com/api/v1/nudges/b22222222222222222222222/responses.json", - headers=NO_SLEEP_HEADERS, - json=[{"id": "c33333333333333333333333", "name": "response_3"}, {"id": "c44444444444444444444444", "name": "response_4"}], - ) - - args = {"authenticator": None, "start_date": "2021-02-11T08:35:49.540Z", "survey_ids": []} - stream1 = Responses(**args) - records = read_all_records(stream1) - assert records == [ - {"id": "c11111111111111111111111", "name": "response_1"}, - {"id": "c22222222222222222222222", "name": "response_2"}, - {"id": "c33333333333333333333333", "name": "response_3"}, - {"id": "c44444444444444444444444", "name": "response_4"}, - ] - - args["survey_ids"] = ["b22222222222222222222222"] - stream2 = Responses(**args) - records = read_all_records(stream2) - assert records == [{"id": "c33333333333333333333333", "name": "response_3"}, {"id": "c44444444444444444444444", "name": "response_4"}] - - args["survey_ids"] = ["not-found"] - stream3 = Responses(**args) - records = read_all_records(stream3) - assert records == [] - - assert mock_surveys_request.call_count == 3 - assert mock_responses_request_1.call_count == 1 - assert mock_responses_request_2.call_count == 2 From cbede4ecf14f30a2d017fb5429146829aadd6287 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Wed, 25 Oct 2023 19:21:16 -0300 Subject: [PATCH 2/2] fix tests --- .../acceptance-test-config.yml | 2 +- .../connectors/source-qualaroo/build.gradle | 8 ----- .../connectors/source-qualaroo/icon.svg | 6 ++++ .../connectors/source-qualaroo/setup.py | 3 +- .../source_qualaroo/components.py | 16 ++++----- .../source_qualaroo/manifest.yaml | 8 +++-- .../source_qualaroo/schemas/responses.json | 3 ++ .../unit_tests/test_components.py | 33 ++++++++++--------- docs/integrations/sources/qualaroo.md | 1 + 9 files changed, 43 insertions(+), 37 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-qualaroo/build.gradle create mode 100644 airbyte-integrations/connectors/source-qualaroo/icon.svg diff --git a/airbyte-integrations/connectors/source-qualaroo/acceptance-test-config.yml b/airbyte-integrations/connectors/source-qualaroo/acceptance-test-config.yml index 788f44eb71b42..6241c7d6d3c7d 100644 --- a/airbyte-integrations/connectors/source-qualaroo/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-qualaroo/acceptance-test-config.yml @@ -19,7 +19,7 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] - incremental: + incremental: bypass_reason: "This connector does not implement incremental sync" full_refresh: tests: diff --git a/airbyte-integrations/connectors/source-qualaroo/build.gradle b/airbyte-integrations/connectors/source-qualaroo/build.gradle deleted file mode 100644 index 396638f891910..0000000000000 --- a/airbyte-integrations/connectors/source-qualaroo/build.gradle +++ /dev/null @@ -1,8 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' -} - -airbytePython { - moduleDirectory 'source_qualaroo' -} diff --git a/airbyte-integrations/connectors/source-qualaroo/icon.svg b/airbyte-integrations/connectors/source-qualaroo/icon.svg new file mode 100644 index 0000000000000..fa02bf3508484 --- /dev/null +++ b/airbyte-integrations/connectors/source-qualaroo/icon.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/airbyte-integrations/connectors/source-qualaroo/setup.py b/airbyte-integrations/connectors/source-qualaroo/setup.py index 9723bf3062f73..840126cac8fbe 100644 --- a/airbyte-integrations/connectors/source-qualaroo/setup.py +++ b/airbyte-integrations/connectors/source-qualaroo/setup.py @@ -6,14 +6,13 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/components.py b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/components.py index f7c362950c145..5e4e619d4d442 100644 --- a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/components.py +++ b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/components.py @@ -2,10 +2,11 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import requests -from typing import Any, List, Mapping from base64 import b64encode from dataclasses import dataclass +from typing import Any, List, Mapping + +import requests from airbyte_cdk.sources.declarative.auth.token import BasicHttpAuthenticator from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor @@ -23,12 +24,11 @@ def token(self): class CustomExtractor(RecordExtractor): - def extract_records(self, response: requests.Response, **kwargs) -> List[Mapping[str, Any]]: - extracted=[] - for rec in response.json(): - if "answered_questions" in rec: - rec["answered_questions"] = list(rec["answered_questions"].values()) - extracted.append(rec) + extracted = [] + for record in response.json(): + if "answered_questions" in record: + record["answered_questions"] = list(record["answered_questions"].values()) + extracted.append(record) return extracted diff --git a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/manifest.yaml b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/manifest.yaml index 4e2cc51af2736..863c36e344947 100644 --- a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/manifest.yaml +++ b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/manifest.yaml @@ -19,6 +19,10 @@ definitions: request_parameters: limit: "500" start_date: "{{config['start_date']}}" + error_handler: + response_filters: + - http_codes: [500] + action: FAIL retriever: type: SimpleRetriever record_selector: @@ -71,7 +75,7 @@ streams: check: type: CheckStream stream_names: - - "surveys" + - "responses" spec: type: Spec @@ -110,7 +114,7 @@ spec: UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. examples: - - '2021-03-01T00:00:00.000Z' + - "2021-03-01T00:00:00.000Z" survey_ids: type: array items: diff --git a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/schemas/responses.json b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/schemas/responses.json index 5dfab07c0dd34..db74aa16cfa5b 100644 --- a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/schemas/responses.json +++ b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/schemas/responses.json @@ -15,6 +15,9 @@ "identity": { "type": ["null", "string"] }, + "location": { + "type": ["null", "string"] + }, "page": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-qualaroo/unit_tests/test_components.py b/airbyte-integrations/connectors/source-qualaroo/unit_tests/test_components.py index 45c352381df1f..bd5b6f4c32c79 100644 --- a/airbyte-integrations/connectors/source-qualaroo/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-qualaroo/unit_tests/test_components.py @@ -2,10 +2,11 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from source_qualaroo.components import CustomAuthenticator, CustomExtractor -import requests import json +import requests +from source_qualaroo.components import CustomAuthenticator, CustomExtractor + def test_token_generation(): @@ -14,19 +15,19 @@ def test_token_generation(): token = authenticator.token expected_token = "Basic ZXhhbXBsZUBnbWFpbC5jb206YXBpX2tleQ==" assert expected_token == token - + + def test_extract_records_with_answered_questions(): - response_data = [ - {"id": 1, "answered_questions": {"q1": "A1", "q2": "A2"}}, - {"id": 2, "answered_questions": {"q3": "A3"}}, - ] - response = requests.Response() - response._content = json.dumps(response_data).encode('utf-8') - extracted_records = CustomExtractor().extract_records(response) - - assert (len(extracted_records), 2) - assert (extracted_records[0]["id"], 1) - assert (extracted_records[0]["answered_questions"], ["A1", "A2"]) - assert (extracted_records[1]["id"], 2) - assert (extracted_records[1]["answered_questions"], ["A3"]) + response_data = [ + {"id": 1, "answered_questions": {"q1": "A1", "q2": "A2"}}, + {"id": 2, "answered_questions": {"q3": "A3"}}, + ] + response = requests.Response() + response._content = json.dumps(response_data).encode("utf-8") + extracted_records = CustomExtractor().extract_records(response) + expected_records = [ + {"id": 1, "answered_questions": ["A1", "A2"]}, + {"id": 2, "answered_questions": ["A3"]} + ] + assert expected_records == extracted_records diff --git a/docs/integrations/sources/qualaroo.md b/docs/integrations/sources/qualaroo.md index d17d032665704..b6ae702205df8 100644 --- a/docs/integrations/sources/qualaroo.md +++ b/docs/integrations/sources/qualaroo.md @@ -41,6 +41,7 @@ Please read [How to get your APIs Token and Key](https://help.qualaroo.com/hc/en | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------| +| 0.3.0 | 2023-10-25 | [31070](https://github.com/airbytehq/airbyte/pull/31070) | Migrate to low-code framework | | 0.2.0 | 2023-05-24 | [26491](https://github.com/airbytehq/airbyte/pull/26491) | Remove authSpecification from spec.json as OAuth is not supported by Qualaroo + update stream schema | | 0.1.2 | 2022-05-24 | [13121](https://github.com/airbytehq/airbyte/pull/13121) | Fix `start_date` and `survey_ids` schema formatting. Separate source and stream files. Add stream_slices | | 0.1.1 | 2022-05-20 | [13042](https://github.com/airbytehq/airbyte/pull/13042) | Update stream specs |