diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index ccf9908768cb9..9d9c5a77a0b99 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -196,6 +196,7 @@ jobs: SOURCE_LEMLIST_TEST_CREDS: ${{ secrets.SOURCE_LEMLIST_TEST_CREDS }} SOURCE_STRAVA_TEST_CREDS: ${{ secrets.SOURCE_STRAVA_TEST_CREDS }} SOURCE_PAYSTACK_TEST_CREDS: ${{ secrets.SOURCE_PAYSTACK_TEST_CREDS }} + SOURCE_DELIGHTED_TEST_CREDS: ${{ secrets.SOURCE_DELIGHTED_TEST_CREDS }} - run: | echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u airbytebot -p ${DOCKER_PASSWORD} ./tools/integrations/manage.sh publish airbyte-integrations/${{ github.event.inputs.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index f7f7a3d2e8bfe..bdf6685cdf3c6 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -191,6 +191,7 @@ jobs: SOURCE_LEMLIST_TEST_CREDS: ${{ secrets.SOURCE_LEMLIST_TEST_CREDS }} SOURCE_STRAVA_TEST_CREDS: ${{ secrets.SOURCE_STRAVA_TEST_CREDS }} SOURCE_PAYSTACK_TEST_CREDS: ${{ secrets.SOURCE_PAYSTACK_TEST_CREDS }} + SOURCE_DELIGHTED_TEST_CREDS: ${{ secrets.SOURCE_DELIGHTED_TEST_CREDS }} - run: | ./tools/bin/ci_integration_test.sh ${{ github.event.inputs.connector }} name: test ${{ github.event.inputs.connector }} diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/cc88c43f-6f53-4e8a-8c4d-b284baaf9635.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/cc88c43f-6f53-4e8a-8c4d-b284baaf9635.json new file mode 100644 index 0000000000000..b7d4dbf677283 --- /dev/null +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/cc88c43f-6f53-4e8a-8c4d-b284baaf9635.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "cc88c43f-6f53-4e8a-8c4d-b284baaf9635", + "name": "Delighted", + "dockerRepository": "airbyte/source-delighted", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://docs.airbyte.io/integrations/sources/delighted" +} diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index fe3c30162e2f7..89522d4246121 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -102,6 +102,12 @@ dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/sources/cockroachdb sourceType: database +- name: Delighted + sourceDefinitionId: cc88c43f-6f53-4e8a-8c4d-b284baaf9635 + dockerRepository: airbyte/source-delighted + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/delighted + sourceType: api - name: Dixa sourceDefinitionId: 0b5c867e-1b12-4d02-ab74-97b2184ff6d7 dockerRepository: airbyte/source-dixa diff --git a/airbyte-integrations/connectors/source-delighted/acceptance-test-config.yml b/airbyte-integrations/connectors/source-delighted/acceptance-test-config.yml index 556630ba8bfa1..e8df991ce5376 100644 --- a/airbyte-integrations/connectors/source-delighted/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-delighted/acceptance-test-config.yml @@ -8,12 +8,13 @@ tests: - config_path: "secrets/config.json" status: "succeed" - config_path: "integration_tests/invalid_config.json" - status: "exception" + status: "failed" discovery: - config_path: "secrets/config.json" basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: ["bounces"] incremental: # TODO if your connector does not implement incremental sync, remove this block - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-delighted/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-delighted/integration_tests/abnormal_state.json index 9fe1a3e8b6f51..08e8cdd7f6aa9 100644 --- a/airbyte-integrations/connectors/source-delighted/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-delighted/integration_tests/abnormal_state.json @@ -2,10 +2,10 @@ "people": { "created_at": "4126108288" }, - "unsubscribed_people": { + "unsubscribes": { "unsubscribed_at": "4126108288" }, - "bounced_people": { + "bounces": { "bounced_at": "4126108288" }, "survey_responses": { diff --git a/airbyte-integrations/connectors/source-delighted/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-delighted/integration_tests/acceptance.py index df2783d1750fc..58c194c5d1376 100644 --- a/airbyte-integrations/connectors/source-delighted/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-delighted/integration_tests/acceptance.py @@ -1,24 +1,6 @@ -# MIT License # -# Copyright (c) 2020 Airbyte +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. # -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. import pytest diff --git a/airbyte-integrations/connectors/source-delighted/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-delighted/integration_tests/configured_catalog.json index 92fc1a6438b96..85a9f5051bdeb 100644 --- a/airbyte-integrations/connectors/source-delighted/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-delighted/integration_tests/configured_catalog.json @@ -12,7 +12,7 @@ }, { "stream": { - "name": "unsubscribed_people", + "name": "unsubscribes", "json_schema": {}, "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] @@ -22,7 +22,7 @@ }, { "stream": { - "name": "bounced_people", + "name": "bounces", "json_schema": {}, "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] diff --git a/airbyte-integrations/connectors/source-delighted/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-delighted/integration_tests/invalid_config.json index f3732995784f2..b2856c90e4be9 100644 --- a/airbyte-integrations/connectors/source-delighted/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-delighted/integration_tests/invalid_config.json @@ -1,3 +1,4 @@ { - "todo-wrong-field": "this should be an incomplete config file, used in standard tests" + "api_key": "wrong api key", + "since": 1625328197 } diff --git a/airbyte-integrations/connectors/source-delighted/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-delighted/integration_tests/sample_config.json index ecc4913b84c74..a64324967e854 100644 --- a/airbyte-integrations/connectors/source-delighted/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-delighted/integration_tests/sample_config.json @@ -1,3 +1,4 @@ { - "fix-me": "TODO" + "api_key": "your api key", + "since": 1625328167 } diff --git a/airbyte-integrations/connectors/source-delighted/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-delighted/integration_tests/sample_state.json index 69f080867ea9a..80f39260626d0 100644 --- a/airbyte-integrations/connectors/source-delighted/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-delighted/integration_tests/sample_state.json @@ -2,10 +2,10 @@ "people": { "created_at": "1601586688" }, - "unsubscribed_people": { + "unsubscribes": { "unsubscribed_at": "1601586688" }, - "bounced_people": { + "bounces": { "bounced_at": "1601586688" }, "survey_responses": { diff --git a/airbyte-integrations/connectors/source-delighted/main.py b/airbyte-integrations/connectors/source-delighted/main.py index 2780d48f93cc4..39b0225242fed 100644 --- a/airbyte-integrations/connectors/source-delighted/main.py +++ b/airbyte-integrations/connectors/source-delighted/main.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + # MIT License # # Copyright (c) 2020 Airbyte diff --git a/airbyte-integrations/connectors/source-delighted/setup.py b/airbyte-integrations/connectors/source-delighted/setup.py index 9c1259719dbac..e5f41ecef635a 100644 --- a/airbyte-integrations/connectors/source-delighted/setup.py +++ b/airbyte-integrations/connectors/source-delighted/setup.py @@ -1,24 +1,6 @@ -# MIT License # -# Copyright (c) 2020 Airbyte +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. # -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. from setuptools import find_packages, setup diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/TODO.md b/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/TODO.md deleted file mode 100644 index cf1efadb3c9c9..0000000000000 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/TODO.md +++ /dev/null @@ -1,25 +0,0 @@ -# TODO: Define your stream schemas -Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). - -The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. - -The schema of a stream is the return value of `Stream.get_json_schema`. - -## Static schemas -By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. - -Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. - -## Dynamic schemas -If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). - -## Dynamically modifying static schemas -Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: -``` -def get_json_schema(self): - schema = super().get_json_schema() - schema['dynamically_determined_property'] = "property" - return schema -``` - -Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/bounced_people.json b/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/bounced_people.json deleted file mode 100644 index f651c7a46478e..0000000000000 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/bounced_people.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "person_id": { - "type": "string" - }, - "email": { - "type": "string" - }, - "name": { - "type": "string" - }, - "bounced_at": { - "type": "integer" - } - } - } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/bounces.json b/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/bounces.json new file mode 100644 index 0000000000000..76c573c99b2a5 --- /dev/null +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/bounces.json @@ -0,0 +1,18 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "person_id": { + "type": "string" + }, + "email": { + "type": "string" + }, + "name": { + "type": ["null", "string"] + }, + "bounced_at": { + "type": "integer" + } + } +} diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/people.json b/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/people.json index 94dc23f7de695..ca209db77baa6 100644 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/people.json +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/people.json @@ -6,10 +6,10 @@ "type": "string" }, "name": { - "type": "string" + "type": ["null", "string"] }, "email": { - "type": "string" + "type": ["null", "string"] }, "created_at": { "type": "integer" diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/unsubscribed_people.json b/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/unsubscribes.json similarity index 89% rename from airbyte-integrations/connectors/source-delighted/source_delighted/schemas/unsubscribed_people.json rename to airbyte-integrations/connectors/source-delighted/source_delighted/schemas/unsubscribes.json index 79e4cbab38e50..2276326b74bb3 100644 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/unsubscribed_people.json +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/unsubscribes.json @@ -9,7 +9,7 @@ "type": "string" }, "name": { - "type": "string" + "type": ["null", "string"] }, "unsubscribed_at": { "type": "integer" diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py index 67b945c23f98d..2f25139320201 100644 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py @@ -1,38 +1,19 @@ -# MIT License # -# Copyright (c) 2020 Airbyte +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. # -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. import base64 from abc import ABC -from typing import (Any, Iterable, List, Mapping, MutableMapping, Optional, - Tuple) +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple from urllib.parse import parse_qsl, urlparse import requests +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator -from requests.auth import HTTPBasicAuth # Basic full refresh stream @@ -73,7 +54,6 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp class IncrementalDelightedStream(DelightedStream, ABC): - # Getting page size as 'limit' from parrent class @property def limit(self): @@ -97,14 +77,13 @@ def request_params(self, stream_state=None, **kwargs): class People(IncrementalDelightedStream): - def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: return "people.json" -class UnsubscribedPeople(IncrementalDelightedStream): +class Unsubscribes(IncrementalDelightedStream): cursor_field = "unsubscribed_at" primary_key = "person_id" @@ -114,7 +93,7 @@ def path( return "unsubscribes.json" -class BouncedPeople(IncrementalDelightedStream): +class Bounces(IncrementalDelightedStream): cursor_field = "bounced_at" primary_key = "person_id" @@ -142,6 +121,10 @@ def request_params(self, stream_state=None, **kwargs): # Source class SourceDelighted(AbstractSource): + def _get_authenticator(self, config): + token = base64.b64encode(f"{config['api_key']}:".encode("utf-8")).decode("utf-8") + return TokenAuthenticator(token=token, auth_method="Basic") + def check_connection(self, logger, config) -> Tuple[bool, any]: """ @@ -151,22 +134,18 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: :param logger: logger object :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ - api_key = config["api_key"] try: - session = requests.get('https://api.delighted.com/v1/people.json', auth=HTTPBasicAuth(api_key, '')) - session.raise_for_status() + auth = self._get_authenticator(config) + args = {"authenticator": auth, "since": config["since"]} + stream = SurveyResponses(**args) + records = stream.read_records(sync_mode=SyncMode.full_refresh) + next(records) return True, None - except requests.exceptions.RequestException as e: + except Exception as e: return False, e def streams(self, config: Mapping[str, Any]) -> List[Stream]: - token = base64.b64encode(f"{config['api_key']}:".encode("utf-8")).decode("utf-8") - auth = TokenAuthenticator(token=token, auth_method="Basic") - + auth = self._get_authenticator(config) args = {"authenticator": auth, "since": config["since"]} - - return [People(**args), - UnsubscribedPeople(**args), - BouncedPeople(**args), - SurveyResponses(**args)] + return [People(**args), Unsubscribes(**args), Bounces(**args), SurveyResponses(**args)] diff --git a/airbyte-integrations/connectors/source-delighted/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-delighted/unit_tests/unit_test.py index f03f99f7c46e7..e1814314fc3b0 100644 --- a/airbyte-integrations/connectors/source-delighted/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-delighted/unit_tests/unit_test.py @@ -1,24 +1,6 @@ -# MIT License # -# Copyright (c) 2020 Airbyte +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. # -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. def test_example_method(): diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index a9dc7323efc28..e94024b3aaa5d 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -64,6 +64,7 @@ write_standard_creds source-braintree "$BRAINTREE_TEST_CREDS" write_standard_creds source-cart "$CART_TEST_CREDS" write_standard_creds source-chargebee "$CHARGEBEE_INTEGRATION_TEST_CREDS" write_standard_creds source-close-com "$SOURCE_CLOSE_COM_CREDS" +write_standard_creds source-delighted "$SOURCE_DELIGHTED_TEST_CREDS" write_standard_creds source-drift "$DRIFT_INTEGRATION_TEST_CREDS" write_standard_creds source-dixa "$SOURCE_DIXA_TEST_CREDS" write_standard_creds source-exchange-rates "$EXCHANGE_RATES_TEST_CREDS"