Publish PR 4551: new source delighted correct format (airbytehq#7540)

* Add Delighted source with people, unsubscribes and bounces streams

* Add survey responses stream

* Add documentation for Delighted source

* publish delighted source

* correct acceptance test

Co-authored-by: Rodrigo Parra <rodpar07@gmail.com>

.github/workflows/publish-command.yml

@@ -196,6 +196,7 @@ jobs:
           SOURCE_LEMLIST_TEST_CREDS: ${{ secrets.SOURCE_LEMLIST_TEST_CREDS }}
           SOURCE_STRAVA_TEST_CREDS: ${{ secrets.SOURCE_STRAVA_TEST_CREDS }}
           SOURCE_PAYSTACK_TEST_CREDS: ${{ secrets.SOURCE_PAYSTACK_TEST_CREDS }}
+          SOURCE_DELIGHTED_TEST_CREDS: ${{ secrets.SOURCE_DELIGHTED_TEST_CREDS }}
       - run: |
           echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u airbytebot -p ${DOCKER_PASSWORD}
           ./tools/integrations/manage.sh publish airbyte-integrations/${{ github.event.inputs.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache

.github/workflows/test-command.yml

@@ -191,6 +191,7 @@ jobs:
           SOURCE_LEMLIST_TEST_CREDS: ${{ secrets.SOURCE_LEMLIST_TEST_CREDS }}
           SOURCE_STRAVA_TEST_CREDS: ${{ secrets.SOURCE_STRAVA_TEST_CREDS }}
           SOURCE_PAYSTACK_TEST_CREDS: ${{ secrets.SOURCE_PAYSTACK_TEST_CREDS }}
+          SOURCE_DELIGHTED_TEST_CREDS: ${{ secrets.SOURCE_DELIGHTED_TEST_CREDS }}
       - run: |
           ./tools/bin/ci_integration_test.sh ${{ github.event.inputs.connector }}
         name: test ${{ github.event.inputs.connector }}

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/cc88c43f-6f53-4e8a-8c4d-b284baaf9635.json

@@ -0,0 +1,7 @@
+{
+  "sourceDefinitionId": "cc88c43f-6f53-4e8a-8c4d-b284baaf9635",
+  "name": "Delighted",
+  "dockerRepository": "airbyte/source-delighted",
+  "dockerImageTag": "0.1.0",
+  "documentationUrl": "https://docs.airbyte.io/integrations/sources/delighted"
+}

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -102,6 +102,12 @@
   dockerImageTag: 0.1.2
   documentationUrl: https://docs.airbyte.io/integrations/sources/cockroachdb
   sourceType: database
+- name: Delighted
+  sourceDefinitionId: cc88c43f-6f53-4e8a-8c4d-b284baaf9635
+  dockerRepository: airbyte/source-delighted
+  dockerImageTag: 0.1.0
+  documentationUrl: https://docs.airbyte.io/integrations/sources/delighted
+  sourceType: api
 - name: Dixa
   sourceDefinitionId: 0b5c867e-1b12-4d02-ab74-97b2184ff6d7
   dockerRepository: airbyte/source-dixa

airbyte-integrations/connectors/source-delighted/acceptance-test-config.yml

@@ -8,12 +8,13 @@ tests:
     - config_path: "secrets/config.json"
       status: "succeed"
     - config_path: "integration_tests/invalid_config.json"
-      status: "exception"
+      status: "failed"
   discovery:
     - config_path: "secrets/config.json"
   basic_read:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
+      empty_streams: ["bounces"]
   incremental: # TODO if your connector does not implement incremental sync, remove this block
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"

airbyte-integrations/connectors/source-delighted/integration_tests/abnormal_state.json

@@ -2,10 +2,10 @@
   "people": {
     "created_at": "4126108288"
   },
-  "unsubscribed_people": {
+  "unsubscribes": {
     "unsubscribed_at": "4126108288"
   },
-  "bounced_people": {
+  "bounces": {
     "bounced_at": "4126108288"
   },
   "survey_responses": {

airbyte-integrations/connectors/source-delighted/integration_tests/acceptance.py

@@ -1,24 +1,6 @@
-# MIT License
 #
-# Copyright (c) 2020 Airbyte
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in all
-# copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
 
 
 import pytest

airbyte-integrations/connectors/source-delighted/integration_tests/configured_catalog.json

@@ -12,7 +12,7 @@
     },
     {
       "stream": {
-        "name": "unsubscribed_people",
+        "name": "unsubscribes",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh"],
         "source_defined_primary_key": [["id"]]
@@ -22,7 +22,7 @@
     },
     {
       "stream": {
-        "name": "bounced_people",
+        "name": "bounces",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh"],
         "source_defined_primary_key": [["id"]]

airbyte-integrations/connectors/source-delighted/integration_tests/invalid_config.json

@@ -1,3 +1,4 @@
 {
-  "todo-wrong-field": "this should be an incomplete config file, used in standard tests"
+  "api_key": "wrong api key",
+  "since": 1625328197
 }

airbyte-integrations/connectors/source-delighted/integration_tests/sample_config.json

@@ -1,3 +1,4 @@
 {
-  "fix-me": "TODO"
+  "api_key": "your api key",
+  "since": 1625328167
 }

airbyte-integrations/connectors/source-delighted/integration_tests/sample_state.json

@@ -2,10 +2,10 @@
   "people": {
     "created_at": "1601586688"
   },
-  "unsubscribed_people": {
+  "unsubscribes": {
     "unsubscribed_at": "1601586688"
   },
-  "bounced_people": {
+  "bounces": {
     "bounced_at": "1601586688"
   },
   "survey_responses": {

airbyte-integrations/connectors/source-delighted/main.py

@@ -1,3 +1,7 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+
 # MIT License
 #
 # Copyright (c) 2020 Airbyte

airbyte-integrations/connectors/source-delighted/setup.py

@@ -1,24 +1,6 @@
-# MIT License
 #
-# Copyright (c) 2020 Airbyte
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in all
-# copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
 
 
 from setuptools import find_packages, setup

airbyte-integrations/connectors/source-delighted/source_delighted/schemas/bounces.json

@@ -0,0 +1,18 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "person_id": {
+      "type": "string"
+    },
+    "email": {
+      "type": "string"
+    },
+    "name": {
+      "type": ["null", "string"]
+    },
+    "bounced_at": {
+      "type": "integer"
+    }
+  }
+}

airbyte-integrations/connectors/source-delighted/source_delighted/schemas/people.json

@@ -6,10 +6,10 @@
       "type": "string"
     },
     "name": {
-      "type": "string"
+      "type": ["null", "string"]
     },
     "email": {
-      "type": "string"
+      "type": ["null", "string"]
     },
     "created_at": {
       "type": "integer"

airbyte-integrations/connectors/source-delighted/source_delighted/schemas/unsubscribed_people.json → airbyte-integrations/connectors/source-delighted/source_delighted/schemas/unsubscribes.json

@@ -9,7 +9,7 @@
       "type": "string"
     },
     "name": {
-      "type": "string"
+      "type": ["null", "string"]
     },
     "unsubscribed_at": {
       "type": "integer"

airbyte-integrations/connectors/source-delighted/source_delighted/source.py

@@ -1,38 +1,19 @@
-# MIT License
 #
-# Copyright (c) 2020 Airbyte
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in all
-# copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
 
 
 import base64
 from abc import ABC
-from typing import (Any, Iterable, List, Mapping, MutableMapping, Optional,
-                    Tuple)
+from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple
 from urllib.parse import parse_qsl, urlparse
 
 import requests
+from airbyte_cdk.models import SyncMode
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.http import HttpStream
 from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator
-from requests.auth import HTTPBasicAuth
 
 
 # Basic full refresh stream
@@ -73,7 +54,6 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp
 
 
 class IncrementalDelightedStream(DelightedStream, ABC):
-
     # Getting page size as 'limit' from parrent class
     @property
     def limit(self):
@@ -97,14 +77,13 @@ def request_params(self, stream_state=None, **kwargs):
 
 
 class People(IncrementalDelightedStream):
-
     def path(
         self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
     ) -> str:
         return "people.json"
 
 
-class UnsubscribedPeople(IncrementalDelightedStream):
+class Unsubscribes(IncrementalDelightedStream):
     cursor_field = "unsubscribed_at"
     primary_key = "person_id"
 
@@ -114,7 +93,7 @@ def path(
         return "unsubscribes.json"
 
 
-class BouncedPeople(IncrementalDelightedStream):
+class Bounces(IncrementalDelightedStream):
     cursor_field = "bounced_at"
     primary_key = "person_id"
 
@@ -142,6 +121,10 @@ def request_params(self, stream_state=None, **kwargs):
 
 # Source
 class SourceDelighted(AbstractSource):
+    def _get_authenticator(self, config):
+        token = base64.b64encode(f"{config['api_key']}:".encode("utf-8")).decode("utf-8")
+        return TokenAuthenticator(token=token, auth_method="Basic")
+
     def check_connection(self, logger, config) -> Tuple[bool, any]:
         """
 
@@ -151,22 +134,18 @@ def check_connection(self, logger, config) -> Tuple[bool, any]:
         :param logger:  logger object
         :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise.
         """
-        api_key = config["api_key"]
 
         try:
-            session = requests.get('https://api.delighted.com/v1/people.json', auth=HTTPBasicAuth(api_key, ''))
-            session.raise_for_status()
+            auth = self._get_authenticator(config)
+            args = {"authenticator": auth, "since": config["since"]}
+            stream = SurveyResponses(**args)
+            records = stream.read_records(sync_mode=SyncMode.full_refresh)
+            next(records)
             return True, None
-        except requests.exceptions.RequestException as e:
+        except Exception as e:
             return False, e
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
-        token = base64.b64encode(f"{config['api_key']}:".encode("utf-8")).decode("utf-8")
-        auth = TokenAuthenticator(token=token, auth_method="Basic")
-
+        auth = self._get_authenticator(config)
         args = {"authenticator": auth, "since": config["since"]}
-
-        return [People(**args),
-                UnsubscribedPeople(**args),
-                BouncedPeople(**args),
-                SurveyResponses(**args)]
+        return [People(**args), Unsubscribes(**args), Bounces(**args), SurveyResponses(**args)]

airbyte-integrations/connectors/source-delighted/unit_tests/unit_test.py

@@ -1,24 +1,6 @@
-# MIT License
 #
-# Copyright (c) 2020 Airbyte
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in all
-# copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
 
 
 def test_example_method():