From 4249411c2d789815b646013a50ad9e0d072f926b Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Thu, 11 Nov 2021 18:01:23 +0200 Subject: [PATCH 1/5] stream can be empty for "check" Signed-off-by: Sergey Chvalyuk --- .../badc5925-0485-42be-8caa-b34096cb71b5.json | 2 +- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-integrations/connectors/source-surveymonkey/Dockerfile | 2 +- .../source-surveymonkey/source_surveymonkey/source.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/badc5925-0485-42be-8caa-b34096cb71b5.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/badc5925-0485-42be-8caa-b34096cb71b5.json index d0e74a25ca0c7c..dc0ea8587af63a 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/badc5925-0485-42be-8caa-b34096cb71b5.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/badc5925-0485-42be-8caa-b34096cb71b5.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "badc5925-0485-42be-8caa-b34096cb71b5", "name": "Survey Monkey", "dockerRepository": "airbyte/source-surveymonkey", - "dockerImageTag": "0.1.3", + "dockerImageTag": "0.1.4", "documentationUrl": "https://docs.airbyte.io/integrations/sources/surveymonkey" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 46ce0b9aa4d236..bd683f3a28b10f 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -555,7 +555,7 @@ - name: Survey Monkey sourceDefinitionId: badc5925-0485-42be-8caa-b34096cb71b5 dockerRepository: airbyte/source-surveymonkey - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/sources/surveymonkey sourceType: api - name: Tempo diff --git a/airbyte-integrations/connectors/source-surveymonkey/Dockerfile b/airbyte-integrations/connectors/source-surveymonkey/Dockerfile index c5dacbc7389257..51e56e8a1bccdf 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/Dockerfile +++ b/airbyte-integrations/connectors/source-surveymonkey/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/source-surveymonkey diff --git a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/source.py b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/source.py index 27fafa79c5f9f1..6362f80ea2f11f 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/source.py +++ b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/source.py @@ -21,7 +21,7 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> start_date = pendulum.parse(config["start_date"]) stream = Surveys(authenticator=authenticator, start_date=start_date) records = stream.read_records(sync_mode=SyncMode.full_refresh) - next(records) + next(records, None) return True, None except Exception as e: return False, repr(e) From 2c58ef882d9f301b200d37f3f36cc3f6b4386152 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Thu, 11 Nov 2021 18:08:21 +0200 Subject: [PATCH 2/5] add changelog Signed-off-by: Sergey Chvalyuk --- docs/integrations/sources/surveymonkey.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/integrations/sources/surveymonkey.md b/docs/integrations/sources/surveymonkey.md index 6297caf0410745..573ad38d3e87ed 100644 --- a/docs/integrations/sources/surveymonkey.md +++ b/docs/integrations/sources/surveymonkey.md @@ -55,6 +55,7 @@ Please read this [docs](https://developer.surveymonkey.com/api/v3/#getting-start | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.4 | 2021-11-11 | [7868](https://github.com/airbytehq/airbyte/pull/7868) | Stream can be empty for 'check' | | 0.1.3 | 2021-11-01 | [7433](https://github.com/airbytehq/airbyte/pull/7433) | Remove unsused oAuth flow parameters | | 0.1.2 | 2021-10-27 | [7433](https://github.com/airbytehq/airbyte/pull/7433) | Add OAuth support | | 0.1.1 | 2021-09-10 | [5983](https://github.com/airbytehq/airbyte/pull/5983) | Fix caching for gzip compressed http response | From 5039be5808f83f42c59d7ef113fe2fb8c371b88b Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Thu, 18 Nov 2021 11:52:50 +0200 Subject: [PATCH 3/5] Improve 'check' using '/users/me' API call Signed-off-by: Sergey Chvalyuk --- .../source_surveymonkey/source.py | 36 +++++++++++++++---- docs/integrations/sources/surveymonkey.md | 2 +- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/source.py b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/source.py index 6362f80ea2f11f..a688e65034d326 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/source.py +++ b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/source.py @@ -5,8 +5,8 @@ from typing import Any, List, Mapping, Tuple import pendulum +import requests from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator @@ -15,13 +15,30 @@ class SourceSurveymonkey(AbstractSource): + + SCOPES = { + "collectors_read", + "contacts_read", + "groups_read", + "library_read", + "responses_read", + "responses_read_detail", + "roles_read", + "surveys_read", + "users_read", + "webhooks_read", + "workgroups_members_read", + "workgroups_read", + "workgroups_shares_read", + } + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: + url = "https://api.surveymonkey.com/v3/users/me" + authenticator = self.get_authenticator(config) try: - authenticator = self.get_authenticator(config) - start_date = pendulum.parse(config["start_date"]) - stream = Surveys(authenticator=authenticator, start_date=start_date) - records = stream.read_records(sync_mode=SyncMode.full_refresh) - next(records, None) + response = requests.get(url=url, headers=authenticator.get_auth_header()) + response.raise_for_status() + self._check_scopes(response.json()) return True, None except Exception as e: return False, repr(e) @@ -36,3 +53,10 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: def get_authenticator(config: Mapping[str, Any]): token = config["access_token"] return TokenAuthenticator(token=token) + + @classmethod + def _check_scopes(cls, response_json): + granted_scopes = response_json["scopes"]["granted"] + missed_scopes = cls.SCOPES - set(granted_scopes) + if missed_scopes: + raise Exception("missed required scopes: " + ", ".join(missed_scopes)) diff --git a/docs/integrations/sources/surveymonkey.md b/docs/integrations/sources/surveymonkey.md index 573ad38d3e87ed..d8192a45a0117e 100644 --- a/docs/integrations/sources/surveymonkey.md +++ b/docs/integrations/sources/surveymonkey.md @@ -55,7 +55,7 @@ Please read this [docs](https://developer.surveymonkey.com/api/v3/#getting-start | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| 0.1.4 | 2021-11-11 | [7868](https://github.com/airbytehq/airbyte/pull/7868) | Stream can be empty for 'check' | +| 0.1.4 | 2021-11-11 | [7868](https://github.com/airbytehq/airbyte/pull/7868) | Improve 'check' using '/users/me' API call | | 0.1.3 | 2021-11-01 | [7433](https://github.com/airbytehq/airbyte/pull/7433) | Remove unsused oAuth flow parameters | | 0.1.2 | 2021-10-27 | [7433](https://github.com/airbytehq/airbyte/pull/7433) | Add OAuth support | | 0.1.1 | 2021-09-10 | [5983](https://github.com/airbytehq/airbyte/pull/5983) | Fix caching for gzip compressed http response | From 783b90213153c21cade431c0534a54df2973f86f Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Thu, 18 Nov 2021 14:08:29 +0200 Subject: [PATCH 4/5] improve code logic Signed-off-by: Sergey Chvalyuk --- .../source-surveymonkey/source_surveymonkey/source.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/source.py b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/source.py index a688e65034d326..48a6fbbca0adf9 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/source.py +++ b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/source.py @@ -38,8 +38,7 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> try: response = requests.get(url=url, headers=authenticator.get_auth_header()) response.raise_for_status() - self._check_scopes(response.json()) - return True, None + return self._check_scopes(response.json()) except Exception as e: return False, repr(e) @@ -59,4 +58,5 @@ def _check_scopes(cls, response_json): granted_scopes = response_json["scopes"]["granted"] missed_scopes = cls.SCOPES - set(granted_scopes) if missed_scopes: - raise Exception("missed required scopes: " + ", ".join(missed_scopes)) + return False, "missed required scopes: " + ", ".join(missed_scopes) + return True, None From 19d761efd05c57b867e28501ad6ebaf2769587ac Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Thu, 18 Nov 2021 14:51:16 +0200 Subject: [PATCH 5/5] source_specs.yaml updated Signed-off-by: Sergey Chvalyuk --- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index eefc469674fae7..e3243aa63eab19 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -5660,7 +5660,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-surveymonkey:0.1.3" +- dockerImage: "airbyte/source-surveymonkey:0.1.4" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/surveymonkey" connectionSpecification: