From 98dc5cd43ae36d4ed4668a45fe88db159968b2ff Mon Sep 17 00:00:00 2001 From: oleh Date: Tue, 4 Jan 2022 12:36:49 +0200 Subject: [PATCH 1/4] 9271 Fix pagination for delighted source --- .../connectors/source-delighted/Dockerfile | 2 +- .../acceptance-test-config.yml | 2 +- .../schemas/survey_responses.json | 32 ++++++------- .../source_delighted/source.py | 45 ++++++++++--------- docs/integrations/sources/delighted.md | 1 + 5 files changed, 43 insertions(+), 39 deletions(-) diff --git a/airbyte-integrations/connectors/source-delighted/Dockerfile b/airbyte-integrations/connectors/source-delighted/Dockerfile index 4853739994eb2..1e1396f91cea8 100644 --- a/airbyte-integrations/connectors/source-delighted/Dockerfile +++ b/airbyte-integrations/connectors/source-delighted/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-delighted diff --git a/airbyte-integrations/connectors/source-delighted/acceptance-test-config.yml b/airbyte-integrations/connectors/source-delighted/acceptance-test-config.yml index e8df991ce5376..5158a03f40509 100644 --- a/airbyte-integrations/connectors/source-delighted/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-delighted/acceptance-test-config.yml @@ -15,7 +15,7 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: ["bounces"] - incremental: # TODO if your connector does not implement incremental sync, remove this block + incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" future_state_path: "integration_tests/abnormal_state.json" diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/survey_responses.json b/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/survey_responses.json index 00d220722441c..3cd7f3bf1249b 100644 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/survey_responses.json +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/survey_responses.json @@ -27,7 +27,21 @@ "type": ["null", "integer"] }, "person_properties": { - "type": "object" + "type": ["object", "null"], + "properties": { + "Delighted Source": { + "type": ["null", "string"] + }, + "Delighted Device Type": { + "type": ["null", "string"] + }, + "Delighted Operating System": { + "type": ["null", "string"] + }, + "Delighted Browser": { + "type": ["null", "string"] + } + } }, "notes": { "type": "array", @@ -102,19 +116,5 @@ "required": ["id", "value", "question"] } } - }, - "required": [ - "id", - "person", - "survey_type", - "score", - "comment", - "permalink", - "created_at", - "updated_at", - "person_properties", - "notes", - "tags", - "additional_answers" - ] + } } diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py index 2f25139320201..2bbea1bc357f1 100644 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py @@ -23,6 +23,7 @@ class DelightedStream(HttpStream, ABC): # Page size limit = 100 + page = 1 # Define primary key to all streams as primary key primary_key = "id" @@ -32,12 +33,10 @@ def __init__(self, since: int, **kwargs): self.since = since def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - # Getting next page link - next_page = response.links.get("next", None) - if next_page: - return dict(parse_qsl(urlparse(next_page.get("url")).query)) - else: - return None + response_data = response.json() + if response_data and len(response_data) == self.limit: + self.page += 1 + return {"page": self.page} def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None @@ -49,8 +48,7 @@ def request_params( return params def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - records = response.json() - yield from records + yield from response.json() class IncrementalDelightedStream(DelightedStream, ABC): @@ -77,19 +75,23 @@ def request_params(self, stream_state=None, **kwargs): class People(IncrementalDelightedStream): - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: + def path(self, **kwargs) -> str: return "people.json" + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + # Getting next page link + next_page = response.links.get("next", None) + if next_page: + return {"page_info": dict(parse_qsl(urlparse(next_page.get("url")).query)).get("page_info")} + else: + return None + class Unsubscribes(IncrementalDelightedStream): cursor_field = "unsubscribed_at" primary_key = "person_id" - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: + def path(self, **kwargs) -> str: return "unsubscribes.json" @@ -97,18 +99,14 @@ class Bounces(IncrementalDelightedStream): cursor_field = "bounced_at" primary_key = "person_id" - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: + def path(self, **kwargs) -> str: return "bounces.json" class SurveyResponses(IncrementalDelightedStream): cursor_field = "updated_at" - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: + def path(self, **kwargs) -> str: return "survey_responses.json" def request_params(self, stream_state=None, **kwargs): @@ -148,4 +146,9 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: def streams(self, config: Mapping[str, Any]) -> List[Stream]: auth = self._get_authenticator(config) args = {"authenticator": auth, "since": config["since"]} - return [People(**args), Unsubscribes(**args), Bounces(**args), SurveyResponses(**args)] + return [ + Bounces(**args), + People(**args), + SurveyResponses(**args), + Unsubscribes(**args), + ] diff --git a/docs/integrations/sources/delighted.md b/docs/integrations/sources/delighted.md index 6c4865d00fe32..b39af63d7fa5d 100644 --- a/docs/integrations/sources/delighted.md +++ b/docs/integrations/sources/delighted.md @@ -37,4 +37,5 @@ This connector supports `API PASSWORD` as the authentication method. | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.1 | 2022-01-04 | [9275](https://github.com/airbytehq/airbyte/pull/9275) | Fix pagination handling for `survey_responses`, `bounces` and `unsubscribes` streams | | 0.1.0 | 2021-10-27 | [4551](https://github.com/airbytehq/airbyte/pull/4551) | Add Delighted source connector | \ No newline at end of file From f88d6acd6159a3465945581b89173bec3223e756 Mon Sep 17 00:00:00 2001 From: oleh Date: Tue, 4 Jan 2022 15:42:34 +0200 Subject: [PATCH 2/4] 9271 Implement change request --- .../source_delighted/schemas/survey_responses.json | 12 ++++-------- .../source-delighted/source_delighted/source.py | 2 -- docs/integrations/sources/delighted.md | 4 ++-- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/survey_responses.json b/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/survey_responses.json index 3cd7f3bf1249b..2d97340ea9e8c 100644 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/survey_responses.json +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/schemas/survey_responses.json @@ -90,12 +90,10 @@ "text": { "type": "string" } - }, - "required": ["id", "text"] + } } } - }, - "required": ["free_response", "scale", "select_one", "select_many"] + } }, "question": { "type": "object", @@ -109,11 +107,9 @@ "text": { "type": "string" } - }, - "required": ["id", "type", "text"] + } } - }, - "required": ["id", "value", "question"] + } } } } diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py index 2bbea1bc357f1..9ab644b2fdd60 100644 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py @@ -83,8 +83,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, next_page = response.links.get("next", None) if next_page: return {"page_info": dict(parse_qsl(urlparse(next_page.get("url")).query)).get("page_info")} - else: - return None class Unsubscribes(IncrementalDelightedStream): diff --git a/docs/integrations/sources/delighted.md b/docs/integrations/sources/delighted.md index b39af63d7fa5d..9402469ee0d7b 100644 --- a/docs/integrations/sources/delighted.md +++ b/docs/integrations/sources/delighted.md @@ -37,5 +37,5 @@ This connector supports `API PASSWORD` as the authentication method. | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| 0.1.1 | 2022-01-04 | [9275](https://github.com/airbytehq/airbyte/pull/9275) | Fix pagination handling for `survey_responses`, `bounces` and `unsubscribes` streams | -| 0.1.0 | 2021-10-27 | [4551](https://github.com/airbytehq/airbyte/pull/4551) | Add Delighted source connector | \ No newline at end of file +| 0.1.1 | 2022-01-04 | [9275](https://github.com/airbytehq/airbyte/pull/9275) | Fix pagination handling for `survey_responses`, `bounces` and `unsubscribes` streams | +| 0.1.0 | 2021-10-27 | [4551](https://github.com/airbytehq/airbyte/pull/4551) | Add Delighted source connector | From beae12868b730ff34c2712773e3b359347a2d4b5 Mon Sep 17 00:00:00 2001 From: oleh Date: Tue, 4 Jan 2022 16:12:34 +0200 Subject: [PATCH 3/4] 9271 Implement change request --- .../connectors/source-delighted/source_delighted/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py index 9ab644b2fdd60..80e738728afaa 100644 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py @@ -34,7 +34,7 @@ def __init__(self, since: int, **kwargs): def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: response_data = response.json() - if response_data and len(response_data) == self.limit: + if len(response_data) == self.limit: self.page += 1 return {"page": self.page} From e96c8c07d8fcffb64e50da0f4cb242eb6b8f1011 Mon Sep 17 00:00:00 2001 From: oleh Date: Tue, 4 Jan 2022 16:31:32 +0200 Subject: [PATCH 4/4] 9271 Bump connector's version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 5e160473d5b31..6cff1a6aea120 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -147,7 +147,7 @@ - name: Delighted sourceDefinitionId: cc88c43f-6f53-4e8a-8c4d-b284baaf9635 dockerRepository: airbyte/source-delighted - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/delighted icon: delighted.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 3b0d4c2c573be..f5afc38ea579f 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1280,7 +1280,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-delighted:0.1.0" +- dockerImage: "airbyte/source-delighted:0.1.1" spec: documentationUrl: "https://docsurl.com" connectionSpecification: