From 2c6f0fd2625ad4233d59de97f7c247f22d01f9f2 Mon Sep 17 00:00:00 2001 From: Luis Gomez Date: Sat, 26 Mar 2022 21:21:29 -0500 Subject: [PATCH 1/6] Handled pagination on Users stream --- .../connectors/source-notion/source_notion/streams.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/airbyte-integrations/connectors/source-notion/source_notion/streams.py b/airbyte-integrations/connectors/source-notion/source_notion/streams.py index 3643152b94cf52..d0b08787598491 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/streams.py +++ b/airbyte-integrations/connectors/source-notion/source_notion/streams.py @@ -149,6 +149,12 @@ class Users(NotionStream): def path(self, **kwargs) -> str: return "users" + def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + params = {"page_size": self.page_size} + if next_page_token: + params["start_cursor"] = next_page_token["next_cursor"] + return params + class Databases(IncrementalNotionStream): """ From 11cfa8358da51f490bad4fcc4d30dfedb22f7d2f Mon Sep 17 00:00:00 2001 From: Luis Gomez Date: Sat, 26 Mar 2022 21:21:48 -0500 Subject: [PATCH 2/6] Added unit tests --- .../source-notion/unit_tests/test_streams.py | 61 ++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py index 61fd6f16ed4857..e4433eb2012f4b 100644 --- a/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py @@ -7,7 +7,8 @@ import pytest import requests -from source_notion.streams import NotionStream +from airbyte_cdk.models import SyncMode +from source_notion.streams import NotionStream, Users @pytest.fixture @@ -76,3 +77,61 @@ def test_backoff_time(patch_base_class): stream = NotionStream(config=MagicMock()) expected_backoff_time = None assert stream.backoff_time(response_mock) == expected_backoff_time + + +def test_users_request_params(patch_base_class): + stream = Users(config=MagicMock()) + + # No next_page_token. First pull + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_params = {"page_size": 100} + assert stream.request_params(**inputs) == expected_params + + # When getting pages after the first pull. + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": {"next_cursor": "123"}} + expected_params = {"start_cursor": "123", "page_size": 100} + assert stream.request_params(**inputs) == expected_params + + +def test_user_stream_handles_pagination_correclty(requests_mock): + """ + Test shows that Users stream uses pagination as per Notion API docs. + """ + + response_body = { + "object": "list", + "results": [{"id": f"{x}"} for x in range(100)], + "next_cursor": "test_cursor_1", + "has_more": True, + "type": "user" + } + url = "https://api.notion.com/v1/users?page_size=100" + requests_mock.get(url, json=response_body) + + response_body = { + "object": "list", + "results": [{"id": f"{x}"} for x in range(100, 200)], + "next_cursor": "test_cursor_2", + "has_more": True, + "type": "user" + } + url = "https://api.notion.com/v1/users?page_size=100&start_cursor=test_cursor_1" + requests_mock.get(url, json=response_body) + + response_body = { + "object": "list", + "results": [{"id": f"{x}"} for x in range(200, 220)], + "next_cursor": None, + "has_more": False, + "type": "user" + } + url = "https://api.notion.com/v1/users?page_size=100&start_cursor=test_cursor_2" + requests_mock.get(url, json=response_body) + + stream = Users(config=MagicMock()) + + user_ids = set() + for record in stream.read_records(sync_mode=SyncMode.full_refresh): + user_ids.add(record["id"]) + + assert len(user_ids) == 220 From fda250ecf078624eff22eb745c3eaa7337c947ce Mon Sep 17 00:00:00 2001 From: Luis Gomez Date: Mon, 28 Mar 2022 09:27:57 -0500 Subject: [PATCH 3/6] Refactored unit test --- .../source-notion/unit_tests/test_streams.py | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py index e4433eb2012f4b..c597f82e051abf 100644 --- a/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py @@ -3,6 +3,7 @@ # from http import HTTPStatus +import random from unittest.mock import MagicMock import pytest @@ -100,38 +101,33 @@ def test_user_stream_handles_pagination_correclty(requests_mock): response_body = { "object": "list", - "results": [{"id": f"{x}"} for x in range(100)], - "next_cursor": "test_cursor_1", + "results": [{"id": f"{x}", "object": "user", "type": ["person", "bot"][random.randint(0, 1)]} for x in range(100)], + "next_cursor": "bc48234b-77b2-41a6-95a3-6a8abb7887d5", "has_more": True, "type": "user" } - url = "https://api.notion.com/v1/users?page_size=100" - requests_mock.get(url, json=response_body) + requests_mock.get("https://api.notion.com/v1/users?page_size=100", json=response_body) response_body = { "object": "list", - "results": [{"id": f"{x}"} for x in range(100, 200)], - "next_cursor": "test_cursor_2", + "results": [{"id": f"{x}", "object": "user", "type": ["person", "bot"][random.randint(0, 1)]} for x in range(100, 200)], + "next_cursor": "67030467-b97b-4729-8fd6-2fb33d012da4", "has_more": True, "type": "user" } - url = "https://api.notion.com/v1/users?page_size=100&start_cursor=test_cursor_1" - requests_mock.get(url, json=response_body) + requests_mock.get("https://api.notion.com/v1/users?page_size=100&start_cursor=bc48234b-77b2-41a6-95a3-6a8abb7887d5", json=response_body) response_body = { "object": "list", - "results": [{"id": f"{x}"} for x in range(200, 220)], + "results": [{"id": f"{x}", "object": "user", "type": ["person", "bot"][random.randint(0, 1)]} for x in range(200, 220)], "next_cursor": None, "has_more": False, "type": "user" } - url = "https://api.notion.com/v1/users?page_size=100&start_cursor=test_cursor_2" - requests_mock.get(url, json=response_body) + requests_mock.get("https://api.notion.com/v1/users?page_size=100&start_cursor=67030467-b97b-4729-8fd6-2fb33d012da4", json=response_body) stream = Users(config=MagicMock()) - user_ids = set() - for record in stream.read_records(sync_mode=SyncMode.full_refresh): - user_ids.add(record["id"]) - - assert len(user_ids) == 220 + records = stream.read_records(sync_mode=SyncMode.full_refresh) + records_length = sum(1 for _ in records) + assert records_length == 220 From 1fbb529621594f2df6dcc9f5fbbf0126cf237c6d Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 22 Apr 2022 11:38:47 +0200 Subject: [PATCH 4/6] bump version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-integrations/connectors/source-notion/Dockerfile | 2 +- docs/integrations/sources/notion.md | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 007992c6f0f9ad..62e033a859129c 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -508,7 +508,7 @@ - name: Notion sourceDefinitionId: 6e00b415-b02e-4160-bf02-58176a0ae687 dockerRepository: airbyte/source-notion - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.io/integrations/sources/notion icon: notion.svg sourceType: api diff --git a/airbyte-integrations/connectors/source-notion/Dockerfile b/airbyte-integrations/connectors/source-notion/Dockerfile index e6bb3527b120c9..a2c17dc89b80e5 100644 --- a/airbyte-integrations/connectors/source-notion/Dockerfile +++ b/airbyte-integrations/connectors/source-notion/Dockerfile @@ -34,5 +34,5 @@ COPY source_notion ./source_notion ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/source-notion diff --git a/docs/integrations/sources/notion.md b/docs/integrations/sources/notion.md index 98c9de9bd44576..6c6c390178e5a0 100644 --- a/docs/integrations/sources/notion.md +++ b/docs/integrations/sources/notion.md @@ -57,6 +57,7 @@ Please register on Notion and follow this [docs](https://developers.notion.com/d | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.3 | 2022-04-22 | [11452](https://github.com/airbytehq/airbyte/pull/11452) | Use pagination for User stream | | 0.1.2 | 2022-01-11 | [9084](https://github.com/airbytehq/airbyte/pull/9084) | Fix documentation URL | | 0.1.1 | 2021-12-30 | [9207](https://github.com/airbytehq/airbyte/pull/9207) | Update connector fields title/description | | 0.1.0 | 2021-10-17 | [7092](https://github.com/airbytehq/airbyte/pull/7092) | Initial Release | From ac5965c7fc8c56a0ef6f2e40a76335aa73e650b7 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 22 Apr 2022 15:56:00 +0200 Subject: [PATCH 5/6] update schemas --- .../source-notion/source_notion/schemas/blocks.json | 2 ++ .../source-notion/source_notion/schemas/databases.json | 9 +++++++++ .../source_notion/schemas/shared/heading.json | 1 + .../source_notion/schemas/shared/text_element.json | 1 + 4 files changed, 13 insertions(+) diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json index 4bc8809113fd93..65af36a7fa66d5 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json @@ -57,6 +57,7 @@ "type": "object", "additionalProperties": false, "properties": { + "color": { "type": "string" }, "text": { "type": "array", "items": { "$ref": "rich_text.json" } }, "icon": { "$ref": "icon.json" }, "children": { "type": "array", "items": { "type": "object" } } @@ -75,6 +76,7 @@ "type": "object", "additionalProperties": false, "properties": { + "color": { "type": "string" }, "text": { "type": "array", "items": { "$ref": "rich_text.json" } }, "language": { "enum": [ diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/databases.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/databases.json index d24616a275582b..7c434f3bfd5dd8 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/databases.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/databases.json @@ -21,6 +21,15 @@ "$ref": "rich_text.json" } }, + "last_edited_by": { + "$ref": "user.json" + }, + "created_by": { + "$ref": "user.json" + }, + "archived": { + "type": "boolean" + }, "icon": { "$ref": "icon.json" }, diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/heading.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/heading.json index 9bf06eb9189ca9..f99e50bd251037 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/heading.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/heading.json @@ -3,6 +3,7 @@ "type": ["null", "object"], "additionalProperties": false, "properties": { + "color": { "type": "string" }, "text": { "type": "array", "items": { "$ref": "rich_text.json" } } } } diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/text_element.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/text_element.json index 7f122b53b9c252..35d86f664146f0 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/text_element.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/text_element.json @@ -3,6 +3,7 @@ "type": "object", "additionalProperties": false, "properties": { + "color": { "type": "string" }, "text": { "type": "array", "items": { "$ref": "rich_text.json" } }, "children": { "type": "array", "items": { "type": "object" } } } From e241d2194f4ab4f171ea5084b0d7e8e41ee59fe4 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 22 Apr 2022 16:13:06 +0200 Subject: [PATCH 6/6] update source specs --- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 29615c5c7d2a01..59d0498cd65eba 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -5542,7 +5542,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-notion:0.1.2" +- dockerImage: "airbyte/source-notion:0.1.3" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/notion" connectionSpecification: