From c01052697c1abca8fb20c0a6a5cb0b31f29cc4d8 Mon Sep 17 00:00:00 2001 From: Daryna Ishchenko <80129833+darynaishchenko@users.noreply.github.com> Date: Mon, 9 Oct 2023 13:43:29 +0300 Subject: [PATCH] :sparkles: Source Notion: added new fields for blocks and pages streams (#31053) Co-authored-by: darynaishchenko --- .../connectors/source-notion/Dockerfile | 2 +- .../integration_tests/expected_records.jsonl | 2 +- .../connectors/source-notion/metadata.yaml | 2 +- .../source_notion/schemas/blocks.json | 8 +++++++- .../source_notion/schemas/pages.json | 17 +++++++++++++++++ .../source_notion/schemas/shared/parent.json | 5 ++++- docs/integrations/sources/notion.md | 5 +++-- 7 files changed, 34 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-notion/Dockerfile b/airbyte-integrations/connectors/source-notion/Dockerfile index 71a8dcfd90ad8..e2a45f9973597 100644 --- a/airbyte-integrations/connectors/source-notion/Dockerfile +++ b/airbyte-integrations/connectors/source-notion/Dockerfile @@ -34,5 +34,5 @@ COPY source_notion ./source_notion ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.1.2 +LABEL io.airbyte.version=1.2.0 LABEL io.airbyte.name=airbyte/source-notion diff --git a/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.jsonl index 335e3021715c2..be115b818e340 100644 --- a/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.jsonl @@ -7,4 +7,4 @@ {"stream":"users","data":{"object":"user","id":"5612c094-99ec-4ba3-ac7f-df8d84c8d6be","name":"Sherif Nada","avatar_url":"https://s3-us-west-2.amazonaws.com/public.notion-static.com/305f7efc-2862-4342-ba99-5023f3e34717/6246757.png","type":"person","person":{"email":"sherif@airbyte.io"}},"emitted_at":1687166004972} {"stream":"users","data":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a","name":"Airyte","avatar_url":null,"type":"person","person":{"email":"integration-test@airbyte.io"}},"emitted_at":1687166004973} {"stream":"users","data":{"object":"user","id":"c1ff0160-b2af-497a-aab7-8b61e625e4e3","name":"Gil Cho","avatar_url":"https://lh3.googleusercontent.com/a/ALm5wu0ElXfvy3YfVUyRn-aB9EZy5AZ1ougHuNyCGmO2=s100","type":"person","person":{"email":"gil@airbyte.io"}},"emitted_at":1687166004973} -{"stream":"databases","data":{"object":"database","id":"a1298679-9f79-48a8-a991-834cd72eca17","cover":null,"icon":{"type":"emoji","emoji":"šŸš˜"},"created_time":"2021-10-19T13:33:00.000Z","created_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"last_edited_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"last_edited_time":"2023-06-15T09:18:00.000Z","title":[{"type":"text","text":{"content":"Roadmap","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Roadmap","href":null}],"description":[{"type":"text","text":{"content":"Use this template to track all of your project work.\n\nā›° ","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Use this template to track all of your project work.\n\nā›° ","href":null},{"type":"text","text":{"content":"Epics","link":null},"annotations":{"bold":true,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Epics","href":null},{"type":"text","text":{"content":" are large overarching initiatives.\nšŸƒā€ā™‚ļø ","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":" are large overarching initiatives.\nšŸƒā€ā™‚ļø ","href":null},{"type":"text","text":{"content":"Sprints","link":null},"annotations":{"bold":true,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Sprints","href":null},{"type":"text","text":{"content":" are time-bounded pushes to complete a set of tasks.\nšŸ”Ø ","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":" are time-bounded pushes to complete a set of tasks.\nšŸ”Ø ","href":null},{"type":"text","text":{"content":"Tasks","link":null},"annotations":{"bold":true,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Tasks","href":null},{"type":"text","text":{"content":" are the actions that make up epics.\nšŸž ","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":" are the actions that make up epics.\nšŸž ","href":null},{"type":"text","text":{"content":"Bugs","link":null},"annotations":{"bold":true,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Bugs","href":null},{"type":"text","text":{"content":" are tasks to fix things.\n\n","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":" are tasks to fix things.\n\n","href":null},{"type":"text","text":{"content":"ā†“","link":null},"annotations":{"bold":true,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"ā†“","href":null},{"type":"text","text":{"content":" Click ","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":" Click ","href":null},{"type":"text","text":{"content":"By Status","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":true,"color":"default"},"plain_text":"By Status","href":null},{"type":"text","text":{"content":" to isolate epics, sprints, tasks or bugs. Sort tasks by status, engineer or product manager. Switch to calendar view to see when work is scheduled to be completed.","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":" to isolate epics, sprints, tasks or bugs. Sort tasks by status, engineer or product manager. Switch to calendar view to see when work is scheduled to be completed.","href":null}],"is_inline":false,"properties":[{"name":"Engineers","value":{"id":"%24v1Q","name":"Engineers","type":"people","people":{}}},{"name":"Tasks","value":{"id":"6%3Dyp","name":"Tasks","type":"relation","relation":{"database_id":"a1298679-9f79-48a8-a991-834cd72eca17","type":"dual_property","dual_property":{"synced_property_name":"Epic","synced_property_id":"L%5BK%3C"}}}},{"name":"Type","value":{"id":"9dB%5E","name":"Type","type":"select","select":{"options":[{"id":"ca62f85e-a4ac-474f-b493-82d2df005dff","name":"Epic ā›°ļø","color":"green"},{"id":"3f806034-9c48-4519-871e-60c9c32d73d8","name":"Task šŸ”Ø","color":"yellow"},{"id":"1497e06a-abf3-4c81-a619-debfa0c70621","name":"Bug šŸž","color":"red"}]}}},{"name":"Sprint","value":{"id":"Jz.%40","name":"Sprint","type":"multi_select","multi_select":{"options":[{"id":"8d033c95-5515-4662-b8f3-60cb7d86487a","name":"Sprint 21","color":"default"},{"id":"bf3fcc55-aefc-43a8-82a0-2d4ac1e74d30","name":"Sprint 22","color":"default"},{"id":"7d78a5e4-28ef-4b21-8495-998fa6655014","name":"Sprint 23","color":"default"},{"id":"257e46d2-4a27-4298-b8c7-9b9bfb603bbd","name":"Sprint 20","color":"default"},{"id":"fbdb3f96-7979-4027-a461-aab8abda1ca8","name":"Sprint 24","color":"default"}]}}},{"name":"Epic","value":{"id":"L%5BK%3C","name":"Epic","type":"relation","relation":{"database_id":"a1298679-9f79-48a8-a991-834cd72eca17","type":"dual_property","dual_property":{"synced_property_name":"Tasks","synced_property_id":"6%3Dyp"}}}},{"name":"Timeline","value":{"id":"_G%2Bl","name":"Timeline","type":"date","date":{}}},{"name":"Created","value":{"id":"iwS0","name":"Created","type":"created_time","created_time":{}}},{"name":"Product Manager","value":{"id":"ma%3AW","name":"Product Manager","type":"people","people":{}}},{"name":"Priority","value":{"id":"%7BMEq","name":"Priority","type":"select","select":{"options":[{"id":"09fy","name":"P1 šŸ”„","color":"red"},{"id":"e1b2f058-4989-4dee-a873-4e88f58d4d0a","name":"P2","color":"orange"},{"id":"0bb46e0b-be4f-4b9c-87c2-b990868e9f92","name":"P3","color":"yellow"},{"id":"1a5512c5-39ad-4fb7-959f-68f596849eeb","name":"P4","color":"green"},{"id":"28c7cce9-7163-4407-9569-3f070da82ad1","name":"P5","color":"blue"}]}}},{"name":"Status","value":{"id":"%7CF4-","name":"Status","type":"select","select":{"options":[{"id":"c224a5a5-c284-431e-a65d-90a71712bcac","name":"Not Started","color":"red"},{"id":"ab7c2b08-ed87-4c04-b30f-fa62440f75d5","name":"In Progress","color":"yellow"},{"id":"c410e525-9a47-4ed2-9e72-299abee65575","name":"Complete šŸ™Œ","color":"green"}]}}},{"name":"Projects","value":{"id":"title","name":"Projects","type":"title","title":{}}}],"parent":{"type":"workspace","workspace":true},"url":"https://www.notion.so/a12986799f7948a8a991834cd72eca17","public_url":null,"archived":false},"emitted_at":1687166005743} \ No newline at end of file +{"stream": "databases", "data": {"object": "database", "id": "a1298679-9f79-48a8-a991-834cd72eca17", "cover": null, "icon": {"type": "emoji", "emoji": "\ud83d\ude98"}, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "ec324c09-af75-40f0-b91a-49ded74fdaf5"}, "last_edited_time": "2023-09-13T00:06:00.000Z", "title": [{"type": "text", "text": {"content": "Roadmap", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Roadmap", "href": null}], "description": [{"type": "text", "text": {"content": "Use this template to track all of your project work.\n\n\u26f0 ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Use this template to track all of your project work.\n\n\u26f0 ", "href": null}, {"type": "text", "text": {"content": "Epics", "link": null}, "annotations": {"bold": true, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Epics", "href": null}, {"type": "text", "text": {"content": " are large overarching initiatives.\n\ud83c\udfc3\u200d\u2642\ufe0f ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": " are large overarching initiatives.\n\ud83c\udfc3\u200d\u2642\ufe0f ", "href": null}, {"type": "text", "text": {"content": "Sprints", "link": null}, "annotations": {"bold": true, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Sprints", "href": null}, {"type": "text", "text": {"content": " are time-bounded pushes to complete a set of tasks.\n\ud83d\udd28 ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": " are time-bounded pushes to complete a set of tasks.\n\ud83d\udd28 ", "href": null}, {"type": "text", "text": {"content": "Tasks", "link": null}, "annotations": {"bold": true, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Tasks", "href": null}, {"type": "text", "text": {"content": " are the actions that make up epics.\n\ud83d\udc1e ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": " are the actions that make up epics.\n\ud83d\udc1e ", "href": null}, {"type": "text", "text": {"content": "Bugs", "link": null}, "annotations": {"bold": true, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Bugs", "href": null}, {"type": "text", "text": {"content": " are tasks to fix things.\n\n", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": " are tasks to fix things.\n\n", "href": null}, {"type": "text", "text": {"content": "\u2193", "link": null}, "annotations": {"bold": true, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "\u2193", "href": null}, {"type": "text", "text": {"content": " Click ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": " Click ", "href": null}, {"type": "text", "text": {"content": "By Status", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": true, "color": "default"}, "plain_text": "By Status", "href": null}, {"type": "text", "text": {"content": " to isolate epics, sprints, tasks or bugs. Sort tasks by status, engineer or product manager. Switch to calendar view to see when work is scheduled to be completed.", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": " to isolate epics, sprints, tasks or bugs. Sort tasks by status, engineer or product manager. Switch to calendar view to see when work is scheduled to be completed.", "href": null}], "is_inline": false, "properties": [{"name": "Engineers", "value": {"id": "%24v1Q", "name": "Engineers", "type": "people", "people": {}}}, {"name": "Tasks", "value": {"id": "6%3Dyp", "name": "Tasks", "type": "relation", "relation": {"database_id": "a1298679-9f79-48a8-a991-834cd72eca17", "type": "dual_property", "dual_property": {"synced_property_name": "Epic", "synced_property_id": "L%5BK%3C"}}}}, {"name": "Type", "value": {"id": "9dB%5E", "name": "Type", "type": "select", "select": {"options": [{"id": "ca62f85e-a4ac-474f-b493-82d2df005dff", "name": "Epic \u26f0\ufe0f", "color": "green"}, {"id": "3f806034-9c48-4519-871e-60c9c32d73d8", "name": "Task \ud83d\udd28", "color": "yellow"}, {"id": "1497e06a-abf3-4c81-a619-debfa0c70621", "name": "Bug \ud83d\udc1e", "color": "red"}]}}}, {"name": "Sprint", "value": {"id": "Jz.%40", "name": "Sprint", "type": "multi_select", "multi_select": {"options": [{"id": "8d033c95-5515-4662-b8f3-60cb7d86487a", "name": "Sprint 21", "color": "default"}, {"id": "bf3fcc55-aefc-43a8-82a0-2d4ac1e74d30", "name": "Sprint 22", "color": "default"}, {"id": "7d78a5e4-28ef-4b21-8495-998fa6655014", "name": "Sprint 23", "color": "default"}, {"id": "257e46d2-4a27-4298-b8c7-9b9bfb603bbd", "name": "Sprint 20", "color": "default"}, {"id": "fbdb3f96-7979-4027-a461-aab8abda1ca8", "name": "Sprint 24", "color": "default"}]}}}, {"name": "Epic", "value": {"id": "L%5BK%3C", "name": "Epic", "type": "relation", "relation": {"database_id": "a1298679-9f79-48a8-a991-834cd72eca17", "type": "dual_property", "dual_property": {"synced_property_name": "Tasks", "synced_property_id": "6%3Dyp"}}}}, {"name": "Timeline", "value": {"id": "_G%2Bl", "name": "Timeline", "type": "date", "date": {}}}, {"name": "Created", "value": {"id": "iwS0", "name": "Created", "type": "created_time", "created_time": {}}}, {"name": "Product Manager", "value": {"id": "ma%3AW", "name": "Product Manager", "type": "people", "people": {}}}, {"name": "Priority", "value": {"id": "%7BMEq", "name": "Priority", "type": "select", "select": {"options": [{"id": "09fy", "name": "P1 \ud83d\udd25", "color": "red"}, {"id": "e1b2f058-4989-4dee-a873-4e88f58d4d0a", "name": "P2", "color": "orange"}, {"id": "0bb46e0b-be4f-4b9c-87c2-b990868e9f92", "name": "P3", "color": "yellow"}, {"id": "1a5512c5-39ad-4fb7-959f-68f596849eeb", "name": "P4", "color": "green"}, {"id": "28c7cce9-7163-4407-9569-3f070da82ad1", "name": "P5", "color": "blue"}]}}}, {"name": "Status", "value": {"id": "%7CF4-", "name": "Status", "type": "select", "select": {"options": [{"id": "c224a5a5-c284-431e-a65d-90a71712bcac", "name": "Not Started", "color": "red"}, {"id": "ab7c2b08-ed87-4c04-b30f-fa62440f75d5", "name": "In Progress", "color": "yellow"}, {"id": "c410e525-9a47-4ed2-9e72-299abee65575", "name": "Complete \ud83d\ude4c", "color": "green"}]}}}, {"name": "Projects", "value": {"id": "title", "name": "Projects", "type": "title", "title": {}}}], "parent": {"type": "workspace", "workspace": true}, "url": "https://www.notion.so/a12986799f7948a8a991834cd72eca17", "public_url": null, "archived": false}, "emitted_at": 1696428596129} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-notion/metadata.yaml b/airbyte-integrations/connectors/source-notion/metadata.yaml index 3d34d88bbc641..95097647865b0 100644 --- a/airbyte-integrations/connectors/source-notion/metadata.yaml +++ b/airbyte-integrations/connectors/source-notion/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 6e00b415-b02e-4160-bf02-58176a0ae687 - dockerImageTag: 1.1.2 + dockerImageTag: 1.2.0 dockerRepository: airbyte/source-notion githubIssueLabel: source-notion icon: notion.svg diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json index 797a17611b39b..d28db5eb5def3 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json @@ -45,7 +45,13 @@ "pdf", "bookmark", "equation", - "unsupported" + "unsupported", + "table_of_contents", + "column_list", + "column", + "divider", + "link_to_page", + "quote" ] }, "paragraph": { "$ref": "text_element.json" }, diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/pages.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/pages.json index 14275b932acee..6152833eda6cf 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/pages.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/pages.json @@ -64,6 +64,23 @@ "rich_text": { "$ref": "rich_text.json" } } }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["rich_text"] + }, + "rich_text": { + "type": ["array"], + "items": { + "$ref": "rich_text.json" + } + } + } + }, { "type": "object", "properties": { diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/parent.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/parent.json index 5c17153c2a14e..e025d1b0028d8 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/parent.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/parent.json @@ -4,7 +4,7 @@ "additionalProperties": true, "properties": { "type": { - "enum": ["database_id", "page_id", "workspace"] + "enum": ["database_id", "page_id", "workspace", "block_id"] }, "database_id": { "type": "string" @@ -12,6 +12,9 @@ "page_id": { "type": "string" }, + "block_id": { + "type": "string" + }, "workspace": { "type": "boolean" } diff --git a/docs/integrations/sources/notion.md b/docs/integrations/sources/notion.md index 1cb3e43f6af6d..3cfd1be784409 100644 --- a/docs/integrations/sources/notion.md +++ b/docs/integrations/sources/notion.md @@ -106,8 +106,9 @@ The connector is restricted by Notion [request limits](https://developers.notion ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------- | -| 1.1.2 | 2023-08-30 | [29999](https://github.com/airbytehq/airbyte/pull/29999) | Update error handling during connection check +|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------| +| 1.2.0 | 2023-10-04 | [31053](https://github.com/airbytehq/airbyte/pull/31053) | Add undeclared fields for blocks and pages streams | +| 1.1.2 | 2023-08-30 | [29999](https://github.com/airbytehq/airbyte/pull/29999) | Update error handling during connection check | | 1.1.1 | 2023-06-14 | [26535](https://github.com/airbytehq/airbyte/pull/26535) | Migrate from deprecated `authSpecification` to `advancedAuth` | | 1.1.0 | 2023-06-08 | [27170](https://github.com/airbytehq/airbyte/pull/27170) | Fix typo in `blocks` schema | | 1.0.9 | 2023-06-08 | [27062](https://github.com/airbytehq/airbyte/pull/27062) | Skip streams with `invalid_start_cursor` error |