Merge pull request #4690 from kishanprmr/notion-trigger

fix(notion): Fix Issue with Skipped Items within the Same Minute Interval

packages/pieces/community/notion/package.json

@@ -1,4 +1,4 @@
 {
   "name": "@activepieces/piece-notion",
-  "version": "0.2.13"
+  "version": "0.2.14"
 }

packages/pieces/community/notion/src/index.ts

@@ -1,9 +1,9 @@
 import { createCustomApiCallAction } from '@activepieces/pieces-common';
 import {
-	OAuth2AuthorizationMethod,
-	OAuth2PropertyValue,
-	PieceAuth,
-	createPiece,
+  OAuth2AuthorizationMethod,
+  OAuth2PropertyValue,
+  PieceAuth,
+  createPiece,
 } from '@activepieces/pieces-framework';
 import { PieceCategory } from '@activepieces/shared';
 import { appendToPage } from './lib/action/append-to-page';
@@ -15,37 +15,43 @@ import { updatedDatabaseItem } from './lib/triggers/updated-database-item';
 import { findDatabaseItem } from './lib/action/find-item';
 
 export const notionAuth = PieceAuth.OAuth2({
-	authUrl: 'https://api.notion.com/v1/oauth/authorize',
-	tokenUrl: 'https://api.notion.com/v1/oauth/token',
-	scope: [],
-	extra: {
-		owner: 'user',
-	},
-	authorizationMethod: OAuth2AuthorizationMethod.HEADER,
-	required: true,
+  authUrl: 'https://api.notion.com/v1/oauth/authorize',
+  tokenUrl: 'https://api.notion.com/v1/oauth/token',
+  scope: [],
+  extra: {
+    owner: 'user',
+  },
+  authorizationMethod: OAuth2AuthorizationMethod.HEADER,
+  required: true,
 });
 
 export const notion = createPiece({
-	displayName: 'Notion',
-	description: 'The all-in-one workspace',
-	logoUrl: 'https://cdn.activepieces.com/pieces/notion.png',
-	categories: [PieceCategory.PRODUCTIVITY],
-	minimumSupportedRelease: '0.5.0',
-	authors: ['ShayPunter', 'kishanprmr', 'MoShizzle', 'khaledmashaly', 'abuaboud'],
-	auth: notionAuth,
-	actions: [
-		createDatabaseItem,
-		updateDatabaseItem,
-		findDatabaseItem,
-		createPage,
-		appendToPage,
-		createCustomApiCallAction({
-			baseUrl: () => 'https://api.notion.com/v1',
-			auth: notionAuth,
-			authMapping: (auth) => ({
-				Authorization: `Bearer ${(auth as OAuth2PropertyValue).access_token}`,
-			}),
-		}),
-	],
-	triggers: [newDatabaseItem, updatedDatabaseItem],
+  displayName: 'Notion',
+  description: 'The all-in-one workspace',
+  logoUrl: 'https://cdn.activepieces.com/pieces/notion.png',
+  categories: [PieceCategory.PRODUCTIVITY],
+  minimumSupportedRelease: '0.5.0',
+  authors: [
+    'ShayPunter',
+    'kishanprmr',
+    'MoShizzle',
+    'khaledmashaly',
+    'abuaboud',
+  ],
+  auth: notionAuth,
+  actions: [
+    createDatabaseItem,
+    updateDatabaseItem,
+    findDatabaseItem,
+    createPage,
+    appendToPage,
+    createCustomApiCallAction({
+      baseUrl: () => 'https://api.notion.com/v1',
+      auth: notionAuth,
+      authMapping: (auth) => ({
+        Authorization: `Bearer ${(auth as OAuth2PropertyValue).access_token}`,
+      }),
+    }),
+  ],
+  triggers: [newDatabaseItem, updatedDatabaseItem],
 });

packages/pieces/community/notion/src/lib/triggers/new-database-item.ts

@@ -120,17 +120,27 @@ const polling: Polling<
   OAuth2PropertyValue,
   { database_id: string | undefined }
 > = {
-  strategy: DedupeStrategy.TIMEBASED,
-  items: async ({ auth, propsValue, lastFetchEpochMS }) => {
+  strategy: DedupeStrategy.LAST_ITEM,
+  items: async ({ auth, propsValue, lastItemId }) => {
+    const lastItem = lastItemId as string;
+    let lastCreateddDate: string | null;
+
+    if (lastItem) {
+      const lastUpdatedEpochMS = Number(lastItem.split('|')[1]);
+      lastCreateddDate = dayjs(lastUpdatedEpochMS).toISOString();
+    } else {
+      lastCreateddDate = lastItem;
+    }
+
     const items = await getResponse(
       auth,
       propsValue.database_id!,
-      lastFetchEpochMS === 0 ? null : dayjs(lastFetchEpochMS).toISOString()
+      lastCreateddDate
     );
-    return items.results.map((item) => {
-      const object = item as { created_time: string };
+    return items.map((item: any) => {
+      const object = item as { created_time: string; id: string };
       return {
-        epochMilliSeconds: dayjs(object.created_time).valueOf(),
+        id: object.id + '|' + dayjs(object.created_time).valueOf(),
         data: item,
       };
     });
@@ -146,22 +156,36 @@ const getResponse = async (
     auth: authentication.access_token,
     notionVersion: '2022-02-22',
   });
-  return notion.databases.query({
-    database_id,
-    filter:
-      startDate == null
-        ? undefined
-        : {
-            timestamp: 'created_time',
-            created_time: {
-              after: startDate,
+  let cursor;
+  let hasMore = true;
+  const results = [];
+
+  do {
+    const response = await notion.databases.query({
+      start_cursor: cursor,
+      database_id,
+      filter:
+        startDate == null
+          ? undefined
+          : {
+              timestamp: 'created_time',
+              created_time: {
+                on_or_after: startDate,
+              },
             },
-          },
-    sorts: [
-      {
-        timestamp: 'created_time',
-        direction: startDate == null ? 'descending' : 'ascending',
-      },
-    ],
-  });
+      sorts: [
+        {
+          timestamp: 'created_time',
+          direction: 'descending',
+        },
+      ],
+    });
+
+    hasMore = response.has_more;
+    cursor = response.next_cursor ?? undefined;
+
+    results.push(...response.results);
+  } while (hasMore);
+
+  return results;
 };

packages/pieces/community/notion/src/lib/triggers/updated-database-item.ts

@@ -120,17 +120,28 @@ const polling: Polling<
   OAuth2PropertyValue,
   { database_id: string | undefined }
 > = {
-  strategy: DedupeStrategy.TIMEBASED,
-  items: async ({ auth, propsValue, lastFetchEpochMS }) => {
+  strategy: DedupeStrategy.LAST_ITEM,
+  items: async ({ auth, propsValue, lastItemId }) => {
+    const lastItem = lastItemId as string;
+    let lastUpdatedDate: string | null;
+
+    if (lastItem) {
+      const lastUpdatedEpochMS = Number(lastItem.split('|')[1]);
+      lastUpdatedDate = dayjs(lastUpdatedEpochMS).toISOString();
+    } else {
+      lastUpdatedDate = lastItem;
+    }
+
     const items = await getResponse(
       auth,
       propsValue.database_id!,
-      lastFetchEpochMS === 0 ? null : dayjs(lastFetchEpochMS).toISOString()
+      lastUpdatedDate
     );
-    return items.results.map((item) => {
-      const object = item as { last_edited_time: string };
+
+    return items.map((item: any) => {
+      const object = item as { last_edited_time: string; id: string };
       return {
-        epochMilliSeconds: dayjs(object.last_edited_time).valueOf(),
+        id: object.id + '|' + dayjs(object.last_edited_time).valueOf(),
         data: item,
       };
     });
@@ -146,22 +157,36 @@ const getResponse = async (
     auth: authentication.access_token,
     notionVersion: '2022-02-22',
   });
-  return notion.databases.query({
-    database_id,
-    filter:
-      startDate == null
-        ? undefined
-        : {
-            timestamp: 'last_edited_time',
-            last_edited_time: {
-              after: startDate,
+
+  let cursor;
+  let hasMore = true;
+  const results = [];
+  do {
+    const response = await notion.databases.query({
+      start_cursor: cursor,
+      database_id,
+      filter:
+        startDate == null
+          ? undefined
+          : {
+              timestamp: 'last_edited_time',
+              last_edited_time: {
+                on_or_after: startDate,
+              },
             },
-          },
-    sorts: [
-      {
-        timestamp: 'last_edited_time',
-        direction: startDate == null ? 'descending' : 'ascending',
-      },
-    ],
-  });
+      sorts: [
+        {
+          timestamp: 'last_edited_time',
+          direction: 'descending',
+        },
+      ],
+    });
+
+    hasMore = response.has_more;
+    cursor = response.next_cursor ?? undefined;
+
+    results.push(...response.results);
+  } while (hasMore);
+
+  return results;
 };