From 927fd3008b38125366fc9146b0a5fa3d3a5509d8 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Fri, 27 Sep 2024 12:30:41 -0300 Subject: [PATCH 1/8] emit changes for previous and current values --- .../sources/updated-page/updated-page.mjs | 85 +++++++++++-------- 1 file changed, 50 insertions(+), 35 deletions(-) diff --git a/components/notion/sources/updated-page/updated-page.mjs b/components/notion/sources/updated-page/updated-page.mjs index 88b2118d9e44d..304570a62be8c 100644 --- a/components/notion/sources/updated-page/updated-page.mjs +++ b/components/notion/sources/updated-page/updated-page.mjs @@ -2,7 +2,6 @@ import notion from "../../notion.app.mjs"; import sampleEmit from "./test-event.mjs"; import base from "../common/base.mjs"; import constants from "../common/constants.mjs"; -import md5 from "md5"; export default { ...base, @@ -41,26 +40,27 @@ export default { }, hooks: { async deploy() { - const properties = await this.getProperties(); + const propertiesToCheck = await this.getPropertiesToCheck(); const propertyValues = {}; const params = this.lastUpdatedSortParam(); const pagesStream = this.notion.getPages(this.databaseId, params); let count = 0; let lastUpdatedTimestamp = 0; for await (const page of pagesStream) { - propertyValues[page.id] = {}; - for (const propertyName of properties) { - const hash = this.calculateHash(page.properties[propertyName]); - propertyValues[page.id][propertyName] = hash; + for (const propertyName of propertiesToCheck) { + const currentValue = this.maybeRemoveFileSubItems(page.properties[propertyName]); + propertyValues[page.id] = { + ...propertyValues[page.id], + [propertyName]: currentValue, + }; } lastUpdatedTimestamp = Math.max( lastUpdatedTimestamp, - Date.parse(page?.last_edited_time), + Date.parse(page.last_edited_time), ); - if (count < 25) { - this.emitEvent(page); + if (count++ < 25) { + this.emitEvent(page, []); } - count++; } this._setPropertyValues(propertyValues); this.setLastUpdatedTimestamp(lastUpdatedTimestamp); @@ -74,18 +74,13 @@ export default { _setPropertyValues(propertyValues) { this.db.set("propertyValues", propertyValues); }, - async getProperties() { + async getPropertiesToCheck() { if (this.properties?.length) { return this.properties; } const { properties } = await this.notion.retrieveDatabase(this.databaseId); return Object.keys(properties); }, - calculateHash(property) { - const clone = structuredClone(property); - this.maybeRemoveFileSubItems(clone); - return md5(JSON.stringify(clone)); - }, maybeRemoveFileSubItems(property) { // Files & Media type: // `url` and `expiry_time` are constantly updated by Notion, so ignore these fields @@ -96,6 +91,7 @@ export default { } } } + return property; }, generateMeta(obj, summary) { const { id } = obj; @@ -107,9 +103,13 @@ export default { ts, }; }, - emitEvent(page) { + emitEvent(page, changes) { const meta = this.generateMeta(page, constants.summaries.PAGE_UPDATED); - this.$emit(page, meta); + const event = { + page, + changes, + }; + this.$emit(event, meta); }, }, async run() { @@ -126,38 +126,53 @@ export default { }, }; let newLastUpdatedTimestamp = lastCheckedTimestamp; - const properties = await this.getProperties(); + const propertiesToCheck = await this.getPropertiesToCheck(); const pagesStream = this.notion.getPages(this.databaseId, params); for await (const page of pagesStream) { + const changes = []; + let propertyHasChanged = false; newLastUpdatedTimestamp = Math.max( newLastUpdatedTimestamp, - Date.parse(page?.last_edited_time), + Date.parse(page.last_edited_time), ); - let propertyChangeFound = false; - for (const propertyName of properties) { - const hash = this.calculateHash(page.properties[propertyName]); - const dbValue = propertyValues[page.id]?.[propertyName]; - if (!propertyValues[page.id] || hash !== dbValue) { - propertyChangeFound = true; + for (const propertyName of propertiesToCheck) { + const previousValue = structuredClone(propertyValues[page.id]?.[propertyName]); + const currentValue = this.maybeRemoveFileSubItems(page.properties[propertyName]); + + const pageExistsInDB = propertyValues[page.id] != null; + const propertyChanged = JSON.stringify(previousValue) !== JSON.stringify(currentValue); + + if (pageExistsInDB && propertyChanged) { + propertyHasChanged = true; propertyValues[page.id] = { ...propertyValues[page.id], - [propertyName]: hash, + [propertyName]: currentValue, }; + changes.push({ + previousValue, + currentValue, + }); } - } - if (!propertyChangeFound && Date.parse(page?.last_edited_time) <= lastCheckedTimestamp) { - continue; - } - if (!this.includeNewPages && page?.last_edited_time === page?.created_time) { - continue; + if (!pageExistsInDB && this.includeNewPages) { + propertyHasChanged = true; + propertyValues[page.id] = { + [propertyName]: currentValue, + }; + changes.push({ + previousValue, + currentValue, + }); + } } - this.emitEvent(page); + if (propertyHasChanged && lastCheckedTimestamp <= Date.parse(page.last_edited_time)) { + this.emitEvent(page, changes); + } - if (Date.parse(page?.last_edited_time) < lastCheckedTimestamp) { + if (Date.parse(page.last_edited_time) < lastCheckedTimestamp) { break; } } From 808eb3de5c7bf8290834360a900ef0c51fad7b81 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Fri, 27 Sep 2024 12:57:51 -0300 Subject: [PATCH 2/8] reorder props --- .../notion/sources/updated-page/updated-page.mjs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/components/notion/sources/updated-page/updated-page.mjs b/components/notion/sources/updated-page/updated-page.mjs index 304570a62be8c..dd26e342d2032 100644 --- a/components/notion/sources/updated-page/updated-page.mjs +++ b/components/notion/sources/updated-page/updated-page.mjs @@ -19,6 +19,12 @@ export default { "databaseId", ], }, + includeNewPages: { + type: "boolean", + label: "Include New Pages", + description: "Set to `true` to emit events when pages are created. Set to `false` to ignore new pages.", + default: true, + }, properties: { propDefinition: [ notion, @@ -31,12 +37,6 @@ export default { description: "Only emit events when one or more of the selected properties have changed", optional: true, }, - includeNewPages: { - type: "boolean", - label: "Include New Pages", - description: "Set to `true` to emit events when pages are created. Set to `false` to ignore new pages.", - default: true, - }, }, hooks: { async deploy() { From a13dd2544549477081138d9ec3f8aefa0ed3cd56 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Fri, 27 Sep 2024 12:58:10 -0300 Subject: [PATCH 3/8] use base64 compression/decompression --- .../notion/sources/updated-page/updated-page.mjs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/components/notion/sources/updated-page/updated-page.mjs b/components/notion/sources/updated-page/updated-page.mjs index dd26e342d2032..7edd0204d0a5e 100644 --- a/components/notion/sources/updated-page/updated-page.mjs +++ b/components/notion/sources/updated-page/updated-page.mjs @@ -2,6 +2,7 @@ import notion from "../../notion.app.mjs"; import sampleEmit from "./test-event.mjs"; import base from "../common/base.mjs"; import constants from "../common/constants.mjs"; +import zlib from "zlib"; export default { ...base, @@ -69,10 +70,15 @@ export default { methods: { ...base.methods, _getPropertyValues() { - return this.db.get("propertyValues"); + const compressed = this.db.get("propertyValues"); + const buffer = Buffer.from(compressed, "base64"); + const decompressed = zlib.inflateSync(buffer).toString(); + return JSON.parse(decompressed); }, _setPropertyValues(propertyValues) { - this.db.set("propertyValues", propertyValues); + const string = JSON.stringify(propertyValues); + const compressed = zlib.deflateSync(string).toString("base64"); + this.db.set("propertyValues", compressed); }, async getPropertiesToCheck() { if (this.properties?.length) { From 66cd35311d299555411187e32bfa1b700aaa523a Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Fri, 27 Sep 2024 13:15:51 -0300 Subject: [PATCH 4/8] change summary for new vs updated pages --- .../sources/updated-page/updated-page.mjs | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/components/notion/sources/updated-page/updated-page.mjs b/components/notion/sources/updated-page/updated-page.mjs index 7edd0204d0a5e..c87b9b1814b9f 100644 --- a/components/notion/sources/updated-page/updated-page.mjs +++ b/components/notion/sources/updated-page/updated-page.mjs @@ -60,7 +60,7 @@ export default { Date.parse(page.last_edited_time), ); if (count++ < 25) { - this.emitEvent(page, []); + this.emitEvent(page, [], true); } } this._setPropertyValues(propertyValues); @@ -109,8 +109,10 @@ export default { ts, }; }, - emitEvent(page, changes) { - const meta = this.generateMeta(page, constants.summaries.PAGE_UPDATED); + emitEvent(page, changes, isNewPage) { + const meta = isNewPage + ? this.generateMeta(page, constants.summaries.PAGE_ADDED) + : this.generateMeta(page, constants.summaries.PAGE_UPDATED); const event = { page, changes, @@ -137,6 +139,7 @@ export default { for await (const page of pagesStream) { const changes = []; + let isNewPage = false; let propertyHasChanged = false; newLastUpdatedTimestamp = Math.max( newLastUpdatedTimestamp, @@ -144,6 +147,10 @@ export default { ); for (const propertyName of propertiesToCheck) { + if (lastCheckedTimestamp > Date.parse(page.last_edited_time)) { + break; + } + const previousValue = structuredClone(propertyValues[page.id]?.[propertyName]); const currentValue = this.maybeRemoveFileSubItems(page.properties[propertyName]); @@ -163,6 +170,7 @@ export default { } if (!pageExistsInDB && this.includeNewPages) { + isNewPage = true; propertyHasChanged = true; propertyValues[page.id] = { [propertyName]: currentValue, @@ -174,12 +182,8 @@ export default { } } - if (propertyHasChanged && lastCheckedTimestamp <= Date.parse(page.last_edited_time)) { - this.emitEvent(page, changes); - } - - if (Date.parse(page.last_edited_time) < lastCheckedTimestamp) { - break; + if (propertyHasChanged) { + this.emitEvent(page, changes, isNewPage); } } From cb0c5fddaac756a3d80284f99049f94dd7ca2f64 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Fri, 27 Sep 2024 14:37:01 -0300 Subject: [PATCH 5/8] move break to outside loop --- components/notion/sources/updated-page/updated-page.mjs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/components/notion/sources/updated-page/updated-page.mjs b/components/notion/sources/updated-page/updated-page.mjs index c87b9b1814b9f..abda8f825b0f1 100644 --- a/components/notion/sources/updated-page/updated-page.mjs +++ b/components/notion/sources/updated-page/updated-page.mjs @@ -141,16 +141,17 @@ export default { const changes = []; let isNewPage = false; let propertyHasChanged = false; + newLastUpdatedTimestamp = Math.max( newLastUpdatedTimestamp, Date.parse(page.last_edited_time), ); - for (const propertyName of propertiesToCheck) { - if (lastCheckedTimestamp > Date.parse(page.last_edited_time)) { - break; - } + if (lastCheckedTimestamp > Date.parse(page.last_edited_time)) { + break; + } + for (const propertyName of propertiesToCheck) { const previousValue = structuredClone(propertyValues[page.id]?.[propertyName]); const currentValue = this.maybeRemoveFileSubItems(page.properties[propertyName]); From fb520a8dbfec6e89d0ef49ac55f7f291f855f3bc Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Fri, 27 Sep 2024 14:46:55 -0300 Subject: [PATCH 6/8] remove page id from event summary --- components/notion/sources/updated-page/updated-page.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/notion/sources/updated-page/updated-page.mjs b/components/notion/sources/updated-page/updated-page.mjs index abda8f825b0f1..7bc73bbafacad 100644 --- a/components/notion/sources/updated-page/updated-page.mjs +++ b/components/notion/sources/updated-page/updated-page.mjs @@ -105,7 +105,7 @@ export default { const ts = Date.now(); return { id: `${id}-${ts}`, - summary: `${summary}: ${title} - ${id}`, + summary: `${summary}: ${title}`, ts, }; }, From 73bceed4b430f230ad2b6e7747434ad8095f5995 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Fri, 27 Sep 2024 14:56:49 -0300 Subject: [PATCH 7/8] add property name to changes --- components/notion/sources/updated-page/updated-page.mjs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/notion/sources/updated-page/updated-page.mjs b/components/notion/sources/updated-page/updated-page.mjs index 7bc73bbafacad..51874695c5be3 100644 --- a/components/notion/sources/updated-page/updated-page.mjs +++ b/components/notion/sources/updated-page/updated-page.mjs @@ -60,7 +60,7 @@ export default { Date.parse(page.last_edited_time), ); if (count++ < 25) { - this.emitEvent(page, [], true); + this.emitEvent(page); } } this._setPropertyValues(propertyValues); @@ -109,7 +109,7 @@ export default { ts, }; }, - emitEvent(page, changes, isNewPage) { + emitEvent(page, changes = [], isNewPage = true) { const meta = isNewPage ? this.generateMeta(page, constants.summaries.PAGE_ADDED) : this.generateMeta(page, constants.summaries.PAGE_UPDATED); @@ -165,6 +165,7 @@ export default { [propertyName]: currentValue, }; changes.push({ + property: propertyName, previousValue, currentValue, }); @@ -177,6 +178,7 @@ export default { [propertyName]: currentValue, }; changes.push({ + property: propertyName, previousValue, currentValue, }); From 6a43f23d425036801ec440dd1f727ff570c7769f Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Fri, 27 Sep 2024 14:57:28 -0300 Subject: [PATCH 8/8] bump versions --- components/notion/package.json | 2 +- components/notion/sources/updated-page/updated-page.mjs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/notion/package.json b/components/notion/package.json index a070594d91183..cd9c3da3601e8 100644 --- a/components/notion/package.json +++ b/components/notion/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/notion", - "version": "0.1.24", + "version": "0.2.0", "description": "Pipedream Notion Components", "main": "notion.app.mjs", "keywords": [ diff --git a/components/notion/sources/updated-page/updated-page.mjs b/components/notion/sources/updated-page/updated-page.mjs index 51874695c5be3..8bbb5d423ddbb 100644 --- a/components/notion/sources/updated-page/updated-page.mjs +++ b/components/notion/sources/updated-page/updated-page.mjs @@ -9,7 +9,7 @@ export default { key: "notion-updated-page", name: "Updated Page in Database", /* eslint-disable-line pipedream/source-name */ description: "Emit new event when a page in a database is updated. To select a specific page, use `Updated Page ID` instead", - version: "0.0.19", + version: "0.1.0", type: "source", dedupe: "unique", props: {