From 0f0e353000ccd6948a6abe1bd0db61cfd115a22e Mon Sep 17 00:00:00 2001 From: Jorge Cortes Date: Tue, 18 Nov 2025 16:29:16 -0500 Subject: [PATCH] [ACTION] Google Drive - Polling-based components as an alternative to webhook-based ones --- components/google_drive/package.json | 2 +- .../new-files-instant-polling.mjs | 190 +++++++++++++++ .../new-files-instant-polling/test-event.mjs | 99 ++++++++ .../new-or-modified-comments-polling.mjs | 112 +++++++++ .../test-event.mjs | 35 +++ .../new-or-modified-folders-polling.mjs | 224 ++++++++++++++++++ .../test-event.mjs | 51 ++++ .../new-spreadsheet-polling.mjs | 188 +++++++++++++++ .../new-spreadsheet-polling/test-event.mjs | 111 +++++++++ 9 files changed, 1011 insertions(+), 1 deletion(-) create mode 100644 components/google_drive/sources/new-files-instant-polling/new-files-instant-polling.mjs create mode 100644 components/google_drive/sources/new-files-instant-polling/test-event.mjs create mode 100644 components/google_drive/sources/new-or-modified-comments-polling/new-or-modified-comments-polling.mjs create mode 100644 components/google_drive/sources/new-or-modified-comments-polling/test-event.mjs create mode 100644 components/google_drive/sources/new-or-modified-folders-polling/new-or-modified-folders-polling.mjs create mode 100644 components/google_drive/sources/new-or-modified-folders-polling/test-event.mjs create mode 100644 components/google_drive/sources/new-spreadsheet-polling/new-spreadsheet-polling.mjs create mode 100644 components/google_drive/sources/new-spreadsheet-polling/test-event.mjs diff --git a/components/google_drive/package.json b/components/google_drive/package.json index ef2890c90152c..a99f5ba9ace2c 100644 --- a/components/google_drive/package.json +++ b/components/google_drive/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/google_drive", - "version": "1.3.1", + "version": "1.4.0", "description": "Pipedream Google_drive Components", "main": "google_drive.app.mjs", "keywords": [ diff --git a/components/google_drive/sources/new-files-instant-polling/new-files-instant-polling.mjs b/components/google_drive/sources/new-files-instant-polling/new-files-instant-polling.mjs new file mode 100644 index 0000000000000..adce569428d1a --- /dev/null +++ b/components/google_drive/sources/new-files-instant-polling/new-files-instant-polling.mjs @@ -0,0 +1,190 @@ +import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; +import googleDrive from "../../google_drive.app.mjs"; +import { getListFilesOpts } from "../../common/utils.mjs"; +import { GOOGLE_DRIVE_FOLDER_MIME_TYPE } from "../../common/constants.mjs"; +import sampleEmit from "./test-event.mjs"; + +export default { + key: "google_drive-new-files-instant-polling", + name: "New Files (Polling)", + description: "Emit new event when a new file is added in your linked Google Drive", + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + googleDrive, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + drive: { + propDefinition: [ + googleDrive, + "watchedDrive", + ], + description: "Defaults to My Drive. To select a [Shared Drive](https://support.google.com/a/users/answer/9310351) instead, select it from this list.", + optional: false, + }, + folders: { + propDefinition: [ + googleDrive, + "folderId", + ({ drive }) => ({ + drive, + baseOpts: { + q: `mimeType = '${GOOGLE_DRIVE_FOLDER_MIME_TYPE}' and trashed = false`, + }, + }), + ], + type: "string[]", + label: "Folders", + description: "The specific folder(s) to watch for new files. Leave blank to watch all files in the Drive.", + optional: true, + default: [], + }, + }, + hooks: { + async deploy() { + // Get initial page token for change tracking + const driveId = this.getDriveId(); + const startPageToken = await this.googleDrive.getPageToken(driveId); + this._setPageToken(startPageToken); + + this._setLastRunTimestamp(Date.now()); + + // Emit sample files from the last 30 days + const daysAgo = new Date(); + daysAgo.setDate(daysAgo.getDate() - 30); + const timeString = daysAgo.toISOString(); + + const args = this.getListFilesOpts({ + q: `mimeType != "application/vnd.google-apps.folder" and createdTime > "${timeString}" and trashed = false`, + orderBy: "createdTime desc", + fields: "*", + pageSize: 5, + }); + + const { files } = await this.googleDrive.listFilesInPage(null, args); + + for (const file of files) { + if (this.shouldProcess(file)) { + await this.emitFile(file); + } + } + }, + }, + methods: { + _getPageToken() { + return this.db.get("pageToken"); + }, + _setPageToken(pageToken) { + this.db.set("pageToken", pageToken); + }, + _getLastRunTimestamp() { + return this.db.get("lastRunTimestamp"); + }, + _setLastRunTimestamp(timestamp) { + this.db.set("lastRunTimestamp", timestamp); + }, + getDriveId(drive = this.drive) { + return this.googleDrive.getDriveId(drive); + }, + getListFilesOpts(args = {}) { + return getListFilesOpts(this.drive, { + q: "mimeType != 'application/vnd.google-apps.folder' and trashed = false", + ...args, + }); + }, + shouldProcess(file) { + // Skip folders + if (file.mimeType === GOOGLE_DRIVE_FOLDER_MIME_TYPE) { + return false; + } + + // Check if specific folders are being watched + if (this.folders?.length > 0) { + const watchedFolders = new Set(this.folders); + if (!file.parents || !file.parents.some((p) => watchedFolders.has(p))) { + return false; + } + } + + return true; + }, + generateMeta(file) { + const { + id: fileId, + name: summary, + createdTime: tsString, + } = file; + const ts = Date.parse(tsString); + + return { + id: `${fileId}-${ts}`, + summary, + ts, + }; + }, + async emitFile(file) { + const meta = this.generateMeta(file); + this.$emit(file, meta); + }, + }, + async run() { + const currentRunTimestamp = Date.now(); + const lastRunTimestamp = this._getLastRunTimestamp(); + + const pageToken = this._getPageToken(); + const driveId = this.getDriveId(); + + const changedFilesStream = this.googleDrive.listChanges(pageToken, driveId); + + for await (const changedFilesPage of changedFilesStream) { + console.log("Changed files page:", changedFilesPage); + const { + changedFiles, + nextPageToken, + } = changedFilesPage; + + console.log(changedFiles.length + ? `Processing ${changedFiles.length} changed files` + : "No changed files since last run"); + + for (const file of changedFiles) { + // Skip folders + if (file.mimeType === GOOGLE_DRIVE_FOLDER_MIME_TYPE) { + continue; + } + + // Get full file metadata including parents + const fullFile = await this.googleDrive.getFile(file.id, { + fields: "*", + }); + + // Check if it's a new file (created after last run) + const fileCreatedTime = Date.parse(fullFile.createdTime); + if (fileCreatedTime <= lastRunTimestamp) { + console.log(`Skipping existing file ${fullFile.name || fullFile.id}`); + continue; + } + + if (!this.shouldProcess(fullFile)) { + console.log(`Skipping file ${fullFile.name || fullFile.id}`); + continue; + } + + await this.emitFile(fullFile); + } + + // Save the next page token after successfully processing + this._setPageToken(nextPageToken); + } + + // Update the last run timestamp after processing all changes + this._setLastRunTimestamp(currentRunTimestamp); + }, + sampleEmit, +}; diff --git a/components/google_drive/sources/new-files-instant-polling/test-event.mjs b/components/google_drive/sources/new-files-instant-polling/test-event.mjs new file mode 100644 index 0000000000000..f4d7ebc492ba5 --- /dev/null +++ b/components/google_drive/sources/new-files-instant-polling/test-event.mjs @@ -0,0 +1,99 @@ +export default { + "kind": "drive#file", + "copyRequiresWriterPermission": false, + "writersCanShare": true, + "viewedByMe": true, + "mimeType": "text/plain", + "parents": [ + "0ANs73yKKVA" + ], + "iconLink": "https://drive-thirdparty.googleusercontent.com/16/type/text/plain", + "shared": false, + "lastModifyingUser": { + "displayName": "John Doe", + "kind": "drive#user", + "me": true, + "permissionId": "077423361841532483", + "emailAddress": "john@doe.com", + "photoLink": "https://lh3.googleusercontent.com/a/default-user=s64" + }, + "owners": [ + { + "displayName": "John Doe", + "kind": "drive#user", + "me": true, + "permissionId": "07742336189483", + "emailAddress": "john@doe.com", + "photoLink": "https://lh3.googleusercontent.com/a/default-user=s64" + } + ], + "webViewLink": "https://drive.google.com/file/d/1LmXTIQ0wqKP7T3-l9r127Maw4Pt2BViTOo/view?usp=drivesdk", + "size": "1024", + "viewersCanCopyContent": true, + "permissions": [ + { + "id": "07742336184153259483", + "displayName": "John Doe", + "type": "user", + "kind": "drive#permission", + "photoLink": "https://lh3.googleusercontent.com/a/default-user=s64", + "emailAddress": "john@doe.com", + "role": "owner", + "deleted": false, + "pendingOwner": false + } + ], + "spaces": [ + "drive" + ], + "id": "1LmXTIQ0wqKP7T3-l9r127Maw4Pt2BViTOo", + "name": "New File.txt", + "starred": false, + "trashed": false, + "explicitlyTrashed": false, + "createdTime": "2023-06-28T12:52:54.570Z", + "modifiedTime": "2023-06-28T12:52:54.570Z", + "modifiedByMeTime": "2023-06-28T12:52:54.570Z", + "viewedByMeTime": "2023-06-29T06:30:58.441Z", + "quotaBytesUsed": "1024", + "version": "1", + "ownedByMe": true, + "isAppAuthorized": false, + "capabilities": { + "canChangeViewersCanCopyContent": true, + "canEdit": true, + "canCopy": true, + "canComment": true, + "canAddChildren": false, + "canDelete": true, + "canDownload": true, + "canListChildren": false, + "canRemoveChildren": false, + "canRename": true, + "canTrash": true, + "canReadRevisions": true, + "canChangeCopyRequiresWriterPermission": true, + "canMoveItemIntoTeamDrive": true, + "canUntrash": true, + "canModifyContent": true, + "canMoveItemOutOfDrive": true, + "canAddMyDriveParent": false, + "canRemoveMyDriveParent": true, + "canMoveItemWithinDrive": true, + "canShare": true, + "canMoveChildrenWithinDrive": false, + "canModifyContentRestriction": true, + "canChangeSecurityUpdateEnabled": false, + "canAcceptOwnership": false, + "canReadLabels": true, + "canModifyLabels": true + }, + "modifiedByMe": true, + "permissionIds": [ + "07742336184153259483" + ], + "linkShareMetadata": { + "securityUpdateEligible": false, + "securityUpdateEnabled": true + } +}; diff --git a/components/google_drive/sources/new-or-modified-comments-polling/new-or-modified-comments-polling.mjs b/components/google_drive/sources/new-or-modified-comments-polling/new-or-modified-comments-polling.mjs new file mode 100644 index 0000000000000..056b552761f48 --- /dev/null +++ b/components/google_drive/sources/new-or-modified-comments-polling/new-or-modified-comments-polling.mjs @@ -0,0 +1,112 @@ +import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; +import googleDrive from "../../google_drive.app.mjs"; +import sampleEmit from "./test-event.mjs"; + +export default { + key: "google_drive-new-or-modified-comments-polling", + name: "New or Modified Comments (Polling)", + description: "Emit new event when a comment is created or modified in the selected file", + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + googleDrive, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + drive: { + propDefinition: [ + googleDrive, + "watchedDrive", + ], + description: "Defaults to My Drive. To select a [Shared Drive](https://support.google.com/a/users/answer/9310351) instead, select it from this list.", + optional: false, + }, + fileId: { + propDefinition: [ + googleDrive, + "fileId", + ({ drive }) => ({ + drive, + }), + ], + description: "The file to watch for comments", + }, + }, + hooks: { + async deploy() { + // Set the init time to now + this._setInitTime(Date.now()); + + // Emit sample comments from the file + const commentsStream = this.googleDrive.listComments(this.fileId, null); + let count = 0; + + for await (const comment of commentsStream) { + if (count >= 5) break; // Limit to 5 sample comments + await this.emitComment(comment); + count++; + } + }, + }, + methods: { + _getInitTime() { + return this.db.get("initTime"); + }, + _setInitTime(initTime) { + this.db.set("initTime", initTime); + }, + _getLastCommentTime() { + return this.db.get("lastCommentTime") || this._getInitTime(); + }, + _setLastCommentTime(commentTime) { + this.db.set("lastCommentTime", commentTime); + }, + generateMeta(comment) { + const { + id: commentId, + content: summary, + modifiedTime: tsString, + } = comment; + const ts = Date.parse(tsString); + + return { + id: `${commentId}-${ts}`, + summary, + ts, + }; + }, + async emitComment(comment) { + const meta = this.generateMeta(comment); + this.$emit(comment, meta); + }, + }, + async run() { + const lastCommentTime = this._getLastCommentTime(); + let maxCommentTime = lastCommentTime; + + const commentsStream = this.googleDrive.listComments( + this.fileId, + lastCommentTime, + ); + + for await (const comment of commentsStream) { + const commentTime = Date.parse(comment.modifiedTime); + if (commentTime <= lastCommentTime) { + continue; + } + + await this.emitComment(comment); + + maxCommentTime = Math.max(maxCommentTime, commentTime); + } + + // Update the last comment time after processing all comments + this._setLastCommentTime(maxCommentTime); + }, + sampleEmit, +}; diff --git a/components/google_drive/sources/new-or-modified-comments-polling/test-event.mjs b/components/google_drive/sources/new-or-modified-comments-polling/test-event.mjs new file mode 100644 index 0000000000000..96fd16eee7f4a --- /dev/null +++ b/components/google_drive/sources/new-or-modified-comments-polling/test-event.mjs @@ -0,0 +1,35 @@ +export default { + "kind": "drive#comment", + "id": "AAAAAvqIXEU", + "createdTime": "2023-02-06T15:13:33.023Z", + "modifiedTime": "2023-06-28T12:52:54.570Z", + "author": { + "displayName": "John Doe", + "kind": "drive#user", + "me": true, + "permissionId": "077423361841532483", + "emailAddress": "john@doe.com", + "photoLink": "https://lh3.googleusercontent.com/a/default-user=s64" + }, + "htmlContent": "John Doe commented:", + "content": "This is a sample comment", + "deleted": false, + "replies": [ + { + "kind": "drive#reply", + "id": "AAAAAvqIXEU/replies/AEnB2UqIXEU", + "createdTime": "2023-02-06T15:13:33.023Z", + "modifiedTime": "2023-06-28T12:52:54.570Z", + "author": { + "displayName": "Jane Doe", + "kind": "drive#user", + "permissionId": "077423361841532483", + "emailAddress": "jane@doe.com", + "photoLink": "https://lh3.googleusercontent.com/a/default-user=s64" + }, + "htmlContent": "Jane Doe replied:", + "content": "This is a reply to the comment", + "deleted": false + } + ] +}; diff --git a/components/google_drive/sources/new-or-modified-folders-polling/new-or-modified-folders-polling.mjs b/components/google_drive/sources/new-or-modified-folders-polling/new-or-modified-folders-polling.mjs new file mode 100644 index 0000000000000..62fdebef1e58c --- /dev/null +++ b/components/google_drive/sources/new-or-modified-folders-polling/new-or-modified-folders-polling.mjs @@ -0,0 +1,224 @@ +import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; +import googleDrive from "../../google_drive.app.mjs"; +import { getListFilesOpts } from "../../common/utils.mjs"; +import { GOOGLE_DRIVE_FOLDER_MIME_TYPE } from "../../common/constants.mjs"; +import sampleEmit from "./test-event.mjs"; + +export default { + key: "google_drive-new-or-modified-folders-polling", + name: "New or Modified Folders (Polling)", + description: "Emit new event when a folder is created or modified in the selected Drive", + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + googleDrive, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + drive: { + propDefinition: [ + googleDrive, + "watchedDrive", + ], + description: "Defaults to My Drive. To select a [Shared Drive](https://support.google.com/a/users/answer/9310351) instead, select it from this list.", + optional: false, + }, + folderId: { + propDefinition: [ + googleDrive, + "folderId", + ({ drive }) => ({ + drive, + }), + ], + label: "Parent Folder", + description: "The ID of the parent folder which contains the folders. If not specified, it will watch all folders from the drive's top-level folder.", + optional: true, + }, + includeSubfolders: { + type: "boolean", + label: "Include Subfolders", + description: "Whether to include subfolders of the parent folder in the changes.", + optional: true, + }, + }, + hooks: { + async deploy() { + // Get initial page token for change tracking + const driveId = this.getDriveId(); + const startPageToken = await this.googleDrive.getPageToken(driveId); + this._setPageToken(startPageToken); + + this._setLastRunTimestamp(Date.now()); + + // Emit sample folders from the last 30 days + const daysAgo = new Date(); + daysAgo.setDate(daysAgo.getDate() - 30); + const timeString = daysAgo.toISOString(); + + const args = this.getListFilesOpts({ + q: `mimeType = "${GOOGLE_DRIVE_FOLDER_MIME_TYPE}" and modifiedTime > "${timeString}" and trashed = false`, + fields: "*", + pageSize: 5, + }); + + const { files } = await this.googleDrive.listFilesInPage(null, args); + + for (const file of files) { + if (await this.shouldProcess(file)) { + await this.emitFolder(file); + } + } + }, + }, + methods: { + _getPageToken() { + return this.db.get("pageToken"); + }, + _setPageToken(pageToken) { + this.db.set("pageToken", pageToken); + }, + _getLastRunTimestamp() { + return this.db.get("lastRunTimestamp"); + }, + _setLastRunTimestamp(timestamp) { + this.db.set("lastRunTimestamp", timestamp); + }, + _getLastModifiedTimeForFile(fileId) { + return this.db.get(fileId); + }, + _setModifiedTimeForFile(fileId, modifiedTime) { + this.db.set(fileId, modifiedTime); + }, + getDriveId(drive = this.drive) { + return this.googleDrive.getDriveId(drive); + }, + getListFilesOpts(args = {}) { + return getListFilesOpts(this.drive, { + ...args, + }); + }, + async getAllParents(folderId) { + const allParents = []; + let currentId = folderId; + + while (currentId) { + const folder = await this.googleDrive.getFile(currentId, { + fields: "parents", + }); + const parents = folder.parents; + + if (parents && parents.length > 0) { + allParents.push(parents[0]); + } + currentId = parents?.[0]; + } + + return allParents; + }, + async shouldProcess(file) { + // Skip if not a folder + if (file.mimeType !== GOOGLE_DRIVE_FOLDER_MIME_TYPE) { + return false; + } + + // If no parent folder specified, process all folders + if (!this.folderId) { + return true; + } + + const root = await this.googleDrive.getFile(this.drive === "My Drive" + ? "root" + : this.drive); + + const allParents = []; + if (this.includeSubfolders) { + allParents.push(...(await this.getAllParents(file.id))); + } else if (file.parents) { + allParents.push(file.parents[0]); + } + + return allParents.includes(this.folderId || root.id); + }, + generateMeta(file) { + const { + id: fileId, + name: summary, + modifiedTime: tsString, + } = file; + const ts = Date.parse(tsString); + + return { + id: `${fileId}-${ts}`, + summary, + ts, + }; + }, + async emitFolder(file) { + const meta = this.generateMeta(file); + this.$emit(file, meta); + }, + }, + async run() { + const currentRunTimestamp = Date.now(); + + const pageToken = this._getPageToken(); + const driveId = this.getDriveId(); + + const changedFilesStream = this.googleDrive.listChanges(pageToken, driveId); + + for await (const changedFilesPage of changedFilesStream) { + console.log("Changed files page:", changedFilesPage); + const { + changedFiles, + nextPageToken, + } = changedFilesPage; + + console.log(changedFiles.length + ? `Processing ${changedFiles.length} changed files` + : "No changed files since last run"); + + for (const file of changedFiles) { + // Skip if not a folder + if (file.mimeType !== GOOGLE_DRIVE_FOLDER_MIME_TYPE) { + continue; + } + + // Get full file metadata + const fullFile = await this.googleDrive.getFile(file.id, { + fields: "*", + }); + + const modifiedTime = Date.parse(fullFile.modifiedTime); + const lastModifiedTimeForFile = this._getLastModifiedTimeForFile(fullFile.id); + + // Skip if not modified since last check + if (lastModifiedTimeForFile === modifiedTime) { + console.log(`Skipping unmodified folder ${fullFile.name || fullFile.id}`); + continue; + } + + if (!await this.shouldProcess(fullFile)) { + console.log(`Skipping folder ${fullFile.name || fullFile.id}`); + continue; + } + + await this.emitFolder(fullFile); + + this._setModifiedTimeForFile(fullFile.id, modifiedTime); + } + + // Save the next page token after successfully processing + this._setPageToken(nextPageToken); + } + + // Update the last run timestamp after processing all changes + this._setLastRunTimestamp(currentRunTimestamp); + }, + sampleEmit, +}; diff --git a/components/google_drive/sources/new-or-modified-folders-polling/test-event.mjs b/components/google_drive/sources/new-or-modified-folders-polling/test-event.mjs new file mode 100644 index 0000000000000..897ffe95208b5 --- /dev/null +++ b/components/google_drive/sources/new-or-modified-folders-polling/test-event.mjs @@ -0,0 +1,51 @@ +export default { + "kind": "drive#file", + "id": "1F1Q9K7L8N2P3R4S5T6U7V8W9X0Y1Z2A3", + "name": "New Folder", + "mimeType": "application/vnd.google-apps.folder", + "parents": [ + "0ANs73yKKVA" + ], + "createdTime": "2023-06-28T12:52:54.570Z", + "modifiedTime": "2023-06-28T12:52:54.570Z", + "modifiedByMeTime": "2023-06-28T12:52:54.570Z", + "viewedByMeTime": "2023-06-29T06:30:58.441Z", + "shared": false, + "ownedByMe": true, + "trashed": false, + "explicitlyTrashed": false, + "owners": [ + { + "displayName": "John Doe", + "kind": "drive#user", + "me": true, + "permissionId": "07742336189483", + "emailAddress": "john@doe.com", + "photoLink": "https://lh3.googleusercontent.com/a/default-user=s64" + } + ], + "lastModifyingUser": { + "displayName": "John Doe", + "kind": "drive#user", + "me": true, + "permissionId": "077423361841532483", + "emailAddress": "john@doe.com", + "photoLink": "https://lh3.googleusercontent.com/a/default-user=s64" + }, + "webViewLink": "https://drive.google.com/drive/folders/1F1Q9K7L8N2P3R4S5T6U7V8W9X0Y1Z2A3?usp=drivesdk", + "version": "1", + "capabilities": { + "canCopy": true, + "canDelete": true, + "canRename": true, + "canTrash": true, + "canUntrash": true, + "canShare": true, + "canListChildren": true, + "canAddChildren": true, + "canRemoveChildren": true + }, + "spaces": [ + "drive" + ] +}; diff --git a/components/google_drive/sources/new-spreadsheet-polling/new-spreadsheet-polling.mjs b/components/google_drive/sources/new-spreadsheet-polling/new-spreadsheet-polling.mjs new file mode 100644 index 0000000000000..7cc1bf7aebd12 --- /dev/null +++ b/components/google_drive/sources/new-spreadsheet-polling/new-spreadsheet-polling.mjs @@ -0,0 +1,188 @@ +import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; +import googleDrive from "../../google_drive.app.mjs"; +import { getListFilesOpts } from "../../common/utils.mjs"; +import { GOOGLE_DRIVE_FOLDER_MIME_TYPE } from "../../common/constants.mjs"; +import sampleEmit from "./test-event.mjs"; + +export default { + key: "google_drive-new-spreadsheet-polling", + name: "New Spreadsheet (Polling)", + description: "Emit new event when a new spreadsheet is created in a drive.", + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + googleDrive, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + drive: { + propDefinition: [ + googleDrive, + "watchedDrive", + ], + description: "Defaults to My Drive. To select a [Shared Drive](https://support.google.com/a/users/answer/9310351) instead, select it from this list.", + optional: false, + }, + folders: { + propDefinition: [ + googleDrive, + "folderId", + ({ drive }) => ({ + drive, + baseOpts: { + q: `mimeType = '${GOOGLE_DRIVE_FOLDER_MIME_TYPE}' and trashed = false`, + }, + }), + ], + type: "string[]", + label: "Folders", + description: "The specific folder(s) to watch for new spreadsheets. Leave blank to watch all folders in the Drive.", + optional: true, + }, + }, + hooks: { + async deploy() { + // Get initial page token for change tracking + const driveId = this.getDriveId(); + const startPageToken = await this.googleDrive.getPageToken(driveId); + this._setPageToken(startPageToken); + + this._setLastRunTimestamp(Date.now()); + + // Emit sample spreadsheets from the last 30 days + const daysAgo = new Date(); + daysAgo.setDate(daysAgo.getDate() - 30); + const timeString = daysAgo.toISOString(); + + const args = this.getListFilesOpts({ + q: `mimeType = "application/vnd.google-apps.spreadsheet" and createdTime > "${timeString}" and trashed = false`, + orderBy: "createdTime desc", + fields: "*", + pageSize: 5, + }); + + const { files } = await this.googleDrive.listFilesInPage(null, args); + + for (const file of files) { + if (this.shouldProcess(file)) { + await this.emitSpreadsheet(file); + } + } + }, + }, + methods: { + _getPageToken() { + return this.db.get("pageToken"); + }, + _setPageToken(pageToken) { + this.db.set("pageToken", pageToken); + }, + _getLastRunTimestamp() { + return this.db.get("lastRunTimestamp"); + }, + _setLastRunTimestamp(timestamp) { + this.db.set("lastRunTimestamp", timestamp); + }, + getDriveId(drive = this.drive) { + return this.googleDrive.getDriveId(drive); + }, + getListFilesOpts(args = {}) { + return getListFilesOpts(this.drive, { + ...args, + }); + }, + shouldProcess(file) { + // Check if it's a spreadsheet + if (!file.mimeType || !file.mimeType.includes("spreadsheet")) { + return false; + } + + // Check if specific folders are being watched + if (this.folders?.length > 0) { + const watchedFolders = new Set(this.folders); + if (!file.parents || !file.parents.some((p) => watchedFolders.has(p))) { + return false; + } + } + + return true; + }, + generateMeta(file) { + const { + id: fileId, + name: summary, + createdTime: tsString, + } = file; + const ts = Date.parse(tsString); + + return { + id: `${fileId}-${ts}`, + summary, + ts, + }; + }, + async emitSpreadsheet(file) { + const meta = this.generateMeta(file); + this.$emit(file, meta); + }, + }, + async run() { + const currentRunTimestamp = Date.now(); + const lastRunTimestamp = this._getLastRunTimestamp(); + + const pageToken = this._getPageToken(); + const driveId = this.getDriveId(); + + const changedFilesStream = this.googleDrive.listChanges(pageToken, driveId); + + for await (const changedFilesPage of changedFilesStream) { + console.log("Changed files page:", changedFilesPage); + const { + changedFiles, + nextPageToken, + } = changedFilesPage; + + console.log(changedFiles.length + ? `Processing ${changedFiles.length} changed files` + : "No changed files since last run"); + + for (const file of changedFiles) { + // Skip if not a spreadsheet + if (!file.mimeType || !file.mimeType.includes("spreadsheet")) { + continue; + } + + // Get full file metadata including parents + const fullFile = await this.googleDrive.getFile(file.id, { + fields: "*", + }); + + // Check if it's a new spreadsheet (created after last run) + const fileCreatedTime = Date.parse(fullFile.createdTime); + if (fileCreatedTime <= lastRunTimestamp) { + console.log(`Skipping existing spreadsheet ${fullFile.name || fullFile.id}`); + continue; + } + + if (!this.shouldProcess(fullFile)) { + console.log(`Skipping spreadsheet ${fullFile.name || fullFile.id}`); + continue; + } + + await this.emitSpreadsheet(fullFile); + } + + // Save the next page token after successfully processing + this._setPageToken(nextPageToken); + } + + // Update the last run timestamp after processing all changes + this._setLastRunTimestamp(currentRunTimestamp); + }, + sampleEmit, +}; diff --git a/components/google_drive/sources/new-spreadsheet-polling/test-event.mjs b/components/google_drive/sources/new-spreadsheet-polling/test-event.mjs new file mode 100644 index 0000000000000..793975b39f199 --- /dev/null +++ b/components/google_drive/sources/new-spreadsheet-polling/test-event.mjs @@ -0,0 +1,111 @@ +export default { + "kind": "drive#file", + "copyRequiresWriterPermission": false, + "writersCanShare": true, + "viewedByMe": true, + "mimeType": "application/vnd.google-apps.spreadsheet", + "exportLinks": { + "application/x-vnd.oasis.opendocument.spreadsheet": "https://docs.google.com/spreadsheets/export?id=1LmXTIQ0wqKP7T3-l9r127MaTXn3pVbTmw4&exportFormat=ods", + "text/tab-separated-values": "https://docs.google.com/spreadsheets/export?id=1LmXTIQ0wqKP7T3-l9r127MaTXnViTOo&exportFormat=tsv", + "application/pdf": "https://docs.google.com/spreadsheets/export?id=1LmXTIQ0wqKP7T3-l9r127MaBViTOo&exportFormat=pdf", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "https://docs.google.com/spreadsheets/export?id=1LmXTIQ0wqKP7T3-l9r127MaTXn3pVbOo&exportFormat=xlsx", + "text/csv": "https://docs.google.com/spreadsheets/export?id=1LmXTIQ0wqKP7T3-l9r127MaTXn3pVbBViTOo&exportFormat=csv", + "application/zip": "https://docs.google.com/spreadsheets/export?id=1LmXTIQ0wqKP7T3-VbTmw4Pt2BViTOo&exportFormat=zip", + "application/vnd.oasis.opendocument.spreadsheet": "https://docs.google.com/spreadsheets/export?id=1LmXTIQ0wqKP7T3-l9r127MaTXn3pVbViTOo&exportFormat=ods" + }, + "parents": [ + "0ANs73yKKVA" + ], + "thumbnailLink": "https://docs.google.com/feeds/vt?gd=true&id=1LmXTIQ0wqKP7T3-l9r127MaTXn3pVbTmw4&v=12&s=AMedNnoAAAAAZKWjrEqscucFpYCyRCJqnd0wtBiDtXYh&sz=s220", + "iconLink": "https://drive-thirdparty.googleusercontent.com/16/type/application/vnd.google-apps.spreadsheet", + "shared": false, + "lastModifyingUser": { + "displayName": "John Doe", + "kind": "drive#user", + "me": true, + "permissionId": "077423361841532483", + "emailAddress": "john@doe.com", + "photoLink": "https://lh3.googleusercontent.com/a/default-user=s64" + }, + "owners": [ + { + "displayName": "John Doe", + "kind": "drive#user", + "me": true, + "permissionId": "07742336189483", + "emailAddress": "john@doe.com", + "photoLink": "https://lh3.googleusercontent.com/a/default-user=s64" + } + ], + "webViewLink": "https://docs.google.com/spreadsheets/d/1LmXTIQ0wqKP7T3-l9r127MaTXn3pVbTmwTOo/edit?usp=drivesdk", + "size": "1024", + "viewersCanCopyContent": true, + "permissions": [ + { + "id": "07742336184153259483", + "displayName": "John Doe", + "type": "user", + "kind": "drive#permission", + "photoLink": "https://lh3.googleusercontent.com/a/default-user=s64", + "emailAddress": "john@doe.com", + "role": "owner", + "deleted": false, + "pendingOwner": false + } + ], + "hasThumbnail": true, + "spaces": [ + "drive" + ], + "id": "1LmXTIQ0wqKP7T3-l9r127Maw4Pt2BViTOo", + "name": "New Spreadsheet", + "starred": false, + "trashed": false, + "explicitlyTrashed": false, + "createdTime": "2023-06-28T12:52:54.570Z", + "modifiedTime": "2023-06-28T12:52:54.570Z", + "modifiedByMeTime": "2023-06-28T12:52:54.570Z", + "viewedByMeTime": "2023-06-29T06:30:58.441Z", + "quotaBytesUsed": "1024", + "version": "53", + "ownedByMe": true, + "isAppAuthorized": false, + "capabilities": { + "canChangeViewersCanCopyContent": true, + "canEdit": true, + "canCopy": true, + "canComment": true, + "canAddChildren": false, + "canDelete": true, + "canDownload": true, + "canListChildren": false, + "canRemoveChildren": false, + "canRename": true, + "canTrash": true, + "canReadRevisions": true, + "canChangeCopyRequiresWriterPermission": true, + "canMoveItemIntoTeamDrive": true, + "canUntrash": true, + "canModifyContent": true, + "canMoveItemOutOfDrive": true, + "canAddMyDriveParent": false, + "canRemoveMyDriveParent": true, + "canMoveItemWithinDrive": true, + "canShare": true, + "canMoveChildrenWithinDrive": false, + "canModifyContentRestriction": true, + "canChangeSecurityUpdateEnabled": false, + "canAcceptOwnership": false, + "canReadLabels": true, + "canModifyLabels": true + }, + "thumbnailVersion": "12", + "modifiedByMe": true, + "permissionIds": [ + "07742336184153259483" + ], + "linkShareMetadata": { + "securityUpdateEligible": false, + "securityUpdateEnabled": true + } +};