[ACTION] Google Drive - Polling-based components as an alternative to webhook-based ones

components/google_drive/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/google_drive",
-  "version": "1.3.1",
+  "version": "1.4.0",
   "description": "Pipedream Google_drive Components",
   "main": "google_drive.app.mjs",
   "keywords": [

components/google_drive/sources/new-files-instant-polling/new-files-instant-polling.mjs

@@ -0,0 +1,190 @@
+import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
+import googleDrive from "../../google_drive.app.mjs";
+import { getListFilesOpts } from "../../common/utils.mjs";
+import { GOOGLE_DRIVE_FOLDER_MIME_TYPE } from "../../common/constants.mjs";
+import sampleEmit from "./test-event.mjs";
+
+export default {
+  key: "google_drive-new-files-instant-polling",
+  name: "New Files (Polling)",
+  description: "Emit new event when a new file is added in your linked Google Drive",
+  version: "0.0.1",
+  type: "source",
+  dedupe: "unique",
+  props: {
+    googleDrive,
+    db: "$.service.db",
+    timer: {
+      type: "$.interface.timer",
+      default: {
+        intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL,
+      },
+    },
+    drive: {
+      propDefinition: [
+        googleDrive,
+        "watchedDrive",
+      ],
+      description: "Defaults to My Drive. To select a [Shared Drive](https://support.google.com/a/users/answer/9310351) instead, select it from this list.",
+      optional: false,
+    },
+    folders: {
+      propDefinition: [
+        googleDrive,
+        "folderId",
+        ({ drive }) => ({
+          drive,
+          baseOpts: {
+            q: `mimeType = '${GOOGLE_DRIVE_FOLDER_MIME_TYPE}' and trashed = false`,
+          },
+        }),
+      ],
+      type: "string[]",
+      label: "Folders",
+      description: "The specific folder(s) to watch for new files. Leave blank to watch all files in the Drive.",
+      optional: true,
+      default: [],
+    },
+  },
+  hooks: {
+    async deploy() {
+      // Get initial page token for change tracking
+      const driveId = this.getDriveId();
+      const startPageToken = await this.googleDrive.getPageToken(driveId);
+      this._setPageToken(startPageToken);
+
+      this._setLastRunTimestamp(Date.now());
+
+      // Emit sample files from the last 30 days
+      const daysAgo = new Date();
+      daysAgo.setDate(daysAgo.getDate() - 30);
+      const timeString = daysAgo.toISOString();
+
+      const args = this.getListFilesOpts({
+        q: `mimeType != "application/vnd.google-apps.folder" and createdTime > "${timeString}" and trashed = false`,
+        orderBy: "createdTime desc",
+        fields: "*",
+        pageSize: 5,
+      });
+
+      const { files } = await this.googleDrive.listFilesInPage(null, args);
+
+      for (const file of files) {
+        if (this.shouldProcess(file)) {
+          await this.emitFile(file);
+        }
+      }
+    },
+  },
+  methods: {
+    _getPageToken() {
+      return this.db.get("pageToken");
+    },
+    _setPageToken(pageToken) {
+      this.db.set("pageToken", pageToken);
+    },
+    _getLastRunTimestamp() {
+      return this.db.get("lastRunTimestamp");
+    },
+    _setLastRunTimestamp(timestamp) {
+      this.db.set("lastRunTimestamp", timestamp);
+    },
+    getDriveId(drive = this.drive) {
+      return this.googleDrive.getDriveId(drive);
+    },
+    getListFilesOpts(args = {}) {
+      return getListFilesOpts(this.drive, {
+        q: "mimeType != 'application/vnd.google-apps.folder' and trashed = false",
+        ...args,
+      });
+    },
+    shouldProcess(file) {
+      // Skip folders
+      if (file.mimeType === GOOGLE_DRIVE_FOLDER_MIME_TYPE) {
+        return false;
+      }
+
+      // Check if specific folders are being watched
+      if (this.folders?.length > 0) {
+        const watchedFolders = new Set(this.folders);
+        if (!file.parents || !file.parents.some((p) => watchedFolders.has(p))) {
+          return false;
+        }
+      }
+
+      return true;
+    },
+    generateMeta(file) {
+      const {
+        id: fileId,
+        name: summary,
+        createdTime: tsString,
+      } = file;
+      const ts = Date.parse(tsString);
+
+      return {
+        id: `${fileId}-${ts}`,
+        summary,
+        ts,
+      };
+    },
+    async emitFile(file) {
+      const meta = this.generateMeta(file);
+      this.$emit(file, meta);
+    },
+  },
+  async run() {
+    const currentRunTimestamp = Date.now();
+    const lastRunTimestamp = this._getLastRunTimestamp();
+
+    const pageToken = this._getPageToken();
+    const driveId = this.getDriveId();
+
+    const changedFilesStream = this.googleDrive.listChanges(pageToken, driveId);
+
+    for await (const changedFilesPage of changedFilesStream) {
+      console.log("Changed files page:", changedFilesPage);
+      const {
+        changedFiles,
+        nextPageToken,
+      } = changedFilesPage;
+
+      console.log(changedFiles.length
+        ? `Processing ${changedFiles.length} changed files`
+        : "No changed files since last run");
+
+      for (const file of changedFiles) {
+        // Skip folders
+        if (file.mimeType === GOOGLE_DRIVE_FOLDER_MIME_TYPE) {
+          continue;
+        }
+
+        // Get full file metadata including parents
+        const fullFile = await this.googleDrive.getFile(file.id, {
+          fields: "*",
+        });
+
+        // Check if it's a new file (created after last run)
+        const fileCreatedTime = Date.parse(fullFile.createdTime);
+        if (fileCreatedTime <= lastRunTimestamp) {
+          console.log(`Skipping existing file ${fullFile.name || fullFile.id}`);
+          continue;
+        }
+
+        if (!this.shouldProcess(fullFile)) {
+          console.log(`Skipping file ${fullFile.name || fullFile.id}`);
+          continue;
+        }
+
+        await this.emitFile(fullFile);
+      }
+
+      // Save the next page token after successfully processing
+      this._setPageToken(nextPageToken);
+    }
+
+    // Update the last run timestamp after processing all changes
+    this._setLastRunTimestamp(currentRunTimestamp);
+  },
+  sampleEmit,
+};

components/google_drive/sources/new-files-instant-polling/test-event.mjs

@@ -0,0 +1,99 @@
+export default {
+  "kind": "drive#file",
+  "copyRequiresWriterPermission": false,
+  "writersCanShare": true,
+  "viewedByMe": true,
+  "mimeType": "text/plain",
+  "parents": [
+    "0ANs73yKKVA"
+  ],
+  "iconLink": "https://drive-thirdparty.googleusercontent.com/16/type/text/plain",
+  "shared": false,
+  "lastModifyingUser": {
+    "displayName": "John Doe",
+    "kind": "drive#user",
+    "me": true,
+    "permissionId": "077423361841532483",
+    "emailAddress": "john@doe.com",
+    "photoLink": "https://lh3.googleusercontent.com/a/default-user=s64"
+  },
+  "owners": [
+    {
+      "displayName": "John Doe",
+      "kind": "drive#user",
+      "me": true,
+      "permissionId": "07742336189483",
+      "emailAddress": "john@doe.com",
+      "photoLink": "https://lh3.googleusercontent.com/a/default-user=s64"
+    }
+  ],
+  "webViewLink": "https://drive.google.com/file/d/1LmXTIQ0wqKP7T3-l9r127Maw4Pt2BViTOo/view?usp=drivesdk",
+  "size": "1024",
+  "viewersCanCopyContent": true,
+  "permissions": [
+    {
+      "id": "07742336184153259483",
+      "displayName": "John Doe",
+      "type": "user",
+      "kind": "drive#permission",
+      "photoLink": "https://lh3.googleusercontent.com/a/default-user=s64",
+      "emailAddress": "john@doe.com",
+      "role": "owner",
+      "deleted": false,
+      "pendingOwner": false
+    }
+  ],
+  "spaces": [
+    "drive"
+  ],
+  "id": "1LmXTIQ0wqKP7T3-l9r127Maw4Pt2BViTOo",
+  "name": "New File.txt",
+  "starred": false,
+  "trashed": false,
+  "explicitlyTrashed": false,
+  "createdTime": "2023-06-28T12:52:54.570Z",
+  "modifiedTime": "2023-06-28T12:52:54.570Z",
+  "modifiedByMeTime": "2023-06-28T12:52:54.570Z",
+  "viewedByMeTime": "2023-06-29T06:30:58.441Z",
+  "quotaBytesUsed": "1024",
+  "version": "1",
+  "ownedByMe": true,
+  "isAppAuthorized": false,
+  "capabilities": {
+    "canChangeViewersCanCopyContent": true,
+    "canEdit": true,
+    "canCopy": true,
+    "canComment": true,
+    "canAddChildren": false,
+    "canDelete": true,
+    "canDownload": true,
+    "canListChildren": false,
+    "canRemoveChildren": false,
+    "canRename": true,
+    "canTrash": true,
+    "canReadRevisions": true,
+    "canChangeCopyRequiresWriterPermission": true,
+    "canMoveItemIntoTeamDrive": true,
+    "canUntrash": true,
+    "canModifyContent": true,
+    "canMoveItemOutOfDrive": true,
+    "canAddMyDriveParent": false,
+    "canRemoveMyDriveParent": true,
+    "canMoveItemWithinDrive": true,
+    "canShare": true,
+    "canMoveChildrenWithinDrive": false,
+    "canModifyContentRestriction": true,
+    "canChangeSecurityUpdateEnabled": false,
+    "canAcceptOwnership": false,
+    "canReadLabels": true,
+    "canModifyLabels": true
+  },
+  "modifiedByMe": true,
+  "permissionIds": [
+    "07742336184153259483"
+  ],
+  "linkShareMetadata": {
+    "securityUpdateEligible": false,
+    "securityUpdateEnabled": true
+  }
+};