Google Drive "New or Modified Files" polling source

components/google_drive/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/google_drive",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Pipedream Google_drive Components",
   "main": "google_drive.app.mjs",
   "keywords": [

components/google_drive/sources/new-or-modified-files-polling/new-or-modified-files-polling.mjs

@@ -0,0 +1,220 @@
+import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
+import googleDrive from "../../google_drive.app.mjs";
+import { getListFilesOpts } from "../../common/utils.mjs";
+import sampleEmit from "./test-event.mjs";
+import { GOOGLE_DRIVE_FOLDER_MIME_TYPE } from "../../common/constants.mjs";
+
+export default {
+  key: "google_drive-new-or-modified-files-polling",
+  name: "New or Modified Files (Polling)",
+  description: "Emit new event when a file in the selected Drive is created, modified or trashed. [See the documentation](https://developers.google.com/drive/api/v3/reference/changes/list)",
+  version: "0.0.1",
+  type: "source",
+  dedupe: "unique",
+  props: {
+    googleDrive,
+    db: "$.service.db",
+    timer: {
+      type: "$.interface.timer",
+      default: {
+        intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL,
+      },
+    },
+    drive: {
+      propDefinition: [
+        googleDrive,
+        "watchedDrive",
+      ],
+      description: "Defaults to My Drive. To select a [Shared Drive](https://support.google.com/a/users/answer/9310351) instead, select it from this list.",
+      optional: false,
+    },
+    files: {
+      type: "string[]",
+      label: "Files",
+      description: "The specific file(s) you want to watch for changes. Leave blank to watch all files in the Drive. Note that events will only be emitted for files directly in these folders (not in their subfolders, unless also selected here)",
+      optional: true,
+      default: [],
+      options({ prevContext }) {
+        const { nextPageToken } = prevContext;
+        return this.googleDrive.listFilesOptions(nextPageToken, this.getListFilesOpts());
+      },
+    },
+    folders: {
+      type: "string[]",
+      label: "Folders",
+      description: "The specific folder(s) to watch for changes. Leave blank to watch all folders in the Drive.",
+      optional: true,
+      default: [],
+      options({ prevContext }) {
+        const { nextPageToken } = prevContext;
+        const baseOpts = {
+          q: "mimeType = 'application/vnd.google-apps.folder' and trashed = false",
+        };
+        const isMyDrive = this.googleDrive.isMyDrive(this.drive);
+        const opts = isMyDrive
+          ? baseOpts
+          : {
+            ...baseOpts,
+            corpora: "drive",
+            driveId: this.getDriveId(),
+            includeItemsFromAllDrives: true,
+            supportsAllDrives: true,
+          };
+        return this.googleDrive.listFilesOptions(nextPageToken, opts);
+      },
+    },
+    newFilesOnly: {
+      type: "boolean",
+      label: "New Files Only",
+      description: "If enabled, only emit events for newly created files (based on `createdTime`)",
+      default: false,
+      optional: true,
+    },
+  },
+  hooks: {
+    async deploy() {
+      // Get initial page token for change tracking
+      const driveId = this.getDriveId();
+      const startPageToken = await this.googleDrive.getPageToken(driveId);
+      this._setPageToken(startPageToken);
+
+      this._setLastRunTimestamp(Date.now());
+
+      // Emit sample events from the last 30 days
+      const daysAgo = new Date();
+      daysAgo.setDate(daysAgo.getDate() - 30);
+      const timeString = daysAgo.toISOString();
+
+      const args = this.getListFilesOpts({
+        q: `mimeType != "application/vnd.google-apps.folder" and modifiedTime > "${timeString}" and trashed = false`,
+        orderBy: "modifiedTime desc",
+        fields: "*",
+        pageSize: 5,
+      });
+
+      const { files } = await this.googleDrive.listFilesInPage(null, args);
+
+      for (const file of files) {
+        if (this.shouldProcess(file)) {
+          await this.emitFile(file);
+        }
+      }
+    },
+  },
+  methods: {
+    _getPageToken() {
+      return this.db.get("pageToken");
+    },
+    _setPageToken(pageToken) {
+      this.db.set("pageToken", pageToken);
+    },
+    _getLastRunTimestamp() {
+      return this.db.get("lastRunTimestamp");
+    },
+    _setLastRunTimestamp(timestamp) {
+      this.db.set("lastRunTimestamp", timestamp);
+    },
+    getDriveId(drive = this.drive) {
+      return this.googleDrive.getDriveId(drive);
+    },
+    getListFilesOpts(args = {}) {
+      return getListFilesOpts(this.drive, {
+        q: "mimeType != 'application/vnd.google-apps.folder' and trashed = false",
+        ...args,
+      });
+    },
+    shouldProcess(file, lastRunTimestamp) {
+      // Skip folders
+      if (file.mimeType === GOOGLE_DRIVE_FOLDER_MIME_TYPE) {
+        return false;
+      }
+
+      // Check if "new files only" mode is enabled
+      if (this.newFilesOnly) {
+        const fileCreatedTime = Date.parse(file.createdTime);
+        // Only process files created after the last run
+        if (fileCreatedTime <= lastRunTimestamp) {
+          return false;
+        }
+      }
+
+      // Check if specific files are being watched
+      if (this.files?.length > 0) {
+        if (!this.files.includes(file.id)) {
+          return false;
+        }
+      }
+
+      // Check if specific folders are being watched
+      if (this.folders?.length > 0) {
+        const watchedFolders = new Set(this.folders);
+        if (!file.parents || !file.parents.some((p) => watchedFolders.has(p))) {
+          return false;
+        }
+      }
+
+      return true;
+    },
+    generateMeta(file) {
+      const {
+        id: fileId,
+        name: summary,
+        modifiedTime: tsString,
+      } = file;
+      const ts = Date.parse(tsString);
+
+      return {
+        id: `${fileId}-${ts}`,
+        summary,
+        ts,
+      };
+    },
+    async emitFile(file) {
+      const meta = this.generateMeta(file);
+      this.$emit(file, meta);
+    },
+  },
+  async run() {
+    // Store the current run timestamp at the start
+    const currentRunTimestamp = Date.now();
+    const lastRunTimestamp = this._getLastRunTimestamp();
+
+    const pageToken = this._getPageToken();
+    const driveId = this.getDriveId();
+
+    const changedFilesStream = this.googleDrive.listChanges(pageToken, driveId);
+
+    for await (const changedFilesPage of changedFilesStream) {
+      console.log("Changed files page:", changedFilesPage);
+      const {
+        changedFiles,
+        nextPageToken,
+      } = changedFilesPage;
+
+      console.log(changedFiles.length
+        ? `Processing ${changedFiles.length} changed files`
+        : "No changed files since last run");
+
+      for (const file of changedFiles) {
+        // Get full file metadata including parents
+        const fullFile = await this.googleDrive.getFile(file.id, {
+          fields: "*",
+        });
+
+        if (!this.shouldProcess(fullFile, lastRunTimestamp)) {
+          console.log(`Skipping file ${fullFile.name || fullFile.id}`);
+          continue;
+        }
+
+        await this.emitFile(fullFile);
+      }
+
+      // Save the next page token after successfully processing
+      this._setPageToken(nextPageToken);
+    }
+
+    // Update the last run timestamp after processing all changes
+    this._setLastRunTimestamp(currentRunTimestamp);
+  },
+  sampleEmit,
+};

components/google_drive/sources/new-or-modified-files-polling/test-event.mjs

@@ -0,0 +1,111 @@
+export default {
+  "kind": "drive#file",
+  "copyRequiresWriterPermission": false,
+  "writersCanShare": true,
+  "viewedByMe": true,
+  "mimeType": "application/vnd.google-apps.spreadsheet",
+  "exportLinks": {
+    "application/x-vnd.oasis.opendocument.spreadsheet": "https://docs.google.com/spreadsheets/export?id=1LmXTIQ0wqKP7T3-l9r127MaTXn3pVbTmw4&exportFormat=ods",
+    "text/tab-separated-values": "https://docs.google.com/spreadsheets/export?id=1LmXTIQ0wqKP7T3-l9r127MaTXnViTOo&exportFormat=tsv",
+    "application/pdf": "https://docs.google.com/spreadsheets/export?id=1LmXTIQ0wqKP7T3-l9r127MaBViTOo&exportFormat=pdf",
+    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "https://docs.google.com/spreadsheets/export?id=1LmXTIQ0wqKP7T3-l9r127MaTXn3pVbOo&exportFormat=xlsx",
+    "text/csv": "https://docs.google.com/spreadsheets/export?id=1LmXTIQ0wqKP7T3-l9r127MaTXn3pVbBViTOo&exportFormat=csv",
+    "application/zip": "https://docs.google.com/spreadsheets/export?id=1LmXTIQ0wqKP7T3-VbTmw4Pt2BViTOo&exportFormat=zip",
+    "application/vnd.oasis.opendocument.spreadsheet": "https://docs.google.com/spreadsheets/export?id=1LmXTIQ0wqKP7T3-l9r127MaTXn3pVbViTOo&exportFormat=ods"
+  },
+  "parents": [
+    "0ANs73yKKVA"
+  ],
+  "thumbnailLink": "https://docs.google.com/feeds/vt?gd=true&id=1LmXTIQ0wqKP7T3-l9r127MaTXn3pVbTmw4&v=12&s=AMedNnoAAAAAZKWjrEqscucFpYCyRCJqnd0wtBiDtXYh&sz=s220",
+  "iconLink": "https://drive-thirdparty.googleusercontent.com/16/type/application/vnd.google-apps.spreadsheet",
+  "shared": false,
+  "lastModifyingUser": {
+    "displayName": "John Doe",
+    "kind": "drive#user",
+    "me": true,
+    "permissionId": "077423361841532483",
+    "emailAddress": "john@doe.com",
+    "photoLink": "https://lh3.googleusercontent.com/a/default-user=s64"
+  },
+  "owners": [
+    {
+      "displayName": "John Doe",
+      "kind": "drive#user",
+      "me": true,
+      "permissionId": "07742336189483",
+      "emailAddress": "john@doe.com",
+      "photoLink": "https://lh3.googleusercontent.com/a/default-user=s64"
+    }
+  ],
+  "webViewLink": "https://docs.google.com/spreadsheets/d/1LmXTIQ0wqKP7T3-l9r127MaTXn3pVbTmwTOo/edit?usp=drivesdk",
+  "size": "1024",
+  "viewersCanCopyContent": true,
+  "permissions": [
+    {
+      "id": "07742336184153259483",
+      "displayName": "John Doe",
+      "type": "user",
+      "kind": "drive#permission",
+      "photoLink": "https://lh3.googleusercontent.com/a/default-user=s64",
+      "emailAddress": "john@doe.com",
+      "role": "owner",
+      "deleted": false,
+      "pendingOwner": false
+    }
+  ],
+  "hasThumbnail": true,
+  "spaces": [
+    "drive"
+  ],
+  "id": "1LmXTIQ0wqKP7T3-l9r127Maw4Pt2BViTOo",
+  "name": "Pipedream 2213",
+  "starred": false,
+  "trashed": false,
+  "explicitlyTrashed": false,
+  "createdTime": "2023-02-06T15:13:33.023Z",
+  "modifiedTime": "2023-06-28T12:52:54.570Z",
+  "modifiedByMeTime": "2023-06-28T12:52:54.570Z",
+  "viewedByMeTime": "2023-06-29T06:30:58.441Z",
+  "quotaBytesUsed": "1024",
+  "version": "53",
+  "ownedByMe": true,
+  "isAppAuthorized": false,
+  "capabilities": {
+    "canChangeViewersCanCopyContent": true,
+    "canEdit": true,
+    "canCopy": true,
+    "canComment": true,
+    "canAddChildren": false,
+    "canDelete": true,
+    "canDownload": true,
+    "canListChildren": false,
+    "canRemoveChildren": false,
+    "canRename": true,
+    "canTrash": true,
+    "canReadRevisions": true,
+    "canChangeCopyRequiresWriterPermission": true,
+    "canMoveItemIntoTeamDrive": true,
+    "canUntrash": true,
+    "canModifyContent": true,
+    "canMoveItemOutOfDrive": true,
+    "canAddMyDriveParent": false,
+    "canRemoveMyDriveParent": true,
+    "canMoveItemWithinDrive": true,
+    "canShare": true,
+    "canMoveChildrenWithinDrive": false,
+    "canModifyContentRestriction": true,
+    "canChangeSecurityUpdateEnabled": false,
+    "canAcceptOwnership": false,
+    "canReadLabels": true,
+    "canModifyLabels": true
+  },
+  "thumbnailVersion": "12",
+  "modifiedByMe": true,
+  "permissionIds": [
+    "07742336184153259483"
+  ],
+  "linkShareMetadata": {
+    "securityUpdateEligible": false,
+    "securityUpdateEnabled": true
+  }
+};