Delayed delete for orphaned files

nodemailer · Aug 31, 2023 · 21d5597 · 21d5597
1 parent 8ebd4c1
commit 21d5597
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 15 deletions.
diff --git a/indexes.yaml b/indexes.yaml
@@ -497,15 +497,14 @@ indexes:
               metadata.c: 1
               metadata.m: 1
 
-# TODO
-#    - collection: attachments.files
-#      type: gridfs # index applies to gridfs database
-#      index:
-#          name: related_attachments_cu
-#          key:
-#              metadata.c: 1
-#              metadata.m: 1
-#              metadata.cu: 1
+    - collection: attachments.files
+      type: gridfs # index applies to gridfs database
+      index:
+          name: related_attachments_cu
+          key:
+              metadata.c: 1
+              metadata.m: 1
+              metadata.cu: 1
 
     - collection: attachments.chunks
       type: gridfs # index applies to gridfs database

diff --git a/lib/attachments/gridstore-storage.js b/lib/attachments/gridstore-storage.js
@@ -11,6 +11,9 @@ const base64Offset = require('./base64-offset');
 // Set to false to disable base64 decoding feature
 const FEATURE_DECODE_ATTACHMENTS = true;
 
+const ORPHANED_ATTACHMENTS_DELAY = 24 * 3600 * 1000;
+const MAX_ORPHANED_ATTACHMENTS = 1000;
+
 class GridstoreStorage {
     constructor(options) {
         this.bucketName = (options.options && options.options.bucket) || 'attachments';
@@ -59,6 +62,7 @@ class GridstoreStorage {
         let metadata = {
             m: attachment.magic,
             c: 1,
+            cu: new Date(),
             esize: attachment.body.length,
             transferEncoding: attachment.transferEncoding
         };
@@ -444,10 +448,28 @@ class GridstoreStorage {
 
     deleteOrphaned(callback) {
         // NB! scattered query
-        let cursor = this.gridfs.collection(this.bucketName + '.files').find({
-            'metadata.c': 0,
-            'metadata.m': 0
-        });
+        let cursor = this.gridfs.collection(this.bucketName + '.files').find(
+            {
+                'metadata.c': 0,
+                'metadata.m': 0,
+                $or: [
+                    {
+                        'metadata.cu': null
+                    },
+                    {
+                        'metadata.cu': {
+                            $lt: new Date(Date.now() - ORPHANED_ATTACHMENTS_DELAY)
+                        }
+                    }
+                ]
+            },
+            {
+                hint: 'related_attachments_cu',
+                comment: 'List orphaned attachments',
+                maxTimeMS: 2 * 60 * 1000,
+                limit: MAX_ORPHANED_ATTACHMENTS
+            }
+        );
 
         let deleted = 0;
         let processNext = () => {

diff --git a/tasks.js b/tasks.js
@@ -486,7 +486,7 @@ async function runTasks() {
             try {
                 await new Promise((resolve, reject) => {
                     // run pseudo task
-                    processTask({ type: 'acme-update', _id: 'acme-update-id', lock: 'acme-update-lock' }, {}, err => {
+                    processTask({ type: 'acme-update', _id: 'acme-update-id', lock: 'acme-update-lock', silent: true }, {}, err => {
                         if (err) {
                             return reject(err);
                         } else {
@@ -535,7 +535,9 @@ async function runTasks() {
 }
 
 function processTask(task, data, callback) {
-    log.verbose('Tasks', 'type=%s id=%s data=%s', task.type, task._id, JSON.stringify(data));
+    if (!data.silent) {
+        log.verbose('Tasks', 'type=%s id=%s data=%s', task.type, task._id, JSON.stringify(data));
+    }
 
     switch (task.type) {
         case 'restore':