[pr] Add an option to run the cleanup algorithm optimized for low memory usage

cleanup.js

@@ -82,14 +82,13 @@ cache.on('cleanup_delete_finish', data => {
     }
 });
 
-const msg = 'Gathering cache files for expiration';
 let spinner = null;
 
 if(logLevel < consts.LOG_DBG && logLevel >= consts.LOG_INFO) {
     spinner = ora({color: 'white'});
 
     cache.on('cleanup_search_progress', data => {
-        spinner.text = `${msg} (${data.deleteCount} of ${data.cacheCount} files, ${filesize(data.deleteSize)})`;
+        spinner.text = `${data.msg} (${data.deleteCount} of ${data.cacheCount} files, ${filesize(data.deleteSize)})`;
     });
 
     cache.on('cleanup_search_finish', () => {
@@ -98,11 +97,12 @@ if(logLevel < consts.LOG_DBG && logLevel >= consts.LOG_INFO) {
 
 } else if(logLevel === consts.LOG_DBG) {
     cache.on('cleanup_search_progress', data => {
-        const txt = `${msg} (${data.deleteCount} of ${data.cacheCount} files, ${filesize(data.deleteSize)})`;
+        const txt = `${data.msg} (${data.deleteCount} of ${data.cacheCount} files, ${filesize(data.deleteSize)})`;
         helpers.log(consts.LOG_DBG, txt);
     });
 }
 
+const msg = 'Gathering cache files for expiration';
 function doCleanup() {
     if (spinner) spinner.start(msg);
     cache.cleanup(dryRun)

lib/cache/cache_fs.js

@@ -43,7 +43,7 @@ class CacheFS extends CacheBase {
         const fileName = path.basename(filePath).toLowerCase();
         const matches = /^([0-9a-f]{32})-([0-9a-f]{32})\./.exec(fileName);
         const result = { guidStr: "", hashStr: ""};
-        if(matches.length === 3) {
+        if(matches && matches.length === 3) {
             result.guidStr = matches[1];
             result.hashStr = matches[2];
         }
@@ -110,78 +110,126 @@ class CacheFS extends CacheBase {
     }
 
     cleanup(dryRun = true) {
-        const self = this;
-
         const expireDuration = moment.duration(this._options.cleanupOptions.expireTimeSpan);
+        const minFileAccessTime = moment().subtract(expireDuration).toDate();
+        const maxCacheSize = this._options.cleanupOptions.maxCacheSize;
+
         if(!expireDuration.isValid() || expireDuration.asMilliseconds() === 0) {
             return Promise.reject(new Error("Invalid expireTimeSpan option"));
         }
 
-        const minFileAccessTime = moment().subtract(expireDuration).toDate();
-        const maxCacheSize = this._options.cleanupOptions.maxCacheSize;
-
-        const allItems = [];
-        const deleteItems = [];
+        let cacheCount = 0;
         let cacheSize = 0;
         let deleteSize = 0;
+        let deletedItemCount = 0;
+        let deleteItems = [];
+        let verb = dryRun ? 'Gathering' : 'Removing';
+        let spinnerMessage = verb + ' expired files';
 
         const progressData = () => {
             return {
-                cacheCount: allItems.length,
+                cacheCount: cacheCount,
                 cacheSize: cacheSize,
-                deleteCount: deleteItems.length,
-                deleteSize: deleteSize
+                deleteCount: deleteItems.length + deletedItemCount,
+                deleteSize: deleteSize,
+                msg: spinnerMessage,
             };
         };
 
-        const progressEvent = () => self.emit('cleanup_search_progress', progressData());
+        const progressEvent = () => this.emit('cleanup_search_progress', progressData());
 
         progressEvent();
         const progressTimer = setInterval(progressEvent, 250);
 
-        return helpers.readDir(self._cachePath, (item) => {
-            item = {path: item.path, stats: pick(item.stats, ['atime', 'size'])};
-            allItems.push(item);
+        return helpers.readDir(this._cachePath, async (item) => {
+
+            if(item.stats.isDirectory()) return next();
+
             cacheSize += item.stats.size;
+            cacheCount ++;
+
             if(item.stats.atime < minFileAccessTime) {
                 deleteSize += item.stats.size;
-                deleteItems.push(item);
+                deletedItemCount++;
+                await this.delete_cache_item(dryRun, item);
             }
         }).then(async () => {
-            if(maxCacheSize > 0 && cacheSize - deleteSize > maxCacheSize) {
-                allItems.sort((a, b) => { return a.stats.atime > b.stats.atime });
-                for(const item of allItems) {
+            if (maxCacheSize <= 0 || cacheSize - deleteSize <= maxCacheSize) {
+                return;
+            }
+
+            let needsSorted = false;
+            cacheCount = 0;
+            spinnerMessage = 'Gathering files to delete to satisfy Max cache size';
+
+            await helpers.readDir(this._cachePath, (item) => {
+                if(item.stats.isDirectory()) return next();
+
+                if(item.stats.atime < minFileAccessTime) {
+                    // already expired items are handled in the previous pass
+                    return next();
+                }
+
+                item = {path: item.path, stats: pick(item.stats, ['atime', 'size'])};
+                cacheCount++;
+
+                if (cacheSize - deleteSize >= maxCacheSize) {
                     deleteSize += item.stats.size;
                     deleteItems.push(item);
-                    if(cacheSize - deleteSize <= maxCacheSize) break;
+                    needsSorted = true;
                 }
-            }
-
-            clearTimeout(progressTimer);
-            self.emit('cleanup_search_finish', progressData());
+                else {
+                    if(needsSorted) {
+                        deleteItems.sort((a, b) => { return a.stats.atime > b.stats.atime });
+                        needsSorted = false;
+                    }
 
-            await Promise.all(
-                deleteItems.map(async (d) => {
-                    const guidHash = CacheFS._extractGuidAndHashFromFilepath(d.path);
+                    let i = deleteItems[deleteItems.length - 1]; // i is the MRU out of the current delete list
 
-                    // Make sure we're only deleting valid cached files
-                    if(guidHash.guidStr.length === 0 || guidHash.hashStr.length === 0)
-                        return;
+                    if (item.stats.atime < i.stats.atime) {
+                        deleteItems = helpers.insertSorted(item, deleteItems, (a, b) => {
+                            if (a.stats.atime === b.stats.atime) return 0;
+                            return a.stats.atime < b.stats.atime ? -1 : 1
+                        });
+                        deleteSize += item.stats.size;
 
-                    if(!dryRun) {
-                        await fs.unlink(d.path);
-                        if(this.reliabilityManager !== null) {
-                            this.reliabilityManager.removeEntry(guidHash.guidStr, guidHash.hashStr);
+                        if (cacheSize - (deleteSize - i.stats.size) < maxCacheSize) {
+                            deleteItems.pop();
+                            deleteSize -= i.stats.size;
                         }
                     }
+                }
+            });
+        }).then(async () => {
+            clearTimeout(progressTimer);
+            this.emit('cleanup_search_finish', progressData());
 
-                    self.emit('cleanup_delete_item', d.path);
+            await Promise.all(
+                deleteItems.map(async (d) => {
+                    await this.delete_cache_item(dryRun, d);
                 })
             );
 
-            self.emit('cleanup_delete_finish', progressData());
+            this.emit('cleanup_delete_finish', progressData());
         });
     }
+
+    async delete_cache_item(dryRun = true, item) {
+        const guidHash = CacheFS._extractGuidAndHashFromFilepath(item.path);
+
+        // Make sure we're only deleting valid cached files
+        if(guidHash.guidStr.length === 0 || guidHash.hashStr.length === 0)
+            return;
+
+        if(!dryRun) {
+            await fs.unlink(item.path);
+            if(this.reliabilityManager !== null) {
+                this.reliabilityManager.removeEntry(guidHash.guidStr, guidHash.hashStr);
+            }
+        }
+
+        this.emit('cleanup_delete_item', item.path);
+    }
 }
 
 class PutTransactionFS extends PutTransaction {

lib/helpers.js

@@ -199,3 +199,27 @@ exports.resolveCacheModule = (module, rootPath) => {
     modulePath = path.resolve(rootPath, 'lib/cache', module);
     return require(modulePath);
 };
+
+exports.insertSorted = (item, arr, compare) => {
+    arr.splice(locationOf(item, arr, compare) + 1, 0, item);
+    return arr;
+};
+
+function locationOf(item, array, compare, start, end) {
+    if (array.length === 0)
+        return -1;
+
+    start = start || 0;
+    end = end || array.length;
+
+    let pivot = (start + end) >> 1;
+
+    let c = compare(item, array[pivot]);
+    if (end - start <= 1) return c == -1 ? pivot - 1 : pivot;
+
+    switch (c) {
+        case -1: return locationOf(item, array, compare, start, pivot);
+        case 0: return pivot;
+        case 1: return locationOf(item, array, compare, pivot, end);
+    }
+}

test/cache_fs.js

@@ -196,6 +196,7 @@ describe("Cache: FS", () => {
                 assert(!rmEntry);
             });
         });
+
     });
 
     describe("PutTransaction API", () => {

test/helpers.js

@@ -181,4 +181,27 @@ describe("Helper functions", () => {
             });
         });
     });
+
+    describe("insertSorted", () => {
+        it("should insert an element into the correct position in an array", async () => {
+            let arr = [1, 2, 4, 5];
+            arr = helpers.insertSorted(3, arr, (a, b) => {
+                if (a === b) return 0;
+                return a < b ? -1 : 1
+            });
+
+            assert.equal(arr[2], 3);
+
+        });
+        it("should insert an element into the correct position in an empty array", async () => {
+            let arr = [];
+            arr = helpers.insertSorted(3, arr, (a, b) => {
+                if (a === b) return 0;
+                return a < b ? -1 : 1
+            });
+
+            assert.equal(arr[0], 3);
+
+        });
+    });
 });