diff --git a/cleanup.js b/cleanup.js index 210e74c..3f1d0f3 100644 --- a/cleanup.js +++ b/cleanup.js @@ -82,14 +82,13 @@ cache.on('cleanup_delete_finish', data => { } }); -const msg = 'Gathering cache files for expiration'; let spinner = null; if(logLevel < consts.LOG_DBG && logLevel >= consts.LOG_INFO) { spinner = ora({color: 'white'}); cache.on('cleanup_search_progress', data => { - spinner.text = `${msg} (${data.deleteCount} of ${data.cacheCount} files, ${filesize(data.deleteSize)})`; + spinner.text = `${data.msg} (${data.deleteCount} of ${data.cacheCount} files, ${filesize(data.deleteSize)})`; }); cache.on('cleanup_search_finish', () => { @@ -98,11 +97,12 @@ if(logLevel < consts.LOG_DBG && logLevel >= consts.LOG_INFO) { } else if(logLevel === consts.LOG_DBG) { cache.on('cleanup_search_progress', data => { - const txt = `${msg} (${data.deleteCount} of ${data.cacheCount} files, ${filesize(data.deleteSize)})`; + const txt = `${data.msg} (${data.deleteCount} of ${data.cacheCount} files, ${filesize(data.deleteSize)})`; helpers.log(consts.LOG_DBG, txt); }); } +const msg = 'Gathering cache files for expiration'; function doCleanup() { if (spinner) spinner.start(msg); cache.cleanup(dryRun) diff --git a/lib/cache/cache_fs.js b/lib/cache/cache_fs.js index 7be2145..e7c0a3f 100644 --- a/lib/cache/cache_fs.js +++ b/lib/cache/cache_fs.js @@ -43,7 +43,7 @@ class CacheFS extends CacheBase { const fileName = path.basename(filePath).toLowerCase(); const matches = /^([0-9a-f]{32})-([0-9a-f]{32})\./.exec(fileName); const result = { guidStr: "", hashStr: ""}; - if(matches.length === 3) { + if(matches && matches.length === 3) { result.guidStr = matches[1]; result.hashStr = matches[2]; } @@ -110,78 +110,126 @@ class CacheFS extends CacheBase { } cleanup(dryRun = true) { - const self = this; - const expireDuration = moment.duration(this._options.cleanupOptions.expireTimeSpan); + const minFileAccessTime = moment().subtract(expireDuration).toDate(); + const maxCacheSize = this._options.cleanupOptions.maxCacheSize; + if(!expireDuration.isValid() || expireDuration.asMilliseconds() === 0) { return Promise.reject(new Error("Invalid expireTimeSpan option")); } - const minFileAccessTime = moment().subtract(expireDuration).toDate(); - const maxCacheSize = this._options.cleanupOptions.maxCacheSize; - - const allItems = []; - const deleteItems = []; + let cacheCount = 0; let cacheSize = 0; let deleteSize = 0; + let deletedItemCount = 0; + let deleteItems = []; + let verb = dryRun ? 'Gathering' : 'Removing'; + let spinnerMessage = verb + ' expired files'; const progressData = () => { return { - cacheCount: allItems.length, + cacheCount: cacheCount, cacheSize: cacheSize, - deleteCount: deleteItems.length, - deleteSize: deleteSize + deleteCount: deleteItems.length + deletedItemCount, + deleteSize: deleteSize, + msg: spinnerMessage, }; }; - const progressEvent = () => self.emit('cleanup_search_progress', progressData()); + const progressEvent = () => this.emit('cleanup_search_progress', progressData()); progressEvent(); const progressTimer = setInterval(progressEvent, 250); - return helpers.readDir(self._cachePath, (item) => { - item = {path: item.path, stats: pick(item.stats, ['atime', 'size'])}; - allItems.push(item); + return helpers.readDir(this._cachePath, async (item) => { + + if(item.stats.isDirectory()) return next(); + cacheSize += item.stats.size; + cacheCount ++; + if(item.stats.atime < minFileAccessTime) { deleteSize += item.stats.size; - deleteItems.push(item); + deletedItemCount++; + await this.delete_cache_item(dryRun, item); } }).then(async () => { - if(maxCacheSize > 0 && cacheSize - deleteSize > maxCacheSize) { - allItems.sort((a, b) => { return a.stats.atime > b.stats.atime }); - for(const item of allItems) { + if (maxCacheSize <= 0 || cacheSize - deleteSize <= maxCacheSize) { + return; + } + + let needsSorted = false; + cacheCount = 0; + spinnerMessage = 'Gathering files to delete to satisfy Max cache size'; + + await helpers.readDir(this._cachePath, (item) => { + if(item.stats.isDirectory()) return next(); + + if(item.stats.atime < minFileAccessTime) { + // already expired items are handled in the previous pass + return next(); + } + + item = {path: item.path, stats: pick(item.stats, ['atime', 'size'])}; + cacheCount++; + + if (cacheSize - deleteSize >= maxCacheSize) { deleteSize += item.stats.size; deleteItems.push(item); - if(cacheSize - deleteSize <= maxCacheSize) break; + needsSorted = true; } - } - - clearTimeout(progressTimer); - self.emit('cleanup_search_finish', progressData()); + else { + if(needsSorted) { + deleteItems.sort((a, b) => { return a.stats.atime > b.stats.atime }); + needsSorted = false; + } - await Promise.all( - deleteItems.map(async (d) => { - const guidHash = CacheFS._extractGuidAndHashFromFilepath(d.path); + let i = deleteItems[deleteItems.length - 1]; // i is the MRU out of the current delete list - // Make sure we're only deleting valid cached files - if(guidHash.guidStr.length === 0 || guidHash.hashStr.length === 0) - return; + if (item.stats.atime < i.stats.atime) { + deleteItems = helpers.insertSorted(item, deleteItems, (a, b) => { + if (a.stats.atime === b.stats.atime) return 0; + return a.stats.atime < b.stats.atime ? -1 : 1 + }); + deleteSize += item.stats.size; - if(!dryRun) { - await fs.unlink(d.path); - if(this.reliabilityManager !== null) { - this.reliabilityManager.removeEntry(guidHash.guidStr, guidHash.hashStr); + if (cacheSize - (deleteSize - i.stats.size) < maxCacheSize) { + deleteItems.pop(); + deleteSize -= i.stats.size; } } + } + }); + }).then(async () => { + clearTimeout(progressTimer); + this.emit('cleanup_search_finish', progressData()); - self.emit('cleanup_delete_item', d.path); + await Promise.all( + deleteItems.map(async (d) => { + await this.delete_cache_item(dryRun, d); }) ); - self.emit('cleanup_delete_finish', progressData()); + this.emit('cleanup_delete_finish', progressData()); }); } + + async delete_cache_item(dryRun = true, item) { + const guidHash = CacheFS._extractGuidAndHashFromFilepath(item.path); + + // Make sure we're only deleting valid cached files + if(guidHash.guidStr.length === 0 || guidHash.hashStr.length === 0) + return; + + if(!dryRun) { + await fs.unlink(item.path); + if(this.reliabilityManager !== null) { + this.reliabilityManager.removeEntry(guidHash.guidStr, guidHash.hashStr); + } + } + + this.emit('cleanup_delete_item', item.path); + } } class PutTransactionFS extends PutTransaction { diff --git a/lib/helpers.js b/lib/helpers.js index ef1c3aa..ceb06d9 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -199,3 +199,27 @@ exports.resolveCacheModule = (module, rootPath) => { modulePath = path.resolve(rootPath, 'lib/cache', module); return require(modulePath); }; + +exports.insertSorted = (item, arr, compare) => { + arr.splice(locationOf(item, arr, compare) + 1, 0, item); + return arr; +}; + +function locationOf(item, array, compare, start, end) { + if (array.length === 0) + return -1; + + start = start || 0; + end = end || array.length; + + let pivot = (start + end) >> 1; + + let c = compare(item, array[pivot]); + if (end - start <= 1) return c == -1 ? pivot - 1 : pivot; + + switch (c) { + case -1: return locationOf(item, array, compare, start, pivot); + case 0: return pivot; + case 1: return locationOf(item, array, compare, pivot, end); + } +} \ No newline at end of file diff --git a/test/cache_fs.js b/test/cache_fs.js index fc4ef49..f996c62 100644 --- a/test/cache_fs.js +++ b/test/cache_fs.js @@ -196,6 +196,7 @@ describe("Cache: FS", () => { assert(!rmEntry); }); }); + }); describe("PutTransaction API", () => { diff --git a/test/helpers.js b/test/helpers.js index 6f5d154..e431c42 100644 --- a/test/helpers.js +++ b/test/helpers.js @@ -181,4 +181,27 @@ describe("Helper functions", () => { }); }); }); + + describe("insertSorted", () => { + it("should insert an element into the correct position in an array", async () => { + let arr = [1, 2, 4, 5]; + arr = helpers.insertSorted(3, arr, (a, b) => { + if (a === b) return 0; + return a < b ? -1 : 1 + }); + + assert.equal(arr[2], 3); + + }); + it("should insert an element into the correct position in an empty array", async () => { + let arr = []; + arr = helpers.insertSorted(3, arr, (a, b) => { + if (a === b) return 0; + return a < b ? -1 : 1 + }); + + assert.equal(arr[0], 3); + + }); + }); }); \ No newline at end of file