Merge 62a5aa7 into 02f53ec

README.md

@@ -61,7 +61,6 @@ Command                          | Description
 `-l`, `--log-level <n>`          | The level of log verbosity. Valid values are 0 (silent) through 5 (debug). The default is 3.
 `-w`, `--workers <n>`            | The number of worker threads to spawn. The default is 0.
 `-m`, `--mirror [host:port]`     | Mirror transactions to another cache server. Repeat this option for multiple mirrors.
-`-m`, `--monitor-parent-process <n>` | Monitor a parent process and exit if it dies.
 `--dump-config`                  | Write the active configuration to the console.
 `--save-config [path]`           | Write the active configuration to the specified file and exit. Defaults to `./default.yml`.
 `--NODE_CONFIG_DIR=<path>`       | The directory to search for config files. This is equivalent to setting the `NODE_CONFIG_DIR` environment variable. If not specified, the built-in configuration is used.
@@ -76,7 +75,10 @@ By default, running `unity-cache-server` uses the built-in configuration file. T
 ### General Options
 Option                              | Default     | Description
 ----------------------------------- | ----------- | -----------
+Global.logLevel                     |3            | Logging level; override with the --log-level CLI command
 Cache.options.processor.putWhitelist|[]           | Only allow PUT transactions (uploads) from the specified array of IP addresses (string values)
+Cache.options.workers               |1            | Number of worker threads; override with the --worker CLI command
+Server.port                         |8126         | The port on which the Cache Server listens. Override with the --port CLI command
 Server.options.allowIPv6            |false        | Listen for client connections on both IPv4 and IPv6
 #### Examples (Mac/Linux)
 

cleanup.js

@@ -1,19 +1,12 @@
 #!/usr/bin/env node
 const helpers = require('./lib/helpers');
 helpers.initConfigDir(__dirname);
-const config = require('config');
 
 const consts = require('./lib/constants');
-const program = require('commander');
+const cmd = require('commander');
 const moment = require('moment');
-const filesize =require('filesize');
-const ora = require('ora');
-const VERSION = require('./package.json').version;
-
-function myParseInt(val, def) {
-    val = parseInt(val);
-    return (!val && val !== 0) ? def : val;
-}
+const { version } = require('./package.json');
+const { UnityCacheServer } = require('./lib/unity_cache_server');
 
 function parseTimeSpan(val) {
     if(!moment.duration(val).isValid())
@@ -25,97 +18,59 @@ function parseTimeSpan(val) {
     return val;
 }
 
-const defaultCacheModule = config.get("Cache.defaultModule");
-
-program.description("Unity Cache Server - Cache Cleanup\n\n  Removes old files from supported cache modules.")
-    .version(VERSION)
-    .allowUnknownOption(true)
-    .option('-c --cache-module [path]', 'Use cache module at specified path', defaultCacheModule)
-    .option('-P, --cache-path [path]', 'Specify the path of the cache directory')
-    .option('-l, --log-level <n>', 'Specify the level of log verbosity. Valid values are 0 (silent) through 5 (debug)', myParseInt, consts.DEFAULT_LOG_LEVEL)
-    .option('-e, --expire-time-span <timeSpan>', 'Override the configured file expiration timespan. Both ASP.NET style time spans (days.minutes:hours:seconds, e.g. \'15.23:59:59\') and ISO 8601 time spans (e.g. \'P15DT23H59M59S\') are supported.', parseTimeSpan)
-    .option('-s, --max-cache-size <bytes>', 'Override the configured maximum cache size. Files will be removed from the cache until the max cache size is satisfied, using a Least Recently Used search. A value of 0 disables this check.', myParseInt)
-    .option('-d, --delete', 'Delete cached files that match the configured criteria. Without this, the default behavior is to dry-run which will print diagnostic information only.')
-    .option('-D, --daemon <interval>', 'Daemon mode: execute the cleanup script at the given interval in seconds as a foreground process.', myParseInt)
-    .option('--NODE_CONFIG_DIR=<path>', 'Specify the directory to search for config files. This is equivalent to setting the NODE_CONFIG_DIR environment variable. Without this option, the built-in configuration is used.');
-
-program.parse(process.argv);
-
-helpers.setLogLevel(program.logLevel);
-
-const CacheModule = helpers.resolveCacheModule(program.cacheModule, __dirname);
-
-if(!CacheModule.properties.cleanup) {
-    helpers.log(consts.LOG_ERR, "Configured cache module does not support cleanup script.");
-    process.exit(1);
-}
+const optionMap = {
+    cacheModule: {
+        flags: "-c --cache-module <path>",
+        description: "Use cache module at specified path",
+        configKey: consts.CLI_CONFIG_KEYS.CACHE_MODULE
+    },
+    cachePath: {
+        flags: "-P, --cache-path <path>",
+        description: "Specify the path of the cache directory",
+        configKey: consts.CLI_CONFIG_KEYS.CACHE_PATH
+    },
+    expireTimeSpan: {
+        flags: "-e, --expire-time-span <timeSpan>",
+        description: "Override the configured file expiration timespan. Both ASP.NET style time spans (days.minutes:hours:seconds, e.g. \'15.23:59:59\') and ISO 8601 time spans (e.g. \'P15DT23H59M59S\') are supported.', parseTimeSpan)",
+        validator: parseTimeSpan
+    },
+    maxCacheSize: {
+        flags: "-s, --max-cache-size <bytes>",
+        description: "Override the configured maximum cache size. Files will be removed from the cache until the max cache size is satisfied, using a Least Recently Used search. A value of 0 disables this check.",
+        validator: parseInt,
+        default: 0
+    },
+    delete: {
+        flags: "-d, --delete",
+        description: "Delete cached files that match the configured criteria. Without this, the default behavior is to dry-run which will print diagnostic information only.",
+        defaultValue: false
+    },
+    daemon: {
+        flags: "-D, --daemon <interval>",
+        description: "Daemon mode: execute the cleanup script at the given interval in seconds as a foreground process.",
+        validator: parseInt
+    }
+};
 
-const cache = new CacheModule();
+cmd.description("Unity Cache Server - Cache Cleanup\n\n  Removes old files from supported cache modules.").version(version).allowUnknownOption(true);
+UnityCacheServer.handleCommandLine(cmd, optionMap);
 
 const cacheOpts = { cleanupOptions: {} };
-
-if(program.cachePath !== null) {
-    cacheOpts.cachePath = program.cachePath;
+if(cmd.expireTimeSpan !== null) {
+    cacheOpts.cleanupOptions.expireTimeSpan = cmd.expireTimeSpan;
 }
 
-if(program.hasOwnProperty('expireTimeSpan')) {
-    cacheOpts.cleanupOptions.expireTimeSpan = program.expireTimeSpan;
+if(cmd.maxCacheSize > 0) {
+    cacheOpts.cleanupOptions.maxCacheSize = cmd.maxCacheSize;
 }
 
-if(program.hasOwnProperty('maxCacheSize')) {
-    cacheOpts.cleanupOptions.maxCacheSize = program.maxCacheSize;
-}
-
-const dryRun = !program.delete;
-const logLevel = helpers.getLogLevel();
-
-cache._options = cacheOpts;
-helpers.log(consts.LOG_INFO, `Cache path is ${cache._cachePath}`);
-
-cache.on('cleanup_delete_item', item => helpers.log(consts.LOG_DBG, item));
+const dryRun = !cmd.delete;
+const daemon = cmd.hasOwnProperty('daemon') ? cmd.daemon : 0;
 
-cache.on('cleanup_delete_finish', data => {
-    const pct = data.cacheSize > 0 ? (data.deleteSize/data.cacheSize).toPrecision(2) * 100 : 0;
-    helpers.log(consts.LOG_INFO, `Found ${data.deleteCount} expired files of ${data.cacheCount}. ${filesize(data.deleteSize)} of ${filesize(data.cacheSize)} (${pct}%).`);
-    if(dryRun) {
-        helpers.log(consts.LOG_INFO, "Nothing deleted; run with --delete to remove expired files from the cache.");
-    }
-});
-
-let spinner = null;
-
-if(logLevel < consts.LOG_DBG && logLevel >= consts.LOG_INFO) {
-    spinner = ora({color: 'white'});
-
-    cache.on('cleanup_search_progress', data => {
-        spinner.text = `${data.msg} (${data.deleteCount} of ${data.cacheCount} files, ${filesize(data.deleteSize)})`;
-    });
-
-    cache.on('cleanup_search_finish', () => {
-        spinner.stop();
-    });
-
-} else if(logLevel === consts.LOG_DBG) {
-    cache.on('cleanup_search_progress', data => {
-        const txt = `${data.msg} (${data.deleteCount} of ${data.cacheCount} files, ${filesize(data.deleteSize)})`;
-        helpers.log(consts.LOG_DBG, txt);
+UnityCacheServer.initCache(cacheOpts)
+    .then(() => UnityCacheServer.cleanup(dryRun, daemon * 1000))
+    .then(() => process.exit(0))
+    .catch((err) => {
+        console.log(err);
+        process.exit(1);
     });
-}
-
-const msg = 'Gathering cache files for expiration';
-function doCleanup() {
-    if (spinner) spinner.start(msg);
-    cache.cleanup(dryRun)
-        .catch(err => {
-            if (spinner) spinner.stop();
-            helpers.log(consts.LOG_ERR, err);
-            process.exit(1);
-        });
-}
-
-if(program.hasOwnProperty('daemon') && program.daemon > 0) {
-    setInterval(doCleanup, program.daemon * 1000);
-}
-else {
-    doCleanup();
-}

config/default.yml

@@ -1,4 +1,6 @@
 Cache:
+  ## the cachePath key, if set, will override 'cachePath' within the individual module options below
+  #cachePath: ".cache"
   defaultModule: "cache_fs"
   options:
     processor:
@@ -34,9 +36,13 @@ Cache:
         saveUnreliableVersionArtifacts: true
         multiClient: false
 Mirror:
+  addresses: []
   options:
     queueProcessDelay: 2000
     connectionIdleTimeout: 10000
 Server:
+  port: 8126
   options:
-    allowIpv6: false
+    allowIpv6: false
+Global:
+  logLevel: 3

config/local-test.yml

@@ -0,0 +1,6 @@
+Cache:
+  options:
+    cache_fs:
+      persistence: false
+    cache_ram:
+      persistence: false

lib/cache/cache_base.js

@@ -3,7 +3,6 @@ const EventEmitter = require('events');
 const cluster = require('cluster');
 const consts = require('../constants');
 const helpers = require('../helpers');
-const config = require('config');
 const path = require('path');
 const fs = require('fs-extra');
 const defaultsDeep = require('lodash').defaultsDeep;
@@ -32,7 +31,7 @@ class CacheBase extends EventEmitter {
     }
 
     get _options() {
-        const opts = config.get(this._optionsPath);
+        const opts = require('config').get(this._optionsPath);
         return defaultsDeep(this._optionOverrides, opts);
     }
 
@@ -134,7 +133,7 @@ class CacheBase extends EventEmitter {
 
     /**
      *
-     * @returns {Promise<any>}
+     * @returns {Promise<null>}
      */
     async shutdown() {
         if(!cluster.isMaster) return Promise.resolve();

lib/cache/cache_fs.js

@@ -96,7 +96,7 @@ class CacheFS extends CacheBase {
         const self = this;
         await super.endPutTransaction(transaction);
 
-        await Promise.all(
+        return Promise.all(
             transaction.files.map(async (file) => {
                 try {
                     const filePath = await self._writeFileToCache(file.type, transaction.guid, transaction.hash, file.file);
@@ -109,7 +109,7 @@ class CacheFS extends CacheBase {
         );
     }
 
-    cleanup(dryRun = true) {
+    async cleanup(dryRun = true) {
         const expireDuration = moment.duration(this._options.cleanupOptions.expireTimeSpan);
         const minFileAccessTime = moment().subtract(expireDuration).toDate();
         const maxCacheSize = this._options.cleanupOptions.maxCacheSize;
@@ -141,31 +141,34 @@ class CacheFS extends CacheBase {
         progressEvent();
         const progressTimer = setInterval(progressEvent, 250);
 
-        return helpers.readDir(this._cachePath, async (item) => {
-
-            if(item.stats.isDirectory()) return next();
+        await helpers.readDir(this._cachePath, (item) => {
+            if(item.stats.isDirectory()) return;
 
             cacheSize += item.stats.size;
             cacheCount ++;
 
             if(item.stats.atime < minFileAccessTime) {
                 deleteSize += item.stats.size;
                 deletedItemCount++;
-                await this.delete_cache_item(dryRun, item);
-            }
-        }).then(async () => {
-            if (maxCacheSize <= 0 || cacheSize - deleteSize <= maxCacheSize) {
-                return;
+                deleteItems.push(item);
             }
+        });
+
+        await Promise.all(
+            deleteItems.map(d => this.delete_cache_item(dryRun, d))
+        );
+
+        deleteItems.length = 0;
 
+        if (maxCacheSize > 0 && cacheSize - deleteSize > maxCacheSize) {
             let needsSorted = false;
             cacheCount = 0;
             spinnerMessage = 'Gathering files to delete to satisfy Max cache size';
 
             await helpers.readDir(this._cachePath, (item) => {
-                if(item.stats.isDirectory()) return next();
+                if (item.stats.isDirectory()) return next();
 
-                if(item.stats.atime < minFileAccessTime) {
+                if (item.stats.atime < minFileAccessTime) {
                     // already expired items are handled in the previous pass
                     return next();
                 }
@@ -179,8 +182,12 @@ class CacheFS extends CacheBase {
                     needsSorted = true;
                 }
                 else {
-                    if(needsSorted) {
-                        deleteItems.sort((a, b) => { return a.stats.atime > b.stats.atime });
+                    if (needsSorted) {
+                        deleteItems.sort((a, b) => {
+                            if (a.stats.atime === b.stats.atime) return 0;
+                            return a.stats.atime > b.stats.atime ? 1 : -1
+                        });
+
                         needsSorted = false;
                     }
 
@@ -200,18 +207,16 @@ class CacheFS extends CacheBase {
                     }
                 }
             });
-        }).then(async () => {
-            clearTimeout(progressTimer);
-            this.emit('cleanup_search_finish', progressData());
+        }
 
-            await Promise.all(
-                deleteItems.map(async (d) => {
-                    await this.delete_cache_item(dryRun, d);
-                })
-            );
+        clearTimeout(progressTimer);
+        this.emit('cleanup_search_finish', progressData());
 
-            this.emit('cleanup_delete_finish', progressData());
-        });
+        await Promise.all(
+            deleteItems.map(d => this.delete_cache_item(dryRun, d))
+        );
+
+        this.emit('cleanup_delete_finish', progressData());
     }
 
     async delete_cache_item(dryRun = true, item) {

lib/client/client.js

@@ -82,7 +82,7 @@ class CacheClient {
 
     /**
      *
-     * @returns {Promise<any>}
+     * @returns {Promise<null>}
      */
     quit() {
         if(!this._client) return Promise.resolve();

lib/constants.js

@@ -18,6 +18,15 @@ const constants = {
         INFO: 'i',
         BIN: 'a',
         RESOURCE: 'r'
+    },
+    CLI_CONFIG_KEYS: {
+        CACHE_MODULE: "Cache.defaultModule",
+        CACHE_PATH: "Cache.cachePath",
+        WORKERS: "Cache.options.workers",
+        LOG_LEVEL: "Global.logLevel",
+        PORT: "Server.port",
+        MIRROR: "Mirror.addresses",
+        ALLOW_IP_V6: "Server.options.allowIpv6"
     }
 };