Merge pull request #264 from stelcheck/vaults/redis

redis vault

docs-sources/includes/_Archivist.md

@@ -85,6 +85,7 @@ targets are supported:
 | client                | Special vault type (client-side archivist support required). |
 | couchbase             | [Couchbase](https://www.couchbase.com/) interface            |
 | mysql                 | [MySQL](https://www.mysql.com/) interface.                   |
+| redis                 | [Redis](https://redis.io/) interface.                        |
 
 Vaults can have different configuration for different environments, as long as the Archivist
 API set used in your project is provided by the different vault backends you wish to use.
@@ -279,7 +280,29 @@ If you want to change how this information is stored, by adding columns, etc, yo
 serializer method to do so. For example, consider the following example if you want to add a
 timestamp to a `lastChanged INT UNSIGNED NOT NULL` column.
 
-<br><br><br><br><br><br><br><br><br><br><br><br><br><br>
+### Redis
+
+ ```yaml
+type: redis
+config:
+    port: 6379
+    host: "127.0.0.1"
+    options: {}
+    prefix: "key/prefix/"
+```
+ The `options` object is described in the [node-redis readme](https://npmjs.org/package/redis).
+Both `options` and `prefix` are optional. The option `return_buffers` is turned on by default by the
+Archivist, because the default serialization will prepend values with meta data (in order to
+preserve mediaType awareness).
+
+ operation | supported | implementation
+----------|:---------:|---------------
+list      |           |
+get       | ✔         | `redis.get()`
+add       | ✔         | `redis.set('NX')`
+set       | ✔         | `redis.set()`
+touch     | ✔         | `redis.expire()`
+del       | ✔         | `redis.del()`
 
 ## Topics
 

lib/archivist/configuration.js

@@ -718,12 +718,12 @@ exports.openVaults = function (_logger, cb) {
 			vault.open((err) => {
 				if (err) {
 					logger.alert
-					.details('Early errors may mean one of the following:')
-					.details('  1. The remote server is down or inaccessible')
-					.details('  2. You need to run `npm run archivist:create` to create')
-					.details('     and configure storage for your vault backend')
-					.details('See error data for more details.')
-					.log(`Failed to set up vault ${vaultName}`);
+						.details('Early errors may mean one of the following:')
+						.details('  1. The remote server is down or inaccessible')
+						.details('  2. You need to run `npm run archivist:create` to create')
+						.details('     and configure storage for your vault backend')
+						.details('See error data for more details.')
+						.log(`Failed to set up vault ${vaultName}`);
 				}
 
 				return callback(err);

lib/archivist/index.js

@@ -33,13 +33,8 @@ exports.openVaults = function (cb) {
 exports.listPeerDependencies = function () {
 	return {
 		'Archivist Couchbase vault': ['couchbase'],
-		'Archivist DynamoDB vault': ['aws-sdk'],
-		'Archivist Elasticsearch vault': ['es'],
-		'Archivist Manta vault': ['manta', 'memorystream'],
-		'Archivist Memcached vault': ['memcached'],
 		'Archivist MySQL vault': ['mysql'],
-		'Archivist Redis vault': ['redis'],
-		'Archivist SQLite vault': ['sqlite3']
+		'Archivist Redis vault': ['redis']
 	};
 };
 

lib/archivist/vaults/couchbase/index.js

@@ -18,9 +18,9 @@ var Archive = require('./Archive');
 
 exports.defaultTopicApi = require('./defaultTopicApi');
 
- /**
-  * UninitializedError class
-  */
+/**
+ * UninitializedError class
+ */
 class UninitializedError extends Error {
 	/**
 	 * Constructor

lib/archivist/vaults/mysql/index.js

@@ -14,9 +14,9 @@ var Archive = require('./Archive');
 
 exports.defaultTopicApi = require('./defaultTopicApi');
 
- /**
-  * UninitializedError class
-  */
+/**
+ * UninitializedError class
+ */
 class UninitializedError extends Error {
 	/**
 	 * Constructor

lib/archivist/vaults/redis/Archive.js

@@ -0,0 +1,44 @@
+// Archivist bindings for the RedisVault API
+
+function Archive(vault) {
+	this.vault = vault;
+}
+
+
+module.exports = Archive;
+
+
+Archive.prototype.get = function (api, value, cb) {
+	this.vault.get(api.createKey(value.topic, value.index), function (error, data) {
+		if (error) {
+			return cb(error);
+		}
+
+		if (data === undefined) {
+			return cb();
+		}
+
+		try {
+			api.deserialize(data, value);
+		} catch (error) {
+			return cb(error);
+		}
+
+		return cb();
+	});
+};
+
+
+Archive.prototype.set = function (api, value, cb) {
+	this.vault.set(api.createKey(value.topic, value.index), api.serialize(value), value.expirationTime, cb);
+};
+
+
+Archive.prototype.touch = function (api, value, cb) {
+	this.vault.touch(api.createKey(value.topic, value.index), value.expirationTime, cb);
+};
+
+
+Archive.prototype.del = function (api, value, cb) {
+	this.vault.del(api.createKey(value.topic, value.index), cb);
+};

lib/archivist/vaults/redis/defaultTopicApi.js

@@ -0,0 +1,69 @@
+exports.serialize = function (value) {
+	// throws exceptions on failure
+
+	// the header is:
+	// - 2 bytes to indicate JSON-length in bytes (65535 max header length)
+	// - JSON containing: { mediaType: "a/b" }
+	// - a single byte "\n" delimiter
+
+	// serialize the value to a buffer
+
+	var data = value.setEncoding(['buffer']).data;
+
+	// create the meta data
+
+	var meta = JSON.stringify({
+		mediaType: value.mediaType
+	});
+
+	// create a buffer that will fit both the header and the value
+
+	var jsonSize = Buffer.byteLength(meta);
+	var headerSize = 2 + jsonSize + 1;
+
+	var output = new Buffer(headerSize + data.length);
+
+	// write the meta JSON length
+
+	output.writeUInt16BE(jsonSize, 0);
+
+	// write the meta JSON and trailing \n
+
+	output.write(meta + '\n', 2, jsonSize + 1, 'utf8');
+
+	// append the value
+
+	data.copy(output, headerSize);
+
+	return output;
+};
+
+
+exports.deserialize = function (data, value) {
+	var jsonSize = data.readUInt16BE(0);
+	var meta = JSON.parse(data.toString('utf8', 2, 2 + jsonSize));
+
+	data = data.slice(2 + jsonSize + 1);
+
+	value.setDataFromVault(meta.mediaType, data, 'buffer');
+};
+
+
+exports.createKey = function (topic, index) {
+	// eg: weapons/actorId:123/bag:main
+	// eg: weapons/guildId:123
+
+	var key = topic;
+	var props, i;
+
+	if (index) {
+		props = Object.keys(index);
+		props.sort();
+
+		for (i = 0; i < props.length; i++) {
+			key += '/' + props[i] + ':' + index[props[i]];
+		}
+	}
+
+	return key;
+};

lib/archivist/vaults/redis/index.js

@@ -0,0 +1,151 @@
+// based on node-redis, this vault does not support sharding at this time
+//
+// key format: string
+//
+// references:
+// -----------
+// node-redis: https://github.com/mranney/node_redis
+
+var requirePeer = require('codependency').get('mage');
+var redis = requirePeer('redis');
+var Archive = require('./Archive');
+
+
+exports.defaultTopicApi = require('./defaultTopicApi');
+
+
+// Vault wrapper around node-redis
+
+function RedisVault(name, logger) {
+	// required exposed properties
+
+	this.name = name;                  // the unique vault name
+	this.archive = new Archive(this);  // archivist bindings
+
+	this.client = null;                // node-redis instance
+	this.logger = logger;
+}
+
+
+exports.create = function (name, logger) {
+	return new RedisVault(name, logger);
+};
+
+
+RedisVault.prototype.setup = function (cfg, cb) {
+	var cfgOptions = cfg.options || {};
+
+	// this option-alias is really just to silence jshint
+
+	var returnBuffers = 'return_buffers';
+
+	if (!cfgOptions.hasOwnProperty(returnBuffers)) {
+		cfgOptions[returnBuffers] = true;
+	}
+
+	this.client = redis.createClient(cfg.port, cfg.host, cfgOptions);
+	this.keyPrefix = cfg.prefix || null;
+	this.db = cfg.db || null;
+
+	var logger = this.logger;
+	var name = this.name;
+
+	this.client.on('error', function (error) {
+		logger.emergency('Error on Redis server (name: ' + name + '):', error);
+	});
+
+	setImmediate(cb);
+};
+
+RedisVault.prototype.open = function (cb) {
+	if (!this.db) {
+		return setImmediate(cb);
+	}
+
+	this.client.select(this.db, cb);
+};
+
+RedisVault.prototype.close = function () {
+	this.logger.verbose('Closing vault:', this.name);
+
+	if (this.client) {
+		this.client.quit();
+		this.client = null;
+	}
+};
+
+
+RedisVault.prototype._prefix = function (key) {
+	return this.keyPrefix ? this.keyPrefix + key : key;
+};
+
+/* unprefix will be used once we support readMany
+RedisVault.prototype._unprefix = function (key) {
+	if (!this.keyPrefix) {
+		return key;
+	}
+	var len = this.keyPrefix.length;
+	if (key.substr(0, len) !== this.keyPrefix) {
+		throw new Error('Could not unprefix key "' + key + '" with prefix "' + this.keyPrefix + '"');
+	}
+	return key.substr(len);
+};
+*/
+
+RedisVault.prototype.get = function (key, cb) {
+	key = this._prefix(key);
+
+	this.logger.verbose('get:', key);
+
+	this.client.get([key], function (error, data) {
+		if (error) {
+			return cb(error);
+		}
+
+		data = data || undefined;
+
+		cb(null, data);
+	});
+};
+
+
+function expirationTimeToTTL(expirationTime) {
+	if (expirationTime) {
+		return expirationTime - Math.ceil(Date.now() / 1000);
+	}
+}
+
+
+RedisVault.prototype.set = function (key, data, expirationTime, cb) {
+	key = this._prefix(key);
+
+	this.logger.verbose('set:', key);
+
+	var cmd = expirationTime ?
+		[key, data, 'EX', expirationTimeToTTL(expirationTime)] :
+		[key, data];
+
+	this.client.set(cmd, cb);
+};
+
+
+RedisVault.prototype.touch = function (key, expirationTime, cb) {
+	key = this._prefix(key);
+
+	this.logger.verbose('touch:', key);
+
+	if (!expirationTime) {
+		return cb();
+	}
+
+	this.client.expire([key, expirationTimeToTTL(expirationTime)], cb);
+};
+
+
+RedisVault.prototype.del = function (key, cb) {
+	key = this._prefix(key);
+
+	this.logger.verbose('del:', key);
+
+	this.client.del([key], cb);
+};

lib/config/config.js

@@ -126,7 +126,7 @@ function loadConfigFile(configPath) {
  * @returns {Object}
  */
 function loadEnvironment(config) {
-	if (typeof (config) !== 'object') {
+	if (typeof(config) !== 'object') {
 		return {};
 	}
 
@@ -150,7 +150,7 @@ function loadEnvironment(config) {
 
 		var val = config[key];
 
-		switch (typeof (val)) {
+		switch (typeof(val)) {
 		case 'object':
 			finalConf[key] = loadEnvironment(val);
 			break;
@@ -478,4 +478,4 @@ exports.getConfigList = function () {
 
 exports.setLogger = function (loggerInstance) {
 	logger = loggerInstance.context('config');
-};
+};