From 44642dc802b9d70be1c1f4c1c55dbc0d06cf4876 Mon Sep 17 00:00:00 2001 From: Behrad Date: Tue, 12 Apr 2016 15:57:37 +0430 Subject: [PATCH 01/27] use QlobberDedup to prevent matching before adding subs --- lib/persistence/levelup.js | 9 +++------ lib/persistence/matcher.js | 2 +- lib/persistence/mongo.js | 2 +- lib/persistence/redis.js | 12 ++++-------- 4 files changed, 9 insertions(+), 16 deletions(-) diff --git a/lib/persistence/levelup.js b/lib/persistence/levelup.js index 4a1ce03..768cf93 100644 --- a/lib/persistence/levelup.js +++ b/lib/persistence/levelup.js @@ -149,7 +149,7 @@ LevelUpPersistence.prototype.lookupRetained = function(pattern, cb) { }); stream.on("data", function(data) { - if (matcher.match(data.key).length > 0) { + if (matcher.match(data.key).size > 0) { matched.push(data.value); } }); @@ -176,9 +176,7 @@ LevelUpPersistence.prototype.storeSubscriptions = function(client, done) { qos: subscriptions[key].qos }; var levelKey = util.format("%s:%s", key, client.id); - if (that._subMatcher.match(key).indexOf(levelKey) < 0) { - that._subMatcher.add(key, levelKey); - } + that._subMatcher.add(key, levelKey); that._subscriptions.put(levelKey, sub); }); } else if (done) { @@ -232,8 +230,7 @@ LevelUpPersistence.prototype.lookupSubscriptions = function(client, done) { LevelUpPersistence.prototype.storeOfflinePacket = function(packet, done) { var that = this; var subs = this._subMatcher.match(packet.topic); - - async.each(subs, function(key, cb) { + async.each(Array.from(subs), function(key, cb) { that._subscriptions.get(key, function(err, sub) { if (err) { return cb(err); diff --git a/lib/persistence/matcher.js b/lib/persistence/matcher.js index 51af830..c1da680 100644 --- a/lib/persistence/matcher.js +++ b/lib/persistence/matcher.js @@ -23,7 +23,7 @@ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -var Qlobber = require("qlobber").Qlobber; +var Qlobber = require("qlobber").QlobberDedup; var util = require("util"); function Matcher() { diff --git a/lib/persistence/mongo.js b/lib/persistence/mongo.js index 5b31310..fcc8a0e 100644 --- a/lib/persistence/mongo.js +++ b/lib/persistence/mongo.js @@ -279,7 +279,7 @@ MongoPersistence.prototype.lookupRetained = function(pattern, cb) { }); stream.on("data", function(data) { - if (matcher.match(data.topic).length > 0) { + if (matcher.match(data.topic).size > 0) { data.payload = data.payload.buffer; matched.push(data); } diff --git a/lib/persistence/redis.js b/lib/persistence/redis.js index b460033..070604e 100644 --- a/lib/persistence/redis.js +++ b/lib/persistence/redis.js @@ -105,9 +105,7 @@ function RedisPersistence(options, callback) { } Object.keys(subs).forEach(function(sub) { - if (that._subMatcher.match(sub).indexOf(id) < 0) { - that._subMatcher.add(sub, id); - } + that._subMatcher.add(sub, id); }); if( unsubs ) { @@ -229,7 +227,7 @@ RedisPersistence.prototype.lookupRetained = function(pattern, done) { this._client.hkeys("retained", function(err, topics) { topics.sort(); topics = topics.filter(function(topic) { - return matcher.match(topic).length > 0; + return matcher.match(topic).size > 0; }); async.each(topics, match, function(err) { @@ -286,9 +284,7 @@ RedisPersistence.prototype.storeSubscriptions = function(client, cb) { .pexpire(clientSubKey, this.options.ttl.subscriptions); Object.keys(subscriptions).forEach(function(e) { - if (that._subMatcher.match(e).indexOf(client.id) < 0) { - that._subMatcher.add(e, client.id); - } + that._subMatcher.add(e, client.id); }); op.exec(cb); @@ -353,7 +349,7 @@ RedisPersistence.prototype.storeOfflinePacket = function(packet, done) { var that = this; var matches = this._subMatcher.match(packet.topic); - async.each(matches, function(client, cb) { + async.each(Array.from(matches), function(client, cb) { that._storePacket(client, packet, cb); }, done); }; From 1aefcf891cbfd59e4a19f0a6c1f0b99a8ba1f086 Mon Sep 17 00:00:00 2001 From: Behrad Date: Tue, 12 Apr 2016 15:57:37 +0430 Subject: [PATCH 02/27] use QlobberDedup to prevent matching before adding subs --- lib/persistence/levelup.js | 9 +++------ lib/persistence/matcher.js | 2 +- lib/persistence/mongo.js | 2 +- lib/persistence/redis.js | 12 ++++-------- 4 files changed, 9 insertions(+), 16 deletions(-) diff --git a/lib/persistence/levelup.js b/lib/persistence/levelup.js index 4a1ce03..768cf93 100644 --- a/lib/persistence/levelup.js +++ b/lib/persistence/levelup.js @@ -149,7 +149,7 @@ LevelUpPersistence.prototype.lookupRetained = function(pattern, cb) { }); stream.on("data", function(data) { - if (matcher.match(data.key).length > 0) { + if (matcher.match(data.key).size > 0) { matched.push(data.value); } }); @@ -176,9 +176,7 @@ LevelUpPersistence.prototype.storeSubscriptions = function(client, done) { qos: subscriptions[key].qos }; var levelKey = util.format("%s:%s", key, client.id); - if (that._subMatcher.match(key).indexOf(levelKey) < 0) { - that._subMatcher.add(key, levelKey); - } + that._subMatcher.add(key, levelKey); that._subscriptions.put(levelKey, sub); }); } else if (done) { @@ -232,8 +230,7 @@ LevelUpPersistence.prototype.lookupSubscriptions = function(client, done) { LevelUpPersistence.prototype.storeOfflinePacket = function(packet, done) { var that = this; var subs = this._subMatcher.match(packet.topic); - - async.each(subs, function(key, cb) { + async.each(Array.from(subs), function(key, cb) { that._subscriptions.get(key, function(err, sub) { if (err) { return cb(err); diff --git a/lib/persistence/matcher.js b/lib/persistence/matcher.js index 51af830..c1da680 100644 --- a/lib/persistence/matcher.js +++ b/lib/persistence/matcher.js @@ -23,7 +23,7 @@ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -var Qlobber = require("qlobber").Qlobber; +var Qlobber = require("qlobber").QlobberDedup; var util = require("util"); function Matcher() { diff --git a/lib/persistence/mongo.js b/lib/persistence/mongo.js index 5b31310..fcc8a0e 100644 --- a/lib/persistence/mongo.js +++ b/lib/persistence/mongo.js @@ -279,7 +279,7 @@ MongoPersistence.prototype.lookupRetained = function(pattern, cb) { }); stream.on("data", function(data) { - if (matcher.match(data.topic).length > 0) { + if (matcher.match(data.topic).size > 0) { data.payload = data.payload.buffer; matched.push(data); } diff --git a/lib/persistence/redis.js b/lib/persistence/redis.js index b460033..070604e 100644 --- a/lib/persistence/redis.js +++ b/lib/persistence/redis.js @@ -105,9 +105,7 @@ function RedisPersistence(options, callback) { } Object.keys(subs).forEach(function(sub) { - if (that._subMatcher.match(sub).indexOf(id) < 0) { - that._subMatcher.add(sub, id); - } + that._subMatcher.add(sub, id); }); if( unsubs ) { @@ -229,7 +227,7 @@ RedisPersistence.prototype.lookupRetained = function(pattern, done) { this._client.hkeys("retained", function(err, topics) { topics.sort(); topics = topics.filter(function(topic) { - return matcher.match(topic).length > 0; + return matcher.match(topic).size > 0; }); async.each(topics, match, function(err) { @@ -286,9 +284,7 @@ RedisPersistence.prototype.storeSubscriptions = function(client, cb) { .pexpire(clientSubKey, this.options.ttl.subscriptions); Object.keys(subscriptions).forEach(function(e) { - if (that._subMatcher.match(e).indexOf(client.id) < 0) { - that._subMatcher.add(e, client.id); - } + that._subMatcher.add(e, client.id); }); op.exec(cb); @@ -353,7 +349,7 @@ RedisPersistence.prototype.storeOfflinePacket = function(packet, done) { var that = this; var matches = this._subMatcher.match(packet.topic); - async.each(matches, function(client, cb) { + async.each(Array.from(matches), function(client, cb) { that._storePacket(client, packet, cb); }, done); }; From 312153213a0a75e62482db975162c8061c469287 Mon Sep 17 00:00:00 2001 From: Behrad Date: Sat, 16 Apr 2016 11:40:09 +0430 Subject: [PATCH 03/27] add Array.from polyfill --- lib/persistence/abstract.js | 16 ++++++++++++++++ lib/persistence/levelup.js | 2 +- lib/persistence/redis.js | 2 +- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/lib/persistence/abstract.js b/lib/persistence/abstract.js index c0c0060..b833773 100644 --- a/lib/persistence/abstract.js +++ b/lib/persistence/abstract.js @@ -138,6 +138,22 @@ AbstractPersistence.prototype.wire = function(server) { }; }; +function arrayFrom(object) { + return [].slice.call(object); +} + +/** + * Array.from polyfill + * @param aSet the set to changed to an array + */ +AbstractPersistence.prototype.arrayFrom = function(aSet) { + if (Array.from) { + return Array.from(aSet); + } else { + return arrayFrom(aSet); + } +}; + /** * Close the persistance. * diff --git a/lib/persistence/levelup.js b/lib/persistence/levelup.js index 768cf93..d557b18 100644 --- a/lib/persistence/levelup.js +++ b/lib/persistence/levelup.js @@ -230,7 +230,7 @@ LevelUpPersistence.prototype.lookupSubscriptions = function(client, done) { LevelUpPersistence.prototype.storeOfflinePacket = function(packet, done) { var that = this; var subs = this._subMatcher.match(packet.topic); - async.each(Array.from(subs), function(key, cb) { + async.each(this.arrayFrom(subs), function(key, cb) { that._subscriptions.get(key, function(err, sub) { if (err) { return cb(err); diff --git a/lib/persistence/redis.js b/lib/persistence/redis.js index 070604e..9e0e010 100644 --- a/lib/persistence/redis.js +++ b/lib/persistence/redis.js @@ -349,7 +349,7 @@ RedisPersistence.prototype.storeOfflinePacket = function(packet, done) { var that = this; var matches = this._subMatcher.match(packet.topic); - async.each(Array.from(matches), function(client, cb) { + async.each(this.arrayFrom(matches), function(client, cb) { that._storePacket(client, packet, cb); }, done); }; From 8f4e0f15f61ff15451990f2c49eae3ac77684471 Mon Sep 17 00:00:00 2001 From: Behrad Date: Sat, 16 Apr 2016 18:39:34 +0430 Subject: [PATCH 04/27] take removing unSubs out of multi callback & also reduce extra .publish --- lib/persistence/redis.js | 40 +++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/lib/persistence/redis.js b/lib/persistence/redis.js index 9e0e010..08c6f9d 100644 --- a/lib/persistence/redis.js +++ b/lib/persistence/redis.js @@ -258,36 +258,30 @@ RedisPersistence.prototype.storeSubscriptions = function(client, cb) { } }); - var op = this._client.multi() - .get(clientSubKey, function(err, currentSubs){ - if( err || !currentSubs ) { - return; - } - currentSubs = JSON.parse( currentSubs ); - var unsubs = Object.keys(currentSubs).filter(function(topic){ + this._client.get(clientSubKey, function(err, currentSubs){ + if( !err && currentSubs ) { + currentSubs = JSON.parse(currentSubs); + var unsubs = Object.keys(currentSubs).filter(function (topic) { return !subscriptions[topic]; }); - unsubs.forEach(function(topic) { + unsubs.forEach(function (topic) { that._subMatcher.remove(topic, client.id); }); - that._client.publish(that.options.channel, JSON.stringify({ + } + var op = that._client.multi() + .set(clientSubKey, JSON.stringify(subscriptions)) + .publish(that.options.channel, JSON.stringify({ key: clientSubKey, - unsubs: unsubs, process: that._id - })); - }) - .set(clientSubKey, JSON.stringify(subscriptions)) - .publish(this.options.channel, JSON.stringify({ - key: clientSubKey, - process: this._id - })) - .pexpire(clientSubKey, this.options.ttl.subscriptions); - - Object.keys(subscriptions).forEach(function(e) { - that._subMatcher.add(e, client.id); - }); + })) + .pexpire(clientSubKey, that.options.ttl.subscriptions); - op.exec(cb); + Object.keys(subscriptions).forEach(function(e) { + that._subMatcher.add(e, client.id); + }); + + op.exec(cb); + }); }; RedisPersistence.prototype._cleanClient = function(client, done) { From a1913227153c9559c9e133a9012fc558a9fdf61c Mon Sep 17 00:00:00 2001 From: Behrad Date: Thu, 21 Apr 2016 15:33:24 +0430 Subject: [PATCH 05/27] switched to ioredis, removes redis version check --- lib/persistence/redis.js | 84 +++++++++++++++++++--------------------- package.json | 2 +- 2 files changed, 40 insertions(+), 46 deletions(-) diff --git a/lib/persistence/redis.js b/lib/persistence/redis.js index 08c6f9d..75fd3bd 100644 --- a/lib/persistence/redis.js +++ b/lib/persistence/redis.js @@ -25,7 +25,7 @@ OTHER DEALINGS IN THE SOFTWARE. "use strict"; var AbstractPersistence = require("./abstract"); -var redis = require("redis"); +var Redis = require("ioredis"); var util = require("util"); var Matcher = require("./matcher"); var async = require("async"); @@ -114,37 +114,15 @@ function RedisPersistence(options, callback) { }); } - var redisError = null; - var redisVersions = that._client.server_info.versions; - if ( (redisVersions[0] * 1000 + redisVersions[1]) < 2006 ) { - redisError = 'redis instance version should be no less than 2.6'; - } - if (cb) { - cb(redisError); - } else { - if (redisError) { - throw redisError; - } + cb(); } }); }; var that = this; - this._pubSubClient.subscribe(this.options.channel); - - this._pubSubClient.on("message", function(channel, message) { - if (that._explicitlyClosed()) { - return; - } - var parsed = JSON.parse(message); - if (parsed.process !== that._id) { - newSub(parsed.key, parsed.unsubs); - } - }); - - this._pubSubClient.on("subscribe", function() { + this._pubSubClient.subscribe(this.options.channel, function(){ if (that._explicitlyClosed()) { return; } @@ -161,6 +139,16 @@ function RedisPersistence(options, callback) { }); }); }); + + this._pubSubClient.on("message", function(channel, message) { + if (that._explicitlyClosed()) { + return; + } + var parsed = JSON.parse(message); + if (parsed.process !== that._id) { + newSub(parsed.key, parsed.unsubs); + } + }); } util.inherits(RedisPersistence, AbstractPersistence); @@ -172,21 +160,25 @@ util.inherits(RedisPersistence, AbstractPersistence); */ RedisPersistence.prototype._buildClient = function() { - var options = this.options; - var client = redis.createClient( - options.port || 6379, - options.host || "127.0.0.1", - options.redisOptions); - - if (options.db) { - client.select(options.db); + var options = this.options.redisOptions || {}; + + if (this.options.host) { + options.host = this.options.host; + } + + if (this.options.port) { + options.port = this.options.port; } - if (options.password) { - client.auth(options.password); + if (this.options.db) { + options.db = this.options.db; } - return client; + if (this.options.password) { + options.password = this.options.password; + } + + return new Redis(options); }; RedisPersistence.prototype.storeRetained = function(packet, cb) { @@ -325,11 +317,10 @@ RedisPersistence.prototype.lookupSubscriptions = function(client, cb) { var multi = this._client.multi(); var that = this; - multi.get(key, function(err, result) { - subscriptions = JSON.parse(result) || {}; - }); + multi.get(key); - multi.exec(function(err) { + multi.exec(function(err, result) { + subscriptions = JSON.parse(result[0][1]) || {}; cb(err, subscriptions); }); } @@ -393,7 +384,13 @@ RedisPersistence.prototype.streamOfflinePackets = function(client, cb, done) { } function fetch(multi, key) { - multi.get(key, function(err, result) { + return multi.get(key); + } + + results.reduce(fetch, that._client.multi()).exec(function(err,multiResults){ + multiResults.forEach(function(multiResult, i){ + var key = results[i]; + var result = multiResult[1]; total --; // If we don't get result for given packet key. It means // that packet has expired. Just clean it from client packets key @@ -406,10 +403,7 @@ RedisPersistence.prototype.streamOfflinePackets = function(client, cb, done) { } emit(key, result); }); - return multi; - } - - results.reduce(fetch, that._client.multi()).exec(); + }); }); }; diff --git a/package.json b/package.json index f5af5d0..1f5e88d 100644 --- a/package.json +++ b/package.json @@ -102,7 +102,7 @@ "leveldown": "~1.4.3", "zmq": "~2.14.0", "amqp": "~0.2.4", - "redis": "~2.5.0", + "ioredis": "^1.15.1", "hiredis": "^0.4.1", "mongodb": "~2.1.4" } From 6dc28e87f0c9da3edbcbe098bcaec0cc8afb0cac Mon Sep 17 00:00:00 2001 From: Behrad Date: Thu, 21 Apr 2016 15:35:54 +0430 Subject: [PATCH 06/27] still keep redis for tests --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 1f5e88d..333d205 100644 --- a/package.json +++ b/package.json @@ -103,6 +103,7 @@ "zmq": "~2.14.0", "amqp": "~0.2.4", "ioredis": "^1.15.1", + "redis": "~2.5.0", "hiredis": "^0.4.1", "mongodb": "~2.1.4" } From 5493e30d7d569473341fa663503020021d87ff25 Mon Sep 17 00:00:00 2001 From: Behrad Date: Thu, 21 Apr 2016 16:02:29 +0430 Subject: [PATCH 07/27] add disconnect reason --- lib/client.js | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/client.js b/lib/client.js index a8fd108..abdf74f 100644 --- a/lib/client.js +++ b/lib/client.js @@ -114,7 +114,7 @@ Client.prototype._setup = function() { async.parallel(packet.unsubscriptions.map(that.unsubscribeMapTo.bind(that)), function(err) { if (err) { that.logger.warn(err); - that.close(); + that.close(null, err); return; } that.server.persistClient(that); @@ -126,19 +126,19 @@ Client.prototype._setup = function() { client.on("disconnect", function() { that.logger.debug("disconnect requested"); - that.close(); + that.close(null, "disconnect request"); }); function handleError(err) { that.logger.warn(err); - that.onNonDisconnectClose(); + that.onNonDisconnectClose(err); } client.on("error", handleError); client.removeListener("error", nop); client.on("close", function() { - that.onNonDisconnectClose(); + that.onNonDisconnectClose("close"); }); } @@ -168,7 +168,7 @@ Client.prototype.setUpTimer = function() { } else { this.timer = retimer(function keepaliveTimeout() { that.logger.info("keepalive timeout"); - that.onNonDisconnectClose(); + that.onNonDisconnectClose("keepalive timeout"); }, timeout); } }; @@ -237,7 +237,7 @@ Client.prototype._buildForward = function() { forward = false; } else if (that.inflightCounter >= that.server.opts.maxInflightMessages) { that.logger.warn("too many inflight packets, closing"); - that.close(); + that.close(null, "too many inflight packets"); forward = false; } @@ -363,7 +363,7 @@ Client.prototype.handleConnect = function(packet, completeConnection) { that.clean = packet.clean; if (that.id in that.server.clients){ - that.server.clients[that.id].close(completeConnection); + that.server.clients[that.id].close(completeConnection, "new connection request"); } else { completeConnection(); } @@ -473,7 +473,7 @@ Client.prototype.handleSubscribe = function(packet) { }, function(err, authorized) { if (err) { - that.close(); + that.close(null, err); return; } @@ -507,7 +507,7 @@ Client.prototype.handleAuthorizePublish = function(err, success, packet) { if (err || !success) { if (!this._closed && !this._closing) { - that.close(); + that.close(null, err || "publish not authorized"); } return; } @@ -531,7 +531,7 @@ Client.prototype.handleAuthorizePublish = function(err, success, packet) { * * @api private */ -Client.prototype.onNonDisconnectClose = function() { +Client.prototype.onNonDisconnectClose = function(reason) { var that = this, logger = that.logger, will = that.will; if (this._closed || this._closing) { @@ -547,7 +547,7 @@ Client.prototype.onNonDisconnectClose = function() { }); } - this.close(); + this.close(null, reason); }; /** @@ -556,7 +556,7 @@ Client.prototype.onNonDisconnectClose = function() { * @api public * @param {Function} callback The callback to be called when the Client is closed */ -Client.prototype.close = function(callback) { +Client.prototype.close = function(callback, reason) { callback = callback || nop; @@ -567,7 +567,7 @@ Client.prototype.close = function(callback) { var that = this; if (this.id) { - that.logger.debug("closing client"); + that.logger.debug("closing client, reason: " + reason); if (this.timer) { this.timer.clear(); @@ -581,7 +581,7 @@ Client.prototype.close = function(callback) { that.connection.removeAllListeners(); // ignore all errors after disconnection that.connection.on("error", function() {}); - that.server.emit("clientDisconnected", that); + that.server.emit("clientDisconnected", that, reason); callback(); }; From c457978da384aeca24741207da61495c55c317c3 Mon Sep 17 00:00:00 2001 From: Behrad Date: Fri, 22 Apr 2016 12:40:41 +0430 Subject: [PATCH 08/27] Uniform reason to a string value & include it in tests --- lib/client.js | 8 ++++---- lib/server.js | 4 ++-- test/abstract_server.js | 9 ++++++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/lib/client.js b/lib/client.js index abdf74f..5130fce 100644 --- a/lib/client.js +++ b/lib/client.js @@ -114,7 +114,7 @@ Client.prototype._setup = function() { async.parallel(packet.unsubscriptions.map(that.unsubscribeMapTo.bind(that)), function(err) { if (err) { that.logger.warn(err); - that.close(null, err); + that.close(null, err.message); return; } that.server.persistClient(that); @@ -131,7 +131,7 @@ Client.prototype._setup = function() { function handleError(err) { that.logger.warn(err); - that.onNonDisconnectClose(err); + that.onNonDisconnectClose(err.message); } client.on("error", handleError); @@ -473,7 +473,7 @@ Client.prototype.handleSubscribe = function(packet) { }, function(err, authorized) { if (err) { - that.close(null, err); + that.close(null, err.message); return; } @@ -507,7 +507,7 @@ Client.prototype.handleAuthorizePublish = function(err, success, packet) { if (err || !success) { if (!this._closed && !this._closing) { - that.close(null, err || "publish not authorized"); + that.close(null, (err && err.message) || "publish not authorized"); } return; } diff --git a/lib/server.js b/lib/server.js index 8ea4b36..bb56706 100755 --- a/lib/server.js +++ b/lib/server.js @@ -256,7 +256,7 @@ function Server(opts, callback) { clientId = payload; if(that.clients[clientId] && serverId !== that.id) { - that.clients[clientId].close(); + that.clients[clientId].close(null, "new connection request"); } } ); @@ -576,7 +576,7 @@ Server.prototype.close = function(callback) { } async.each(stuffToClose, function(toClose, cb) { - toClose.close(cb); + toClose.close(cb, "server closed"); }, function() { that.ascoltatore.close(function () { that.logger.info("server closed"); diff --git a/test/abstract_server.js b/test/abstract_server.js index 9f49115..50a2060 100644 --- a/test/abstract_server.js +++ b/test/abstract_server.js @@ -247,7 +247,8 @@ module.exports = function(moscaSettings, createConnection) { async.series([ function(cb) { serverOne = new mosca.Server(settingsOne, function(err, server) { - serverOne.on('clientDisconnected', function(serverClient) { + serverOne.on('clientDisconnected', function(serverClient, reason) { + expect(reason).to.be.equal('new connection request'); expect(serverClient).not.to.be.equal(undefined); done(); }); @@ -862,7 +863,8 @@ module.exports = function(moscaSettings, createConnection) { it("should emit an event when a client is disconnected", function(done) { var client = createConnection(settings.port, settings.host); - instance.on('clientDisconnected', function(serverClient) { + instance.on('clientDisconnected', function(serverClient, reason) { + expect(reason).to.be.equal('disconnect request'); expect(serverClient).not.to.be.equal(undefined); done(); }); @@ -899,7 +901,8 @@ module.exports = function(moscaSettings, createConnection) { it("should emit an event when a client is disconnected without a disconnect", function(done) { var client = createConnection(settings.port, settings.host); - instance.on('clientDisconnected', function(serverClient) { + instance.on('clientDisconnected', function(serverClient, reason) { + expect(reason).to.be.equal('close'); expect(serverClient).not.to.be.equal(undefined); done(); }); From c8c50be35ecf204327a25172d64e471ab95f44ca Mon Sep 17 00:00:00 2001 From: Behrad Date: Thu, 14 Apr 2016 19:25:15 +0430 Subject: [PATCH 09/27] Introduce publishSubscriptions option, so that subscription $SYS topics can be switched off, default is on. --- lib/options.js | 8 ++++++-- lib/server.js | 33 +++++++++++++++++++-------------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/lib/options.js b/lib/options.js index 41d67c1..9ee6069 100755 --- a/lib/options.js +++ b/lib/options.js @@ -61,7 +61,8 @@ function modernize(legacy) { "maxInflightMessages", "stats", "publishNewClient", - "publishClientDisconnect" + "publishClientDisconnect", + "publishSubscriptions" ]; // copy all conserved options @@ -250,7 +251,8 @@ function validate(opts, validationOptions) { 'maxInflightMessages': { type: 'integer' }, 'stats': { type: 'boolean' }, 'publishNewClient': { type: 'boolean' }, - 'publishClientDisconnect': { type: 'boolean' } + 'publishClientDisconnect': { type: 'boolean' }, + 'publishSubscriptions': { type: 'boolean' } } }); @@ -333,6 +335,7 @@ function defaultsLegacy() { stats: false, publishNewClient: true, publishClientDisconnect: true, + publishSubscriptions: true, maxInflightMessages: 1024, logger: { name: "mosca", @@ -368,6 +371,7 @@ function defaultsModern() { stats: false, publishNewClient: true, publishClientDisconnect: true, + publishSubscriptions: true, maxInflightMessages: 1024, logger: { name: "mosca", diff --git a/lib/server.js b/lib/server.js index c7c7913..fcae99e 100755 --- a/lib/server.js +++ b/lib/server.js @@ -67,6 +67,7 @@ var nop = function() {}; * - `stats`, publish the stats every 10s (default false). * - `publishNewClient`, publish message to topic "$SYS/{broker-id}/new/clients" when new client connects. * - `publishClientDisconnect`, publish message to topic "$SYS/{broker-id}/disconnect/clients" when a client disconnects. + * - `publishSubscriptions`, publish message to topic "$SYS/{broker-id}/new/(un)subscribes" when a client subscribes/unsubscribes. * * Interface may contain following properties: * - `type`, name of a build-in type or a custom type factory @@ -262,23 +263,27 @@ function Server(opts, callback) { }); that.on("subscribed", function(topic, client) { - that.publish({ - topic: "$SYS/" + that.id + "/new/subscribes", - payload: JSON.stringify({ - clientId: client.id, - topic: topic - }) - }); + if(that.modernOpts.publishSubscriptions) { + that.publish({ + topic: "$SYS/" + that.id + "/new/subscribes", + payload: JSON.stringify({ + clientId: client.id, + topic: topic + }) + }); + } }); that.on("unsubscribed", function(topic, client) { - that.publish({ - topic: "$SYS/" + that.id + "/new/unsubscribes", - payload: JSON.stringify({ - clientId: client.id, - topic: topic - }) - }); + if(that.modernOpts.publishSubscriptions) { + that.publish({ + topic: "$SYS/" + that.id + "/new/unsubscribes", + payload: JSON.stringify({ + clientId: client.id, + topic: topic + }) + }); + } }); that.on("clientDisconnected", function(client) { From c2120047de32794f57078dba9ad65aeefe0e3074 Mon Sep 17 00:00:00 2001 From: Behrad Date: Sat, 16 Apr 2016 11:17:22 +0430 Subject: [PATCH 10/27] move publishSubscriptions check out of callback --- lib/server.js | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/lib/server.js b/lib/server.js index fcae99e..8ea4b36 100755 --- a/lib/server.js +++ b/lib/server.js @@ -262,8 +262,8 @@ function Server(opts, callback) { ); }); - that.on("subscribed", function(topic, client) { - if(that.modernOpts.publishSubscriptions) { + if(that.modernOpts.publishSubscriptions) { + that.on("subscribed", function(topic, client) { that.publish({ topic: "$SYS/" + that.id + "/new/subscribes", payload: JSON.stringify({ @@ -271,11 +271,9 @@ function Server(opts, callback) { topic: topic }) }); - } - }); + }); - that.on("unsubscribed", function(topic, client) { - if(that.modernOpts.publishSubscriptions) { + that.on("unsubscribed", function(topic, client) { that.publish({ topic: "$SYS/" + that.id + "/new/unsubscribes", payload: JSON.stringify({ @@ -283,8 +281,8 @@ function Server(opts, callback) { topic: topic }) }); - } - }); + }); + } that.on("clientDisconnected", function(client) { if(that.modernOpts.publishClientDisconnect) { From e5da73b651f1b42ba2609ab4539d7d1a66f1d861 Mon Sep 17 00:00:00 2001 From: Behrad Date: Fri, 22 Apr 2016 13:21:06 +0430 Subject: [PATCH 11/27] update async to support ES2015 Set iteration, remove arrayFrom --- lib/persistence/abstract.js | 16 ---------------- lib/persistence/levelup.js | 2 +- lib/persistence/redis.js | 2 +- package.json | 2 +- 4 files changed, 3 insertions(+), 19 deletions(-) diff --git a/lib/persistence/abstract.js b/lib/persistence/abstract.js index b833773..c0c0060 100644 --- a/lib/persistence/abstract.js +++ b/lib/persistence/abstract.js @@ -138,22 +138,6 @@ AbstractPersistence.prototype.wire = function(server) { }; }; -function arrayFrom(object) { - return [].slice.call(object); -} - -/** - * Array.from polyfill - * @param aSet the set to changed to an array - */ -AbstractPersistence.prototype.arrayFrom = function(aSet) { - if (Array.from) { - return Array.from(aSet); - } else { - return arrayFrom(aSet); - } -}; - /** * Close the persistance. * diff --git a/lib/persistence/levelup.js b/lib/persistence/levelup.js index d557b18..2456efd 100644 --- a/lib/persistence/levelup.js +++ b/lib/persistence/levelup.js @@ -230,7 +230,7 @@ LevelUpPersistence.prototype.lookupSubscriptions = function(client, done) { LevelUpPersistence.prototype.storeOfflinePacket = function(packet, done) { var that = this; var subs = this._subMatcher.match(packet.topic); - async.each(this.arrayFrom(subs), function(key, cb) { + async.each(subs, function(key, cb) { that._subscriptions.get(key, function(err, sub) { if (err) { return cb(err); diff --git a/lib/persistence/redis.js b/lib/persistence/redis.js index 08c6f9d..83e339c 100644 --- a/lib/persistence/redis.js +++ b/lib/persistence/redis.js @@ -343,7 +343,7 @@ RedisPersistence.prototype.storeOfflinePacket = function(packet, done) { var that = this; var matches = this._subMatcher.match(packet.topic); - async.each(this.arrayFrom(matches), function(client, cb) { + async.each(matches, function(client, cb) { that._storePacket(client, packet, cb); }, done); }; diff --git a/package.json b/package.json index f5af5d0..666fe5f 100644 --- a/package.json +++ b/package.json @@ -71,7 +71,7 @@ }, "dependencies": { "ascoltatori": "^2.0.0", - "async": "~1.5.2", + "async": "^2.0.0-rc.3", "brfs": "~1.4.2", "bunyan": "^1.5.1", "clone": "^1.0.2", From 18fedc675f8d40f31b7d265f5396f9d8ca7b9444 Mon Sep 17 00:00:00 2001 From: Behrad Date: Fri, 22 Apr 2016 13:22:49 +0430 Subject: [PATCH 12/27] drop node 0.10 support --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6442f00..1eec2eb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,6 @@ addons: packages: - g++-4.8 node_js: - - 0.10 - 0.12 - 4 - 5 From 0522b268ba47f58101c080b9a60153e3a9db14b9 Mon Sep 17 00:00:00 2001 From: Behrad Date: Fri, 22 Apr 2016 13:46:44 +0430 Subject: [PATCH 13/27] updated from #449 --- lib/persistence/abstract.js | 16 ---------------- lib/persistence/levelup.js | 2 +- lib/persistence/redis.js | 2 +- package.json | 2 +- 4 files changed, 3 insertions(+), 19 deletions(-) diff --git a/lib/persistence/abstract.js b/lib/persistence/abstract.js index b833773..c0c0060 100644 --- a/lib/persistence/abstract.js +++ b/lib/persistence/abstract.js @@ -138,22 +138,6 @@ AbstractPersistence.prototype.wire = function(server) { }; }; -function arrayFrom(object) { - return [].slice.call(object); -} - -/** - * Array.from polyfill - * @param aSet the set to changed to an array - */ -AbstractPersistence.prototype.arrayFrom = function(aSet) { - if (Array.from) { - return Array.from(aSet); - } else { - return arrayFrom(aSet); - } -}; - /** * Close the persistance. * diff --git a/lib/persistence/levelup.js b/lib/persistence/levelup.js index d557b18..2456efd 100644 --- a/lib/persistence/levelup.js +++ b/lib/persistence/levelup.js @@ -230,7 +230,7 @@ LevelUpPersistence.prototype.lookupSubscriptions = function(client, done) { LevelUpPersistence.prototype.storeOfflinePacket = function(packet, done) { var that = this; var subs = this._subMatcher.match(packet.topic); - async.each(this.arrayFrom(subs), function(key, cb) { + async.each(subs, function(key, cb) { that._subscriptions.get(key, function(err, sub) { if (err) { return cb(err); diff --git a/lib/persistence/redis.js b/lib/persistence/redis.js index 75fd3bd..852b67b 100644 --- a/lib/persistence/redis.js +++ b/lib/persistence/redis.js @@ -334,7 +334,7 @@ RedisPersistence.prototype.storeOfflinePacket = function(packet, done) { var that = this; var matches = this._subMatcher.match(packet.topic); - async.each(this.arrayFrom(matches), function(client, cb) { + async.each(matches, function(client, cb) { that._storePacket(client, packet, cb); }, done); }; diff --git a/package.json b/package.json index 333d205..aa731dc 100644 --- a/package.json +++ b/package.json @@ -71,7 +71,7 @@ }, "dependencies": { "ascoltatori": "^2.0.0", - "async": "~1.5.2", + "async": "^2.0.0-rc.3", "brfs": "~1.4.2", "bunyan": "^1.5.1", "clone": "^1.0.2", From 2183cb5b5ee5c9d08b1053b60cb8a77ae7966c1e Mon Sep 17 00:00:00 2001 From: Behrad Date: Fri, 22 Apr 2016 14:32:56 +0430 Subject: [PATCH 14/27] completely remove redis dependency --- lib/persistence/redis.js | 4 ++++ package.json | 1 - test/persistence/redis_spec.js | 2 +- test/server_redis.js | 4 ++-- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/persistence/redis.js b/lib/persistence/redis.js index 852b67b..677c6b6 100644 --- a/lib/persistence/redis.js +++ b/lib/persistence/redis.js @@ -388,6 +388,10 @@ RedisPersistence.prototype.streamOfflinePackets = function(client, cb, done) { } results.reduce(fetch, that._client.multi()).exec(function(err,multiResults){ + if(!multiResults && done) { + done(err); + return; + } multiResults.forEach(function(multiResult, i){ var key = results[i]; var result = multiResult[1]; diff --git a/package.json b/package.json index aa731dc..019e1ef 100644 --- a/package.json +++ b/package.json @@ -103,7 +103,6 @@ "zmq": "~2.14.0", "amqp": "~0.2.4", "ioredis": "^1.15.1", - "redis": "~2.5.0", "hiredis": "^0.4.1", "mongodb": "~2.1.4" } diff --git a/test/persistence/redis_spec.js b/test/persistence/redis_spec.js index 11b6bc9..b13335a 100644 --- a/test/persistence/redis_spec.js +++ b/test/persistence/redis_spec.js @@ -2,7 +2,7 @@ var abstract = require("./abstract"); var Redis = require("../../").persistence.Redis; -var redis = require("redis"); +var redis = require("ioredis"); describe("mosca.persistence.Redis", function() { diff --git a/test/server_redis.js b/test/server_redis.js index 8221922..59f26ae 100644 --- a/test/server_redis.js +++ b/test/server_redis.js @@ -2,7 +2,7 @@ var mqtt = require("mqtt"); var async = require("async"); var ascoltatori = require("ascoltatori"); var abstractServerTests = require("./abstract_server"); -var redis = require("redis"); +var redis = require("ioredis"); var createConnection = require("./helpers/createConnection"); describe("mosca.Server with redis persistence", function() { @@ -33,7 +33,7 @@ describe("mosca.Server with redis persistence", function() { }, persistence : { factory: mosca.persistence.Redis - }, + } }; } From 7ffb871f95ceea279f9ef4fa437f2a67692ecd78 Mon Sep 17 00:00:00 2001 From: Behrad Date: Fri, 22 Apr 2016 15:42:30 +0430 Subject: [PATCH 15/27] drop node 0.10 --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6442f00..1eec2eb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,6 @@ addons: packages: - g++-4.8 node_js: - - 0.10 - 0.12 - 4 - 5 From ac276fb258e97a31a5f5bbc148879e0eb874a2bd Mon Sep 17 00:00:00 2001 From: Matteo Collina Date: Mon, 2 May 2016 09:03:16 +0200 Subject: [PATCH 16/27] Added node v6 in .travis --- .travis.yml | 1 + lib/server.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1eec2eb..68c6475 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ node_js: - 0.12 - 4 - 5 + - 6 services: - redis-server - mongodb diff --git a/lib/server.js b/lib/server.js index 98a8878..cdc97cb 100755 --- a/lib/server.js +++ b/lib/server.js @@ -231,7 +231,7 @@ function Server(opts, callback) { } ], function(err, results){ if(err) { - callback(err) + callback(err); } }); From 88e9b28304fc23cf2ca8b3ac602f1e1b4d887d8c Mon Sep 17 00:00:00 2001 From: Matteo Collina Date: Mon, 23 May 2016 11:24:06 +0200 Subject: [PATCH 17/27] removed benchmarks and microtime --- benchmarks/bombing.js | 9 - benchmarks/db-mosquitto.conf | 675 ------------------------------- benchmarks/graph.R | 28 -- benchmarks/no-log-mosquitto.conf | 675 ------------------------------- benchmarks/single_pub.js | 68 ---- benchmarks/single_pub_sub.js | 83 ---- benchmarks/throughputCounter.js | 19 - package.json | 1 - 8 files changed, 1558 deletions(-) delete mode 100644 benchmarks/bombing.js delete mode 100644 benchmarks/db-mosquitto.conf delete mode 100755 benchmarks/graph.R delete mode 100644 benchmarks/no-log-mosquitto.conf delete mode 100755 benchmarks/single_pub.js delete mode 100755 benchmarks/single_pub_sub.js delete mode 100644 benchmarks/throughputCounter.js diff --git a/benchmarks/bombing.js b/benchmarks/bombing.js deleted file mode 100644 index 53810f5..0000000 --- a/benchmarks/bombing.js +++ /dev/null @@ -1,9 +0,0 @@ -var mqtt = require('mqtt'); -var client = mqtt.createClient(1883, "localhost", { clean: true }); - -function publish() { - client.publish("test", "payload"); - setImmediate(publish); -} - -client.on("connect", publish); diff --git a/benchmarks/db-mosquitto.conf b/benchmarks/db-mosquitto.conf deleted file mode 100644 index 38a6eee..0000000 --- a/benchmarks/db-mosquitto.conf +++ /dev/null @@ -1,675 +0,0 @@ -# Config file for mosquitto -# -# See mosquitto.conf(5) for more information. -# -# Default values are shown, uncomment to change. -# -# Use the # character to indicate a comment, but only if it is the -# very first character on the line. - -# ================================================================= -# General configuration -# ================================================================= - -# Time in seconds to wait before resending an outgoing QoS=1 or -# QoS=2 message. -#retry_interval 20 - -# Time in seconds between updates of the $SYS tree. -#sys_interval 10 - -# Time in seconds between cleaning the internal message store of -# unreferenced messages. Lower values will result in lower memory -# usage but more processor time, higher values will have the -# opposite effect. -# Setting a value of 0 means the unreferenced messages will be -# disposed of as quickly as possible. -#store_clean_interval 10 - -# Write process id to a file. Default is a blank string which means -# a pid file shouldn't be written. -# This should be set to /var/run/mosquitto.pid if mosquitto is -# being run automatically on boot with an init script and -# start-stop-daemon or similar. -#pid_file - -# When run as root, drop privileges to this user and its primary -# group. -# Leave blank to stay as root, but this is not recommended. -# If run as a non-root user, this setting has no effect. -# Note that on Windows this has no effect and so mosquitto should -# be started by the user you wish it to run as. -#user mosquitto - -# The maximum number of QoS 1 and 2 messages currently inflight per -# client. -# This includes messages that are partway through handshakes and -# those that are being retried. Defaults to 20. Set to 0 for no -# maximum. Setting to 1 will guarantee in-order delivery of QoS 1 -# and 2 messages. -#max_inflight_messages 20 - -# The maximum number of QoS 1 and 2 messages to hold in a queue -# above those that are currently in-flight. Defaults to 100. Set -# to 0 for no maximum (not recommended). -# See also queue_qos0_messages. -#max_queued_messages 100 - -# Set to true to queue messages with QoS 0 when a persistent client is -# disconnected. These messages are included in the limit imposed by -# max_queued_messages. -# Defaults to false. -# Note that the MQTT v3.1 spec states that only QoS 1 and 2 messages -# should be saved in this situation so this is a non-standard option. -#queue_qos0_messages false - -# This option allows persistent clients (those with clean session set to false) -# to be removed if they do not reconnect within a certain time frame. This is a -# non-standard option. As far as the MQTT spec is concerned, persistent clients -# persist forever. -# Badly designed clients may set clean session to false whilst using a randomly -# generated client id. This leads to persistent clients that will never -# reconnect. This option allows these clients to be removed. -# -# The expiration period should be an integer followed by one of d w m y for -# day, week, month and year respectively. For example -# -# persistent_client_expiration 2m -# persistent_client_expiration 14d -# persistent_client_expiration 1y -# -# As this is a non-standard option, the default if not set is to never expire -# persistent clients. -#persistent_client_expiration - -# If a client is subscribed to multiple subscriptions that overlap, e.g. foo/# -# and foo/+/baz , then MQTT expects that when the broker receives a message on -# a topic that matches both subscriptions, such as foo/bar/baz, then the client -# should only receive the message once. -# Mosquitto keeps track of which clients a message has been sent to in order to -# meet this requirement. The allow_duplicate_messages option allows this -# behaviour to be disabled, which may be useful if you have a large number of -# clients subscribed to the same set of topics and are very concerned about -# minimising memory usage. -# It can be safely set to true if you know in advance that your clients will -# never have overlapping subscriptions, otherwise your clients must be able to -# correctly deal with duplicate messages even when then have QoS=2. -#allow_duplicate_messages false - -# ================================================================= -# Default listener -# ================================================================= - -# IP address/hostname to bind the default listener to. If not -# given, the default listener will not be bound to a specific -# address and so will be accessible to all network interfaces. -# bind_address ip-address/host name -#bind_address - -# Port to use for the default listener. -#port 1883 - -# The maximum number of client connections to allow. This is -# a per listener setting. -# Default is -1, which means unlimited connections. -# Note that other process limits mean that unlimited connections -# are not really possible. Typically the default maximum number of -# connections possible is around 1024. -#max_connections -1 - -# ----------------------------------------------------------------- -# Certificate based SSL/TLS support -# ----------------------------------------------------------------- -# The following options can be used to enable SSL/TLS support for -# this listener. Note that the recommended port for MQTT over TLS -# is 8883, but this must be set manually. -# -# See also the mosquitto-tls man page. - -# At least one of cafile or capath must be defined. They both -# define methods of accessing the PEM encoded Certificate -# Authority certificates that have signed your server certificate -# and that you wish to trust. -# cafile defines the path to a file containing the CA certificates. -# capath defines a directory that will be searched for files -# containing the CA certificates. For capath to work correctly, the -# certificate files must have ".crt" as the file ending and you must run -# "c_rehash " each time you add/remove a certificate. -#cafile -#capath - -# Path to the PEM encoded server certificate. -#certfile - -# Path to the PEM encoded keyfile. -#keyfile - -# By default a TLS enabled listener will operate in a similar fashion to a -# https enabled web server, in that the server has a certificate signed by a CA -# and the client will verify that it is a trusted certificate. The overall aim -# is encryption of the network traffic. By setting require_certificate to true, -# the client must provide a valid certificate in order for the network -# connection to proceed. This allows access to the broker to be controlled -# outside of the mechanisms provided by MQTT. -#require_certificate false - -# If require_certificate is true, you may set use_identity_as_username to true -# to use the CN value from the client certificate as a username. If this is -# true, the password_file option will not be used for this listener. -#use_identity_as_username false - -# If you have require_certificate set to true, you can create a certificate -# revocation list file to revoke access to particular client certificates. If -# you have done this, use crlfile to point to the PEM encoded revocation file. -#crlfile - -# If you wish to control which encryption ciphers are used, use the ciphers -# option. The list of available ciphers can be optained using the "openssl -# ciphers" command and should be provided in the same format as the output of -# that command. -#ciphers - -# ----------------------------------------------------------------- -# Pre-shared-key based SSL/TLS support -# ----------------------------------------------------------------- -# The following options can be used to enable PSK based SSL/TLS support for -# this listener. Note that the recommended port for MQTT over TLS is 8883, but -# this must be set manually. -# -# See also the mosquitto-tls man page and the "Certificate based SSL/TLS -# support" section. Only one of certificate or PSK encryption support can be -# enabled for any listener. - -# The psk_hint option enables pre-shared-key support for this listener and also -# acts as an identifier for this listener. The hint is sent to clients and may -# be used locally to aid authentication. The hint is a free form string that -# doesn't have much meaning in itself, so feel free to be creative. -# If this option is provided, see psk_file to define the pre-shared keys to be -# used or create a security plugin to handle them. -#psk_hint - -# Set use_identity_as_username to have the psk identity sent by the client used -# as its username. Authentication will be carried out using the PSK rather than -# the MQTT username/password and so password_file will not be used for this -# listener. -#use_identity_as_username false - -# When using PSK, the encryption ciphers used will be chosen from the list of -# available PSK ciphers. If you want to control which ciphers are available, -# use the "ciphers" option. The list of available ciphers can be optained -# using the "openssl ciphers" command and should be provided in the same format -# as the output of that command. -#ciphers - -# ================================================================= -# Extra listeners -# ================================================================= - -# Listen on a port/ip address combination. By using this variable -# multiple times, mosquitto can listen on more than one port. If -# this variable is used and neither bind_address nor port given, -# then the default listener will not be started. -# The port number to listen on must be given. Optionally, an ip -# address or host name may be supplied as a second argument. In -# this case, mosquitto will attempt to bind the listener to that -# address and so restrict access to the associated network and -# interface. By default, mosquitto will listen on all interfaces. -# listener port-number [ip address/host name] -#listener - -# The maximum number of client connections to allow. This is -# a per listener setting. -# Default is -1, which means unlimited connections. -# Note that other process limits mean that unlimited connections -# are not really possible. Typically the default maximum number of -# connections possible is around 1024. -#max_connections -1 - -# The listener can be restricted to operating within a topic hierarchy using -# the mount_point option. This is achieved be prefixing the mount_point string -# to all topics for any clients connected to this listener. This prefixing only -# happens internally to the broker; the client will not see the prefix. -#mount_point - -# ----------------------------------------------------------------- -# Certificate based SSL/TLS support -# ----------------------------------------------------------------- -# The following options can be used to enable certificate based SSL/TLS support -# for this listener. Note that the recommended port for MQTT over TLS is 8883, -# but this must be set manually. -# -# See also the mosquitto-tls man page and the "Pre-shared-key based SSL/TLS -# support" section. Only one of certificate or PSK encryption support can be -# enabled for any listener. - -# At least one of cafile or capath must be defined to enable certificate based -# TLS encryption. They both define methods of accessing the PEM encoded -# Certificate Authority certificates that have signed your server certificate -# and that you wish to trust. -# cafile defines the path to a file containing the CA certificates. -# capath defines a directory that will be searched for files -# containing the CA certificates. For capath to work correctly, the -# certificate files must have ".crt" as the file ending and you must run -# "c_rehash " each time you add/remove a certificate. -#cafile -#capath - -# Path to the PEM encoded server certificate. -#certfile - -# Path to the PEM encoded keyfile. -#keyfile - -# By default an TLS enabled listener will operate in a similar fashion to a -# https enabled web server, in that the server has a certificate signed by a CA -# and the client will verify that it is a trusted certificate. The overall aim -# is encryption of the network traffic. By setting require_certificate to true, -# the client must provide a valid certificate in order for the network -# connection to proceed. This allows access to the broker to be controlled -# outside of the mechanisms provided by MQTT. -#require_certificate false - -# If require_certificate is true, you may set use_identity_as_username to true -# to use the CN value from the client certificate as a username. If this is -# true, the password_file option will not be used for this listener. -#use_identity_as_username false - -# If you have require_certificate set to true, you can create a certificate -# revocation list file to revoke access to particular client certificates. If -# you have done this, use crlfile to point to the PEM encoded revocation file. -#crlfile - -# If you wish to control which encryption ciphers are used, use the ciphers -# option. The list of available ciphers can be optained using the "openssl -# ciphers" command and should be provided in the same format as the output of -# that command. -#ciphers - -# ----------------------------------------------------------------- -# Pre-shared-key based SSL/TLS support -# ----------------------------------------------------------------- -# The following options can be used to enable PSK based SSL/TLS support for -# this listener. Note that the recommended port for MQTT over TLS is 8883, but -# this must be set manually. -# -# See also the mosquitto-tls man page and the "Certificate based SSL/TLS -# support" section. Only one of certificate or PSK encryption support can be -# enabled for any listener. - -# The psk_hint option enables pre-shared-key support for this listener and also -# acts as an identifier for this listener. The hint is sent to clients and may -# be used locally to aid authentication. The hint is a free form string that -# doesn't have much meaning in itself, so feel free to be creative. -# If this option is provided, see psk_file to define the pre-shared keys to be -# used or create a security plugin to handle them. -#psk_hint - -# Set use_identity_as_username to have the psk identity sent by the client used -# as its username. Authentication will be carried out using the PSK rather than -# the MQTT username/password and so password_file will not be used for this -# listener. -#use_identity_as_username false - -# When using PSK, the encryption ciphers used will be chosen from the list of -# available PSK ciphers. If you want to control which ciphers are available, -# use the "ciphers" option. The list of available ciphers can be optained -# using the "openssl ciphers" command and should be provided in the same format -# as the output of that command. -#ciphers - -# ================================================================= -# Persistence -# ================================================================= - -# If persistence is enabled, save the in-memory database to disk -# every autosave_interval seconds. If set to 0, the persistence -# database will only be written when mosquitto exits. See also -# autosave_on_changes. -# Note that writing of the persistence database can be forced by -# sending mosquitto a SIGUSR1 signal. -#autosave_interval 1800 - -# If true, mosquitto will count the number of subscription changes, retained -# messages received and queued messages and if the total exceeds -# autosave_interval then the in-memory database will be saved to disk. -# If false, mosquitto will save the in-memory database to disk by treating -# autosave_interval as a time in seconds. -#autosave_on_changes false - -# Save persistent message data to disk (true/false). -# This saves information about all messages, including -# subscriptions, currently in-flight messages and retained -# messages. -# retained_persistence is a synonym for this option. -persistence true - -# The filename to use for the persistent database, not including -# the path. -persistence_file mosquitto.db - -# Location for persistent database. Must include trailing / -# Default is an empty string (current directory). -# Set to /var/lib/mosquitto/ if running as a proper service. -#persistence_location - -# ================================================================= -# Logging -# ================================================================= - -# Places to log to. Use multiple log_dest lines for multiple -# logging destinations. -# Possible destinations are: stdout stderr syslog topic -# stdout and stderr log to the console on the named output. -# syslog uses the userspace syslog facility which usually ends up -# in /var/log/messages or similar. -# topic logs to the broker topic '$SYS/broker/log/', -# where severity is one of D, E, W, N, I which are debug, error, -# warning, notice and information. -# Note that if the broker is running as a Windows service it will default to -# "log_dest none" and neither stdout nor stderr logging is available. -# Use "log_dest none" if you wish to disable logging. -#log_dest stderr - -# Types of messages to log. Use multiple log_type lines for logging -# multiple types of messages. -# Possible types are: debug, error, warning, notice, information, -# none. -# Note that debug type messages are for decoding the incoming -# network packets. -# They are not logged in syslog. -#log_type error -#log_type warning -#log_type notice -#log_type information -log_type none - -# If set to true, client connection and disconnection messages will be included -# in the log. -#connection_messages true - -# If set to true, add a timestamp value to each log message. -#log_timestamp true - -# ================================================================= -# Security -# ================================================================= - -# If set, only clients that have a matching prefix on their -# clientid will be allowed to connect to the broker. By default, -# all clients may connect. -# For example, setting "secure-" here would mean a client "secure- -# client" could connect but another with clientid "mqtt" couldn't. -#clientid_prefixes - -# Boolean value that determines whether clients that connect -# without providing a username are allowed to connect. If set to -# false then a password file should be created (see the -# password_file option) to control authenticated client access. -# Defaults to true. -#allow_anonymous true - -# In addition to the clientid_prefixes, allow_anonymous and TLS -# authentication options, username based authentication is also -# possible. The default support is described in "Default -# authentication and topic access control" below. The auth_plugin -# allows another authentication method to be used. -# Specify the path to the loadable plugin and see the -# "Authentication and topic access plugin options" section below. -#auth_plugin - -# ----------------------------------------------------------------- -# Default authentication and topic access control -# ----------------------------------------------------------------- - -# Control access to the broker using a password file. This file can be -# generated using the mosquitto_passwd utility. If TLS support is not compiled -# into mosquitto (it is recommended that TLS support should be included) then -# plain text passwords are used, in which case the file should be a text file -# with lines in the format: -# username:password -# The password (and colon) may be omitted if desired, although this -# offers very little in the way of security. -# -# See the TLS client require_certificate and use_identity_as_username options -# for alternative authentication options. -#password_file - -# Access may also be controlled using a pre-shared-key file. This requires -# TLS-PSK support and a listener configured to use it. The file should be text -# lines in the format: -# identity:key -# The key should be in hexadecimal format without a leading "0x". -#psk_file - -# Control access to topics on the broker using an access control list -# file. If this parameter is defined then only the topics listed will -# have access. -# If the first character of a line of the ACL file is a # it is treated as a -# comment. -# Topic access is added with lines of the format: -# -# topic [read|write] -# -# The access type is controlled using "read" or "write". This parameter -# is optional - if not given then the access is read/write. -# can contain the + or # wildcards as in subscriptions. -# -# The first set of topics are applied to anonymous clients, assuming -# allow_anonymous is true. User specific topic ACLs are added after a -# user line as follows: -# -# user -# -# The username referred to here is the same as in password_file. It is -# not the clientid. -# -# -# If is also possible to define ACLs based on pattern substitution within the -# topic. The patterns available for substition are: -# -# %c to match the client id of the client -# %u to match the username of the client -# -# The substitution pattern must be the only text for that level of hierarchy. -# -# The form is the same as for the topic keyword, but using pattern as the -# keyword. -# Pattern ACLs apply to all users even if the "user" keyword has previously -# been given. -# -# pattern [read|write] -# -# Example: -# -# pattern write sensor/%u/data -# -#acl_file - -# ----------------------------------------------------------------- -# Authentication and topic access plugin options -# ----------------------------------------------------------------- - -# If the auth_plugin option above is used, define options to pass to the -# plugin here as described by the plugin instructions. All options named -# using the format auth_opt_* will be passed to the plugin, for example: -# -# auth_opt_db_host -# auth_opt_db_port -# auth_opt_db_username -# auth_opt_db_password - - -# ================================================================= -# Bridges -# ================================================================= - -# A bridge is a way of connecting multiple MQTT brokers together. -# Create a new bridge using the "connection" option as described below. Set -# options for the bridges using the remaining parameters. You must specify the -# address and at least one topic to subscribe to. -# Each connection must have a unique name. -# Only a single address per configuration is currently supported, -# unlike in rsmb. -# The direction that the topic will be shared can be chosen by -# specifying out, in or both, where the default value is out. -# The QoS level of the bridged communication can be specified with the next -# topic option. The default QoS level is 0, to change the QoS the topic -# direction must also be given. -# The local and remote prefix options allow a topic to be remapped when it is -# bridged to/from the remote broker. This provides the ability to place a topic -# tree in an appropriate location. -# For more details see the mosquitto.conf man page. -# Multiple topics can be specified per connection, but be careful -# not to create any loops. -# If you are using bridges with cleansession set to false (the default), then -# you may get unexpected behaviour from incoming topics if you change what -# topics you are subscribing to. This is because the remote broker keeps the -# subscription for the old topic. If you have this problem, connect your bridge -# with cleansession set to true, then reconnect with cleansession set to false -# as normal. -#connection -#address [:] -#topic [[[out | in | both] qos-level] local-prefix remote-prefix] - -# Set the client id for this bridge connection. If not defined, -# this defaults to 'name.hostname' where name is the connection -# name and hostname is the hostname of this computer. -#clientid - -# Set the clean session variable for this bridge. -# When set to true, when the bridge disconnects for any reason, all -# messages and subscriptions will be cleaned up on the remote -# broker. Note that with cleansession set to true, there may be a -# significant amount of retained messages sent when the bridge -# reconnects after losing its connection. -# When set to false, the subscriptions and messages are kept on the -# remote broker, and delivered when the bridge reconnects. -#cleansession false - -# If set to true, publish notification messages to the local and remote brokers -# giving information about the state of the bridge connection. Retained -# messages are published to the topic $SYS/broker/connection//state -# unless the notification_topic option is used. -# If the message is 1 then the connection is active, or 0 if the connection has -# failed. -#notifications true - -# Choose the topic on which notification messages for this bridge are -# published. If not set, messages are published on the topic -# $SYS/broker/connection//state -#notification_topic - -# Set the keepalive interval for this bridge connection, in -# seconds. -#keepalive_interval 60 - -# Set the start type of the bridge. This controls how the bridge starts and -# can be one of three types: automatic, lazy and once. Note that RSMB provides -# a fourth start type "manual" which isn't currently supported by mosquitto. -# -# "automatic" is the default start type and means that the bridge connection -# will be started automatically when the broker starts and also restarted -# after a short delay (30 seconds) if the connection fails. -# -# Bridges using the "lazy" start type will be started automatically when the -# number of queued messages exceeds the number set with the "threshold" -# parameter. It will be stopped automatically after the time set by the -# "idle_timeout" parameter. Use this start type if you wish the connection to -# only be active when it is needed. -# -# A bridge using the "once" start type will be started automatically when the -# broker starts but will not be restarted if the connection fails. -#start_type automatic - -# Set the amount of time a bridge using the automatic start type will wait -# until attempting to reconnect. Defaults to 30 seconds. -#restart_timeout 30 - -# Set the amount of time a bridge using the lazy start type must be idle before -# it will be stopped. Defaults to 60 seconds. -#idle_timeout 60 - -# Set the number of messages that need to be queued for a bridge with lazy -# start type to be restarted. Defaults to 10 messages. -# Must be less than max_queued_messages. -#threshold 10 - -# If try_private is set to true, the bridge will attempt to indicate to the -# remote broker that it is a bridge not an ordinary client. If successful, this -# means that loop detection will be more effective and that retained messages -# will be propagated correctly. Not all brokers support this feature so it may -# be necessary to set try_private to false if your bridge does not connect -# properly. -#try_private true - -# Set the username to use when connecting to an MQTT v3.1 broker -# that requires authentication. -#username - -# Set the password to use when connecting to an MQTT v3.1 broker -# that requires authentication. This option is only used if -# username is also set. -#password - -# ----------------------------------------------------------------- -# Certificate based SSL/TLS support -# ----------------------------------------------------------------- -# Either bridge_cafile or bridge_capath must be defined to enable TLS support -# for this bridge. -# bridge_cafile defines the path to a file containing the -# Certificate Authority certificates that have signed the remote broker -# certificate. -# bridge_capath defines a directory that will be searched for files containing -# the CA certificates. For bridge_capath to work correctly, the certificate -# files must have ".crt" as the file ending and you must run "c_rehash " each time you add/remove a certificate. -#bridge_cafile -#bridge_capath - -# Path to the PEM encoded client certificate, if required by the remote broker. -#bridge_certfile - -# Path to the PEM encoded client private key, if required by the remote broker. -#bridge_keyfile - -# ----------------------------------------------------------------- -# PSK based SSL/TLS support -# ----------------------------------------------------------------- -# Pre-shared-key encryption provides an alternative to certificate based -# encryption. A bridge can be configured to use PSK with the bridge_identity -# and bridge_psk options. These are the client PSK identity, and pre-shared-key -# in hexadecimal format with no "0x". Only one of certificate and PSK based -# encryption can be used on one -# bridge at once. -#bridge_identity -#bridge_psk - - -# ================================================================= -# External config files -# ================================================================= - -# External configuration files may be included by using the -# include_dir option. This defines a directory that will be searched -# for config files. All files that end in '.conf' will be loaded as -# a configuration file. It is best to have this as the last option -# in the main file. This option will only be processed from the main -# configuration file. The directory specified must not contain the -# main configuration file. -#include_dir - -# ================================================================= -# Unsupported rsmb options - for the future -# ================================================================= - -#addresses -#round_robin - -# ================================================================= -# rsmb options - unlikely to ever be supported -# ================================================================= - -#ffdc_output -#max_log_entries -#trace_level -#trace_output diff --git a/benchmarks/graph.R b/benchmarks/graph.R deleted file mode 100755 index 5d2dee5..0000000 --- a/benchmarks/graph.R +++ /dev/null @@ -1,28 +0,0 @@ -#! /usr/bin/env Rscript - -args <- commandArgs(TRUE) -mosca <- read.csv(args[1]) -mosquitto <- read.csv(args[2]) -colors <- c("red", "blue") -lines <- c("solid", "solid") - -dfrm <- data.frame(Mosca=mosca$sample, - Mosquitto=mosquitto$sample) - -columns <- factor(colnames(dfrm)) - -quartz("QoS 1 unclean pub/sub latency on the same topic") -matplot(dfrm, type="l", pch=as.integer(columns), - col=colors, lty=lines, ylim=c(0,3000), xlim=c(0,5000), - main="QoS 1 unclean pub/sub latency on the same topic", xlab="Sample", ylab="Time (ms)") - -grid() -legend(0, 3000, - as.character(levels(columns)), - col=colors, lty=lines) - -dev.copy(pdf,args[3]) -dev.off() - -message("Press Return To Continue") -invisible(readLines("stdin", n=1)) diff --git a/benchmarks/no-log-mosquitto.conf b/benchmarks/no-log-mosquitto.conf deleted file mode 100644 index 2fcefe1..0000000 --- a/benchmarks/no-log-mosquitto.conf +++ /dev/null @@ -1,675 +0,0 @@ -# Config file for mosquitto -# -# See mosquitto.conf(5) for more information. -# -# Default values are shown, uncomment to change. -# -# Use the # character to indicate a comment, but only if it is the -# very first character on the line. - -# ================================================================= -# General configuration -# ================================================================= - -# Time in seconds to wait before resending an outgoing QoS=1 or -# QoS=2 message. -#retry_interval 20 - -# Time in seconds between updates of the $SYS tree. -#sys_interval 10 - -# Time in seconds between cleaning the internal message store of -# unreferenced messages. Lower values will result in lower memory -# usage but more processor time, higher values will have the -# opposite effect. -# Setting a value of 0 means the unreferenced messages will be -# disposed of as quickly as possible. -#store_clean_interval 10 - -# Write process id to a file. Default is a blank string which means -# a pid file shouldn't be written. -# This should be set to /var/run/mosquitto.pid if mosquitto is -# being run automatically on boot with an init script and -# start-stop-daemon or similar. -#pid_file - -# When run as root, drop privileges to this user and its primary -# group. -# Leave blank to stay as root, but this is not recommended. -# If run as a non-root user, this setting has no effect. -# Note that on Windows this has no effect and so mosquitto should -# be started by the user you wish it to run as. -#user mosquitto - -# The maximum number of QoS 1 and 2 messages currently inflight per -# client. -# This includes messages that are partway through handshakes and -# those that are being retried. Defaults to 20. Set to 0 for no -# maximum. Setting to 1 will guarantee in-order delivery of QoS 1 -# and 2 messages. -#max_inflight_messages 20 - -# The maximum number of QoS 1 and 2 messages to hold in a queue -# above those that are currently in-flight. Defaults to 100. Set -# to 0 for no maximum (not recommended). -# See also queue_qos0_messages. -#max_queued_messages 100 - -# Set to true to queue messages with QoS 0 when a persistent client is -# disconnected. These messages are included in the limit imposed by -# max_queued_messages. -# Defaults to false. -# Note that the MQTT v3.1 spec states that only QoS 1 and 2 messages -# should be saved in this situation so this is a non-standard option. -#queue_qos0_messages false - -# This option allows persistent clients (those with clean session set to false) -# to be removed if they do not reconnect within a certain time frame. This is a -# non-standard option. As far as the MQTT spec is concerned, persistent clients -# persist forever. -# Badly designed clients may set clean session to false whilst using a randomly -# generated client id. This leads to persistent clients that will never -# reconnect. This option allows these clients to be removed. -# -# The expiration period should be an integer followed by one of d w m y for -# day, week, month and year respectively. For example -# -# persistent_client_expiration 2m -# persistent_client_expiration 14d -# persistent_client_expiration 1y -# -# As this is a non-standard option, the default if not set is to never expire -# persistent clients. -#persistent_client_expiration - -# If a client is subscribed to multiple subscriptions that overlap, e.g. foo/# -# and foo/+/baz , then MQTT expects that when the broker receives a message on -# a topic that matches both subscriptions, such as foo/bar/baz, then the client -# should only receive the message once. -# Mosquitto keeps track of which clients a message has been sent to in order to -# meet this requirement. The allow_duplicate_messages option allows this -# behaviour to be disabled, which may be useful if you have a large number of -# clients subscribed to the same set of topics and are very concerned about -# minimising memory usage. -# It can be safely set to true if you know in advance that your clients will -# never have overlapping subscriptions, otherwise your clients must be able to -# correctly deal with duplicate messages even when then have QoS=2. -#allow_duplicate_messages false - -# ================================================================= -# Default listener -# ================================================================= - -# IP address/hostname to bind the default listener to. If not -# given, the default listener will not be bound to a specific -# address and so will be accessible to all network interfaces. -# bind_address ip-address/host name -#bind_address - -# Port to use for the default listener. -#port 1883 - -# The maximum number of client connections to allow. This is -# a per listener setting. -# Default is -1, which means unlimited connections. -# Note that other process limits mean that unlimited connections -# are not really possible. Typically the default maximum number of -# connections possible is around 1024. -#max_connections -1 - -# ----------------------------------------------------------------- -# Certificate based SSL/TLS support -# ----------------------------------------------------------------- -# The following options can be used to enable SSL/TLS support for -# this listener. Note that the recommended port for MQTT over TLS -# is 8883, but this must be set manually. -# -# See also the mosquitto-tls man page. - -# At least one of cafile or capath must be defined. They both -# define methods of accessing the PEM encoded Certificate -# Authority certificates that have signed your server certificate -# and that you wish to trust. -# cafile defines the path to a file containing the CA certificates. -# capath defines a directory that will be searched for files -# containing the CA certificates. For capath to work correctly, the -# certificate files must have ".crt" as the file ending and you must run -# "c_rehash " each time you add/remove a certificate. -#cafile -#capath - -# Path to the PEM encoded server certificate. -#certfile - -# Path to the PEM encoded keyfile. -#keyfile - -# By default a TLS enabled listener will operate in a similar fashion to a -# https enabled web server, in that the server has a certificate signed by a CA -# and the client will verify that it is a trusted certificate. The overall aim -# is encryption of the network traffic. By setting require_certificate to true, -# the client must provide a valid certificate in order for the network -# connection to proceed. This allows access to the broker to be controlled -# outside of the mechanisms provided by MQTT. -#require_certificate false - -# If require_certificate is true, you may set use_identity_as_username to true -# to use the CN value from the client certificate as a username. If this is -# true, the password_file option will not be used for this listener. -#use_identity_as_username false - -# If you have require_certificate set to true, you can create a certificate -# revocation list file to revoke access to particular client certificates. If -# you have done this, use crlfile to point to the PEM encoded revocation file. -#crlfile - -# If you wish to control which encryption ciphers are used, use the ciphers -# option. The list of available ciphers can be optained using the "openssl -# ciphers" command and should be provided in the same format as the output of -# that command. -#ciphers - -# ----------------------------------------------------------------- -# Pre-shared-key based SSL/TLS support -# ----------------------------------------------------------------- -# The following options can be used to enable PSK based SSL/TLS support for -# this listener. Note that the recommended port for MQTT over TLS is 8883, but -# this must be set manually. -# -# See also the mosquitto-tls man page and the "Certificate based SSL/TLS -# support" section. Only one of certificate or PSK encryption support can be -# enabled for any listener. - -# The psk_hint option enables pre-shared-key support for this listener and also -# acts as an identifier for this listener. The hint is sent to clients and may -# be used locally to aid authentication. The hint is a free form string that -# doesn't have much meaning in itself, so feel free to be creative. -# If this option is provided, see psk_file to define the pre-shared keys to be -# used or create a security plugin to handle them. -#psk_hint - -# Set use_identity_as_username to have the psk identity sent by the client used -# as its username. Authentication will be carried out using the PSK rather than -# the MQTT username/password and so password_file will not be used for this -# listener. -#use_identity_as_username false - -# When using PSK, the encryption ciphers used will be chosen from the list of -# available PSK ciphers. If you want to control which ciphers are available, -# use the "ciphers" option. The list of available ciphers can be optained -# using the "openssl ciphers" command and should be provided in the same format -# as the output of that command. -#ciphers - -# ================================================================= -# Extra listeners -# ================================================================= - -# Listen on a port/ip address combination. By using this variable -# multiple times, mosquitto can listen on more than one port. If -# this variable is used and neither bind_address nor port given, -# then the default listener will not be started. -# The port number to listen on must be given. Optionally, an ip -# address or host name may be supplied as a second argument. In -# this case, mosquitto will attempt to bind the listener to that -# address and so restrict access to the associated network and -# interface. By default, mosquitto will listen on all interfaces. -# listener port-number [ip address/host name] -#listener - -# The maximum number of client connections to allow. This is -# a per listener setting. -# Default is -1, which means unlimited connections. -# Note that other process limits mean that unlimited connections -# are not really possible. Typically the default maximum number of -# connections possible is around 1024. -#max_connections -1 - -# The listener can be restricted to operating within a topic hierarchy using -# the mount_point option. This is achieved be prefixing the mount_point string -# to all topics for any clients connected to this listener. This prefixing only -# happens internally to the broker; the client will not see the prefix. -#mount_point - -# ----------------------------------------------------------------- -# Certificate based SSL/TLS support -# ----------------------------------------------------------------- -# The following options can be used to enable certificate based SSL/TLS support -# for this listener. Note that the recommended port for MQTT over TLS is 8883, -# but this must be set manually. -# -# See also the mosquitto-tls man page and the "Pre-shared-key based SSL/TLS -# support" section. Only one of certificate or PSK encryption support can be -# enabled for any listener. - -# At least one of cafile or capath must be defined to enable certificate based -# TLS encryption. They both define methods of accessing the PEM encoded -# Certificate Authority certificates that have signed your server certificate -# and that you wish to trust. -# cafile defines the path to a file containing the CA certificates. -# capath defines a directory that will be searched for files -# containing the CA certificates. For capath to work correctly, the -# certificate files must have ".crt" as the file ending and you must run -# "c_rehash " each time you add/remove a certificate. -#cafile -#capath - -# Path to the PEM encoded server certificate. -#certfile - -# Path to the PEM encoded keyfile. -#keyfile - -# By default an TLS enabled listener will operate in a similar fashion to a -# https enabled web server, in that the server has a certificate signed by a CA -# and the client will verify that it is a trusted certificate. The overall aim -# is encryption of the network traffic. By setting require_certificate to true, -# the client must provide a valid certificate in order for the network -# connection to proceed. This allows access to the broker to be controlled -# outside of the mechanisms provided by MQTT. -#require_certificate false - -# If require_certificate is true, you may set use_identity_as_username to true -# to use the CN value from the client certificate as a username. If this is -# true, the password_file option will not be used for this listener. -#use_identity_as_username false - -# If you have require_certificate set to true, you can create a certificate -# revocation list file to revoke access to particular client certificates. If -# you have done this, use crlfile to point to the PEM encoded revocation file. -#crlfile - -# If you wish to control which encryption ciphers are used, use the ciphers -# option. The list of available ciphers can be optained using the "openssl -# ciphers" command and should be provided in the same format as the output of -# that command. -#ciphers - -# ----------------------------------------------------------------- -# Pre-shared-key based SSL/TLS support -# ----------------------------------------------------------------- -# The following options can be used to enable PSK based SSL/TLS support for -# this listener. Note that the recommended port for MQTT over TLS is 8883, but -# this must be set manually. -# -# See also the mosquitto-tls man page and the "Certificate based SSL/TLS -# support" section. Only one of certificate or PSK encryption support can be -# enabled for any listener. - -# The psk_hint option enables pre-shared-key support for this listener and also -# acts as an identifier for this listener. The hint is sent to clients and may -# be used locally to aid authentication. The hint is a free form string that -# doesn't have much meaning in itself, so feel free to be creative. -# If this option is provided, see psk_file to define the pre-shared keys to be -# used or create a security plugin to handle them. -#psk_hint - -# Set use_identity_as_username to have the psk identity sent by the client used -# as its username. Authentication will be carried out using the PSK rather than -# the MQTT username/password and so password_file will not be used for this -# listener. -#use_identity_as_username false - -# When using PSK, the encryption ciphers used will be chosen from the list of -# available PSK ciphers. If you want to control which ciphers are available, -# use the "ciphers" option. The list of available ciphers can be optained -# using the "openssl ciphers" command and should be provided in the same format -# as the output of that command. -#ciphers - -# ================================================================= -# Persistence -# ================================================================= - -# If persistence is enabled, save the in-memory database to disk -# every autosave_interval seconds. If set to 0, the persistence -# database will only be written when mosquitto exits. See also -# autosave_on_changes. -# Note that writing of the persistence database can be forced by -# sending mosquitto a SIGUSR1 signal. -#autosave_interval 1800 - -# If true, mosquitto will count the number of subscription changes, retained -# messages received and queued messages and if the total exceeds -# autosave_interval then the in-memory database will be saved to disk. -# If false, mosquitto will save the in-memory database to disk by treating -# autosave_interval as a time in seconds. -#autosave_on_changes false - -# Save persistent message data to disk (true/false). -# This saves information about all messages, including -# subscriptions, currently in-flight messages and retained -# messages. -# retained_persistence is a synonym for this option. -#persistence false - -# The filename to use for the persistent database, not including -# the path. -#persistence_file mosquitto.db - -# Location for persistent database. Must include trailing / -# Default is an empty string (current directory). -# Set to /var/lib/mosquitto/ if running as a proper service. -#persistence_location - -# ================================================================= -# Logging -# ================================================================= - -# Places to log to. Use multiple log_dest lines for multiple -# logging destinations. -# Possible destinations are: stdout stderr syslog topic -# stdout and stderr log to the console on the named output. -# syslog uses the userspace syslog facility which usually ends up -# in /var/log/messages or similar. -# topic logs to the broker topic '$SYS/broker/log/', -# where severity is one of D, E, W, N, I which are debug, error, -# warning, notice and information. -# Note that if the broker is running as a Windows service it will default to -# "log_dest none" and neither stdout nor stderr logging is available. -# Use "log_dest none" if you wish to disable logging. -#log_dest stderr - -# Types of messages to log. Use multiple log_type lines for logging -# multiple types of messages. -# Possible types are: debug, error, warning, notice, information, -# none. -# Note that debug type messages are for decoding the incoming -# network packets. -# They are not logged in syslog. -#log_type error -#log_type warning -#log_type notice -#log_type information -log_type none - -# If set to true, client connection and disconnection messages will be included -# in the log. -#connection_messages true - -# If set to true, add a timestamp value to each log message. -#log_timestamp true - -# ================================================================= -# Security -# ================================================================= - -# If set, only clients that have a matching prefix on their -# clientid will be allowed to connect to the broker. By default, -# all clients may connect. -# For example, setting "secure-" here would mean a client "secure- -# client" could connect but another with clientid "mqtt" couldn't. -#clientid_prefixes - -# Boolean value that determines whether clients that connect -# without providing a username are allowed to connect. If set to -# false then a password file should be created (see the -# password_file option) to control authenticated client access. -# Defaults to true. -#allow_anonymous true - -# In addition to the clientid_prefixes, allow_anonymous and TLS -# authentication options, username based authentication is also -# possible. The default support is described in "Default -# authentication and topic access control" below. The auth_plugin -# allows another authentication method to be used. -# Specify the path to the loadable plugin and see the -# "Authentication and topic access plugin options" section below. -#auth_plugin - -# ----------------------------------------------------------------- -# Default authentication and topic access control -# ----------------------------------------------------------------- - -# Control access to the broker using a password file. This file can be -# generated using the mosquitto_passwd utility. If TLS support is not compiled -# into mosquitto (it is recommended that TLS support should be included) then -# plain text passwords are used, in which case the file should be a text file -# with lines in the format: -# username:password -# The password (and colon) may be omitted if desired, although this -# offers very little in the way of security. -# -# See the TLS client require_certificate and use_identity_as_username options -# for alternative authentication options. -#password_file - -# Access may also be controlled using a pre-shared-key file. This requires -# TLS-PSK support and a listener configured to use it. The file should be text -# lines in the format: -# identity:key -# The key should be in hexadecimal format without a leading "0x". -#psk_file - -# Control access to topics on the broker using an access control list -# file. If this parameter is defined then only the topics listed will -# have access. -# If the first character of a line of the ACL file is a # it is treated as a -# comment. -# Topic access is added with lines of the format: -# -# topic [read|write] -# -# The access type is controlled using "read" or "write". This parameter -# is optional - if not given then the access is read/write. -# can contain the + or # wildcards as in subscriptions. -# -# The first set of topics are applied to anonymous clients, assuming -# allow_anonymous is true. User specific topic ACLs are added after a -# user line as follows: -# -# user -# -# The username referred to here is the same as in password_file. It is -# not the clientid. -# -# -# If is also possible to define ACLs based on pattern substitution within the -# topic. The patterns available for substition are: -# -# %c to match the client id of the client -# %u to match the username of the client -# -# The substitution pattern must be the only text for that level of hierarchy. -# -# The form is the same as for the topic keyword, but using pattern as the -# keyword. -# Pattern ACLs apply to all users even if the "user" keyword has previously -# been given. -# -# pattern [read|write] -# -# Example: -# -# pattern write sensor/%u/data -# -#acl_file - -# ----------------------------------------------------------------- -# Authentication and topic access plugin options -# ----------------------------------------------------------------- - -# If the auth_plugin option above is used, define options to pass to the -# plugin here as described by the plugin instructions. All options named -# using the format auth_opt_* will be passed to the plugin, for example: -# -# auth_opt_db_host -# auth_opt_db_port -# auth_opt_db_username -# auth_opt_db_password - - -# ================================================================= -# Bridges -# ================================================================= - -# A bridge is a way of connecting multiple MQTT brokers together. -# Create a new bridge using the "connection" option as described below. Set -# options for the bridges using the remaining parameters. You must specify the -# address and at least one topic to subscribe to. -# Each connection must have a unique name. -# Only a single address per configuration is currently supported, -# unlike in rsmb. -# The direction that the topic will be shared can be chosen by -# specifying out, in or both, where the default value is out. -# The QoS level of the bridged communication can be specified with the next -# topic option. The default QoS level is 0, to change the QoS the topic -# direction must also be given. -# The local and remote prefix options allow a topic to be remapped when it is -# bridged to/from the remote broker. This provides the ability to place a topic -# tree in an appropriate location. -# For more details see the mosquitto.conf man page. -# Multiple topics can be specified per connection, but be careful -# not to create any loops. -# If you are using bridges with cleansession set to false (the default), then -# you may get unexpected behaviour from incoming topics if you change what -# topics you are subscribing to. This is because the remote broker keeps the -# subscription for the old topic. If you have this problem, connect your bridge -# with cleansession set to true, then reconnect with cleansession set to false -# as normal. -#connection -#address [:] -#topic [[[out | in | both] qos-level] local-prefix remote-prefix] - -# Set the client id for this bridge connection. If not defined, -# this defaults to 'name.hostname' where name is the connection -# name and hostname is the hostname of this computer. -#clientid - -# Set the clean session variable for this bridge. -# When set to true, when the bridge disconnects for any reason, all -# messages and subscriptions will be cleaned up on the remote -# broker. Note that with cleansession set to true, there may be a -# significant amount of retained messages sent when the bridge -# reconnects after losing its connection. -# When set to false, the subscriptions and messages are kept on the -# remote broker, and delivered when the bridge reconnects. -#cleansession false - -# If set to true, publish notification messages to the local and remote brokers -# giving information about the state of the bridge connection. Retained -# messages are published to the topic $SYS/broker/connection//state -# unless the notification_topic option is used. -# If the message is 1 then the connection is active, or 0 if the connection has -# failed. -#notifications true - -# Choose the topic on which notification messages for this bridge are -# published. If not set, messages are published on the topic -# $SYS/broker/connection//state -#notification_topic - -# Set the keepalive interval for this bridge connection, in -# seconds. -#keepalive_interval 60 - -# Set the start type of the bridge. This controls how the bridge starts and -# can be one of three types: automatic, lazy and once. Note that RSMB provides -# a fourth start type "manual" which isn't currently supported by mosquitto. -# -# "automatic" is the default start type and means that the bridge connection -# will be started automatically when the broker starts and also restarted -# after a short delay (30 seconds) if the connection fails. -# -# Bridges using the "lazy" start type will be started automatically when the -# number of queued messages exceeds the number set with the "threshold" -# parameter. It will be stopped automatically after the time set by the -# "idle_timeout" parameter. Use this start type if you wish the connection to -# only be active when it is needed. -# -# A bridge using the "once" start type will be started automatically when the -# broker starts but will not be restarted if the connection fails. -#start_type automatic - -# Set the amount of time a bridge using the automatic start type will wait -# until attempting to reconnect. Defaults to 30 seconds. -#restart_timeout 30 - -# Set the amount of time a bridge using the lazy start type must be idle before -# it will be stopped. Defaults to 60 seconds. -#idle_timeout 60 - -# Set the number of messages that need to be queued for a bridge with lazy -# start type to be restarted. Defaults to 10 messages. -# Must be less than max_queued_messages. -#threshold 10 - -# If try_private is set to true, the bridge will attempt to indicate to the -# remote broker that it is a bridge not an ordinary client. If successful, this -# means that loop detection will be more effective and that retained messages -# will be propagated correctly. Not all brokers support this feature so it may -# be necessary to set try_private to false if your bridge does not connect -# properly. -#try_private true - -# Set the username to use when connecting to an MQTT v3.1 broker -# that requires authentication. -#username - -# Set the password to use when connecting to an MQTT v3.1 broker -# that requires authentication. This option is only used if -# username is also set. -#password - -# ----------------------------------------------------------------- -# Certificate based SSL/TLS support -# ----------------------------------------------------------------- -# Either bridge_cafile or bridge_capath must be defined to enable TLS support -# for this bridge. -# bridge_cafile defines the path to a file containing the -# Certificate Authority certificates that have signed the remote broker -# certificate. -# bridge_capath defines a directory that will be searched for files containing -# the CA certificates. For bridge_capath to work correctly, the certificate -# files must have ".crt" as the file ending and you must run "c_rehash " each time you add/remove a certificate. -#bridge_cafile -#bridge_capath - -# Path to the PEM encoded client certificate, if required by the remote broker. -#bridge_certfile - -# Path to the PEM encoded client private key, if required by the remote broker. -#bridge_keyfile - -# ----------------------------------------------------------------- -# PSK based SSL/TLS support -# ----------------------------------------------------------------- -# Pre-shared-key encryption provides an alternative to certificate based -# encryption. A bridge can be configured to use PSK with the bridge_identity -# and bridge_psk options. These are the client PSK identity, and pre-shared-key -# in hexadecimal format with no "0x". Only one of certificate and PSK based -# encryption can be used on one -# bridge at once. -#bridge_identity -#bridge_psk - - -# ================================================================= -# External config files -# ================================================================= - -# External configuration files may be included by using the -# include_dir option. This defines a directory that will be searched -# for config files. All files that end in '.conf' will be loaded as -# a configuration file. It is best to have this as the last option -# in the main file. This option will only be processed from the main -# configuration file. The directory specified must not contain the -# main configuration file. -#include_dir - -# ================================================================= -# Unsupported rsmb options - for the future -# ================================================================= - -#addresses -#round_robin - -# ================================================================= -# rsmb options - unlikely to ever be supported -# ================================================================= - -#ffdc_output -#max_log_entries -#trace_level -#trace_output diff --git a/benchmarks/single_pub.js b/benchmarks/single_pub.js deleted file mode 100755 index 5a3ae56..0000000 --- a/benchmarks/single_pub.js +++ /dev/null @@ -1,68 +0,0 @@ -#! /usr/bin/env node - -var mosca = require("../"); -var async = require("async"); -var runner = require("async_bench"); -var program = require("commander"); -var mqtt = require("mqtt"); - -function setup(done) { - - var client = mqtt.createClient(1883, "localhost", { clean: true }); - - client.on("connect", function () { - done(null, client); - }); -} - -function teardown(client, callback) { - client.end(); - process.nextTick(callback); -} - -function bench(pubs, client, done) { - client.publish("hello", "world", { qos: program.qos } , function () { - if(pubs === 0) { - done(null, client); - } else { - bench(--pubs, client, done); - } - }); -} - -program - .option("--header", "add header") - .option("-r, --runs ", "the number of runs to execute", parseInt, 10) - .option("-q, --qos ", "the QoS level (0, 1, 2)", parseInt, 0) - .option("-p, --pubs ", "the number of publish to do", parseInt, 1) - .parse(process.argv); - -function toCSV() { - var array = Array.prototype.slice.apply(arguments); - return array.reduce(function (acc, e) { - return acc + ", " + e; - }); -} - -runner({ - preHeat: program.runs, - runs: program.runs, - setup: setup, - bench: async.apply(bench, program.pubs), - teardown: teardown, - complete: function (err, results) { - if (err) { - console.log(err); - process.exit(0); - return; - } - - if(program.header) { - console.log(toCSV("mean", "standard deviation", "median", "mode", "runs")); - } - console.log(toCSV(results.mean, results.stdDev, results.median, results.mode, program.runs)); - setTimeout(function() { - process.exit(0); - }, 10); - } -}); diff --git a/benchmarks/single_pub_sub.js b/benchmarks/single_pub_sub.js deleted file mode 100755 index d01de84..0000000 --- a/benchmarks/single_pub_sub.js +++ /dev/null @@ -1,83 +0,0 @@ -#! /usr/bin/env node - -var mosca = require("../"); -var async = require("async"); -var runner = require("async_bench"); -var program = require("commander"); -var mqtt = require("mqtt"); - -function setup(done) { - - var client = mqtt.createClient(1883, "localhost", { clean: program.clean }); - - client.on("connect", function () { - client.subscribe("hello", { qos: program.qos }, function () { - done(null, client); - }); - }); - - client.on("error", function (err) { - console.log(err); - if(err.errno == 'EADDRINUSE') { - setTimeout(function () { - setup(done); - }, 1000); - } - }); - - client.on("message", function (){ - client.pass(null, client); - }); -} - -function teardown(client, callback) { - client.on("close", callback); - client.end(); -} - -function bench(client, done) { - client.pass = done; - client.publish("hello", "world"); -} - -program - .option("--clean", "use clean clients") - .option("--header", "add header") - .option("-r, --runs ", "the number of runs to execute", parseInt, 10) - .option("-q, --qos ", "the QoS level (0, 1, 2)", parseInt, 0) - .option("-n, --no-stat", "print only the samples, no stats") - .parse(process.argv); - -function toCSV() { - var array = Array.prototype.slice.apply(arguments); - return array.reduce(function (acc, e) { - return acc + ", " + e; - }); -} - -runner({ - preHeat: program.runs, - runs: program.runs, - setup: setup, - bench: bench, - teardown: teardown, - complete: function (err, results, samples) { - if (err) { - console.log(err); - return; - } else if(program.stat) { - if(program.header) { - console.log(toCSV("mean", "standard deviation", "median", "mode", "runs")); - } - console.log(toCSV(results.mean, results.stdDev, results.median, results.mode, program.runs)); - } else { - console.log("sample"); - samples.forEach(function(e) { - console.log(e); - }); - } - setTimeout(function() { - process.exit(0); - }, 100); - } -}); diff --git a/benchmarks/throughputCounter.js b/benchmarks/throughputCounter.js deleted file mode 100644 index 5939533..0000000 --- a/benchmarks/throughputCounter.js +++ /dev/null @@ -1,19 +0,0 @@ -var mqtt = require('mqtt'); - -var client = mqtt.createClient(1883); -var counter = 0; -var interval = 5000; - -function count() { - console.log("msg/s", counter / interval * 1000); - counter = 0; - setTimeout(count, interval); -} - -client.on('connect', function() { - count(); - this.subscribe('test'); - this.on("message", function() { - counter++; - }); -}); diff --git a/package.json b/package.json index 6d701bb..9ae2a3b 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,6 @@ "dox-foundation": "~0.5.4", "istanbul": "~0.4.0", "jshint": "~2.9.1", - "microtime": "~2.0.0", "mocha": "^2.0.1", "mongo-clean": "^1.1.0", "osenv": "^0.1.0", From 61fa934d81ebdb6a442e789028e605a057915d5c Mon Sep 17 00:00:00 2001 From: Matteo Collina Date: Mon, 23 May 2016 19:45:15 +0200 Subject: [PATCH 18/27] mongodb example --- examples/mongodb/start.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 examples/mongodb/start.js diff --git a/examples/mongodb/start.js b/examples/mongodb/start.js new file mode 100644 index 0000000..0d9e220 --- /dev/null +++ b/examples/mongodb/start.js @@ -0,0 +1,14 @@ +'use strict' + +var mosca = require('../../'); +var config = require('./config'); + +var server = new mosca.Server(config); + +server.on('error', function(err){ + console.log(err); +}); + +server.on('ready', function(){ + console.log('Mosca server is up and running'); +}); From 021517e0112169d6e345234ec5bb66cca394aff9 Mon Sep 17 00:00:00 2001 From: Matteo Collina Date: Mon, 23 May 2016 19:47:18 +0200 Subject: [PATCH 19/27] Use steed in place of async. Little speed benefit, but at least I eat my own dog food :). --- lib/cli.js | 4 +- lib/client.js | 75 ++++++++++++++++---------------- lib/persistence/abstract.js | 8 ++-- lib/persistence/levelup.js | 5 ++- lib/persistence/mongo.js | 13 +++--- lib/persistence/redis.js | 19 ++++---- lib/server.js | 25 ++++++----- package.json | 5 ++- test/abstract_server.js | 52 +++++++++++----------- test/authorizer.js | 4 +- test/cli.js | 20 ++++----- test/persistence/abstract.js | 14 +++--- test/persistence/levelup_spec.js | 2 +- test/persistence/mongo_spec.js | 2 +- test/persistence/utils_spec.js | 2 +- test/server.js | 14 +++--- test/server_mongo.js | 2 +- test/server_redis.js | 2 +- test/server_websocket.js | 2 +- test/server_websocket_secure.js | 2 +- 20 files changed, 138 insertions(+), 134 deletions(-) diff --git a/lib/cli.js b/lib/cli.js index 4a29e32..62739f5 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -29,7 +29,7 @@ var commander = require("commander"); var path = require("path"); var Authorizer = require("./authorizer"); var fs = require("fs"); -var async = require("async"); +var steed = require("steed")(); var bunyan = require("bunyan"); var Server = require("./server"); var persistence = require("./persistence"); @@ -193,7 +193,7 @@ function start(program, callback) { return false; }; - async.series([ + steed.series([ function(cb) { server = new Server(opts); server.on("ready", cb); diff --git a/lib/client.js b/lib/client.js index 06e3c62..ff9b62b 100644 --- a/lib/client.js +++ b/lib/client.js @@ -24,9 +24,8 @@ OTHER DEALINGS IN THE SOFTWARE. */ "use strict"; -var async = require("async"), - uuid = require("uuid"); - +var steed = require("steed")(); +var uuid = require("uuid"); var retimer = require('retimer'); function nop() {} @@ -112,7 +111,7 @@ Client.prototype._setup = function() { client.on("unsubscribe", function(packet) { that.setUpTimer(); that.logger.info({ packet: packet }, "unsubscribe received"); - async.parallel(packet.unsubscriptions.map(that.unsubscribeMapTo.bind(that)), function(err) { + steed.map(that, packet.unsubscriptions, that.unsubscribeMapTo, function(err) { if (err) { that.logger.warn(err); that.close(null, err.message); @@ -277,30 +276,28 @@ Client.prototype._buildForward = function() { * * @api private */ -Client.prototype.unsubscribeMapTo = function(topic) { +Client.prototype.unsubscribeMapTo = function(topic, cb) { var that = this; - return function(cb) { - var sub = that.subscriptions[topic]; - if (!sub || !sub.handler) { - that.server.emit("unsubscribed", topic, that); - return cb(); - } + var sub = that.subscriptions[topic]; + if (!sub || !sub.handler) { + that.server.emit("unsubscribed", topic, that); + return cb(); + } - that.server.ascoltatore.unsubscribe(topic, sub.handler, function(err) { - if (err) { - cb(err); - return; - } + that.server.ascoltatore.unsubscribe(topic, sub.handler, function(err) { + if (err) { + cb(err); + return; + } - if (!that._closing || that.clean) { - delete that.subscriptions[topic]; - that.logger.info({ topic: topic }, "unsubscribed"); - that.server.emit("unsubscribed", topic, that); - } + if (!that._closing || that.clean) { + delete that.subscriptions[topic]; + that.logger.info({ topic: topic }, "unsubscribed"); + that.server.emit("unsubscribed", topic, that); + } - cb(); - }); - }; + cb(); + }); }; /** @@ -320,11 +317,11 @@ Client.prototype.handleConnect = function(packet, completeConnection) { // a random ID. // Otherwise, the connection should be rejected. if(!this.id) { - + if(packet.protocolVersion == 4 && packet.clean) { this.id = uuid.v4(); - } + } else { logger.info("identifier rejected"); @@ -461,6 +458,18 @@ Client.prototype.handleAuthorizeSubscribe = function(err, success, s, cb) { ); }; +function handleEachSub (s, cb) { + /*jshint validthis:true */ + var that = this; + if (this.subscriptions[s.topic] === undefined) { + this.server.authorizeSubscribe(that, s.topic, function(err, success) { + that.handleAuthorizeSubscribe(err, success, s, cb); + }); + } else { + cb(null, true); + } +} + /** * Handle a subscribe packet. * @@ -473,21 +482,13 @@ Client.prototype.handleSubscribe = function(packet) { var granted = calculateGranted(this, packet); - async.map(packet.subscriptions, function(s, cb) { - if (that.subscriptions[s.topic] === undefined) { - server.authorizeSubscribe(that, s.topic, function(err, success) { - that.handleAuthorizeSubscribe(err, success, s, cb); - }); - } else { - cb(null, true); - } - }, function(err, authorized) { + steed.map(this, packet.subscriptions, handleEachSub, function(err, authorized) { if (err) { that.close(null, err.message); return; } - + that.server.persistClient(that); packet.subscriptions.forEach(function(sub, index) { @@ -599,7 +600,7 @@ Client.prototype.close = function(callback, reason) { that._closing = true; - async.parallel(Object.keys(that.subscriptions).map(that.unsubscribeMapTo.bind(that)), function(err) { + steed.map(that, Object.keys(that.subscriptions), that.unsubscribeMapTo, function(err) { if (err) { that.logger.info(err); } diff --git a/lib/persistence/abstract.js b/lib/persistence/abstract.js index c0c0060..4b03b95 100644 --- a/lib/persistence/abstract.js +++ b/lib/persistence/abstract.js @@ -24,7 +24,7 @@ OTHER DEALINGS IN THE SOFTWARE. */ "use strict"; -var async = require("async"); +var steed = require("steed")(); var EventEmitter = require("events").EventEmitter; var util = require("util"); @@ -89,7 +89,7 @@ AbstractPersistence.prototype.wire = function(server) { client.emit("error", err); return; } - async.each(matches, function(match, cb) { + steed.each(matches, function(match, cb) { client.forward(match.topic, match.payload, match, pattern, match.qos, cb); }, done); }); @@ -105,10 +105,10 @@ AbstractPersistence.prototype.wire = function(server) { client.emit("error", err); return; } - + var subs = Object.keys(subscriptions); - async.each(subs, function(topic, inCb) { + steed.each(subs, function(topic, inCb) { client.logger.debug({ topic: topic, qos: subscriptions[topic].qos }, "restoring subscription"); client.handleAuthorizeSubscribe( null, true, { diff --git a/lib/persistence/levelup.js b/lib/persistence/levelup.js index 2456efd..788fc46 100644 --- a/lib/persistence/levelup.js +++ b/lib/persistence/levelup.js @@ -25,13 +25,14 @@ OTHER DEALINGS IN THE SOFTWARE. "use strict"; var levelup = require("levelup"); +var from = require("array-from"); var sublevel = require("level-sublevel"); var AbstractPersistence = require("./abstract"); var JSONB = require("json-buffer"); var util = require("util"); var msgpack = require("msgpack5")(); var Matcher = require("./matcher"); -var async = require("async"); +var steed = require("steed"); var extend = require("extend"); var defaults = { valueEncoding: { @@ -230,7 +231,7 @@ LevelUpPersistence.prototype.lookupSubscriptions = function(client, done) { LevelUpPersistence.prototype.storeOfflinePacket = function(packet, done) { var that = this; var subs = this._subMatcher.match(packet.topic); - async.each(subs, function(key, cb) { + steed.map(from(subs), function(key, cb) { that._subscriptions.get(key, function(err, sub) { if (err) { return cb(err); diff --git a/lib/persistence/mongo.js b/lib/persistence/mongo.js index fcc8a0e..a5ef62c 100644 --- a/lib/persistence/mongo.js +++ b/lib/persistence/mongo.js @@ -28,7 +28,7 @@ var AbstractPersistence = require("./abstract"); var mongo = require('mongodb'); var MongoClient = mongo.MongoClient; var util = require("util"); -var async = require("async"); +var steed = require("steed")(); var Matcher = require("./matcher"); var topicPatterns = require("./utils").topicPatterns; var extend = require("extend"); @@ -95,11 +95,11 @@ function MongoPersistence(options, done) { } that.db = db; - async.parallel([ + steed.parallel([ function(cb) { db.collection("subscriptions", function(err, coll) { that._subscriptions = coll; - async.parallel([ + steed.parallel([ that._subscriptions.ensureIndex.bind(that._subscriptions, "client"), that._subscriptions.ensureIndex.bind(that._subscriptions, { "added": 1 }, { expireAfterSeconds: Math.round(that.options.ttl.subscriptions / 1000 )} ) ], cb); @@ -112,7 +112,7 @@ function MongoPersistence(options, done) { } that._packets = coll; - async.series([ + steed.series([ that._packets.ensureIndex.bind(that._packets, "client"), function(cb){ // Check expiration indexes. If not exist, create; If exist but with different TTL, delete and recreate; Otherwise, do nothing. @@ -177,7 +177,6 @@ util.inherits(MongoPersistence, AbstractPersistence); * * @api private */ - MongoPersistence.prototype.storeSubscriptions = function(client, done) { var subscriptions; @@ -188,7 +187,7 @@ MongoPersistence.prototype.storeSubscriptions = function(client, done) { return client.subscriptions[key].qos > 0; }); - async.each(subscriptions, function(key, cb) { + steed.each(subscriptions, function(key, cb) { that._subscriptions.findAndModify({ client: client.id, topic: key @@ -237,7 +236,7 @@ MongoPersistence.prototype.lookupSubscriptions = function(client, done) { } ]; - async.parallel(toExecute, function(err) { + steed.parallel(toExecute, function(err) { done(null, {}); }); }); diff --git a/lib/persistence/redis.js b/lib/persistence/redis.js index 677c6b6..725108f 100644 --- a/lib/persistence/redis.js +++ b/lib/persistence/redis.js @@ -25,10 +25,11 @@ OTHER DEALINGS IN THE SOFTWARE. "use strict"; var AbstractPersistence = require("./abstract"); +var from = require("array-from"); var Redis = require("ioredis"); var util = require("util"); var Matcher = require("./matcher"); -var async = require("async"); +var steed = require("steed")(); var extend = require("extend"); var shortid = require("shortid"); var defaults = { @@ -130,7 +131,7 @@ function RedisPersistence(options, callback) { if (err) { return callback && callback(err, that); } - async.each(keys, function(k,next){ + steed.each(keys, function(k,next){ newSub(k,null,false,next); }, function(err) { if (callback) { @@ -204,7 +205,7 @@ RedisPersistence.prototype.lookupRetained = function(pattern, done) { packet = JSON.parse(packet); packet.payload = new Buffer(packet.payload); - + matched.push(packet); } @@ -222,7 +223,7 @@ RedisPersistence.prototype.lookupRetained = function(pattern, done) { return matcher.match(topic).size > 0; }); - async.each(topics, match, function(err) { + steed.each(topics, match, function(err) { done(err, matched); }); }); @@ -278,7 +279,7 @@ RedisPersistence.prototype.storeSubscriptions = function(client, cb) { RedisPersistence.prototype._cleanClient = function(client, done) { var that = this; - + var key = "client:sub:" + client.id; this._client.get(key, function(err, subs) { @@ -288,7 +289,7 @@ RedisPersistence.prototype._cleanClient = function(client, done) { that._subMatcher.remove(sub, client.id); }); - async.parallel([ + steed.parallel([ function(cb) { that._client.del(key, cb); }, @@ -300,7 +301,7 @@ RedisPersistence.prototype._cleanClient = function(client, done) { done(err, {}); } }); - }); + }); }; RedisPersistence.prototype.lookupSubscriptions = function(client, cb) { @@ -334,7 +335,7 @@ RedisPersistence.prototype.storeOfflinePacket = function(packet, done) { var that = this; var matches = this._subMatcher.match(packet.topic); - async.each(matches, function(client, cb) { + steed.each(from(matches), function(client, cb) { that._storePacket(client, packet, cb); }, done); }; @@ -456,7 +457,7 @@ RedisPersistence.prototype.close = function(done) { var that = this; - async.each([ + steed.each([ "_client", "_pubSubClient" ], function(client, cb) { if (that[client]) { diff --git a/lib/server.js b/lib/server.js index f50bbfa..028ed33 100755 --- a/lib/server.js +++ b/lib/server.js @@ -26,7 +26,7 @@ OTHER DEALINGS IN THE SOFTWARE. var Connection = require("mqtt-connection"); var ws = require("websocket-stream"); -var async = require("async"); +var steed = require("steed")(); var ascoltatori = require("ascoltatori"); var EventEmitter = require("events").EventEmitter; var bunyan = require("bunyan"); @@ -72,14 +72,14 @@ var nop = function() {}; * Interface may contain following properties: * - `type`, name of a build-in type or a custom type factory * - `port`, target port, overrides default port infered from `type` - * - `host`, target host, overrides - * + * - `host`, target host, overrides + * * Built-in interface types: * - `mqtt`, normal mqtt, port: 1883 * - `mqtts`, mqtt over ssl, port: 8883, requires `credentials` * - `http`, mqtt over websocket, port: 3000 * - `https`, mqtt over secure websocket, port: 3001, requires `credentials` - * + * * Events: * - `clientConnected`, when a client is connected; * the client is passed as a parameter. @@ -163,21 +163,21 @@ function Server(opts, callback) { // initialize servers list this.servers = []; - async.series([ + steed.series([ - // async.series: wait for ascoltatore + // steed.series: wait for ascoltatore function (done) { if(that.modernOpts.ascoltatore) { that.ascoltatore = that.modernOpts.ascoltatore; done(); - } + } else { that.ascoltatore = ascoltatori.build(that.modernOpts.backend, done); } }, - // async.series: wait for persistence + // steed.series: wait for persistence function (done) { // REFACTOR: partially move to options.validate and options.populate? @@ -199,9 +199,10 @@ function Server(opts, callback) { } }, - // async.series: iterate over defined interfaces, build servers and listen + // steed.series: iterate over defined interfaces, build servers and listen function (done) { - async.eachSeries(that.modernOpts.interfaces, function (iface, dn) { + + steed.eachSeries(that.modernOpts.interfaces, function (iface, dn) { var fallback = that.modernOpts; var host = iface.host || that.modernOpts.host; var port = iface.port || that.modernOpts.port; @@ -213,7 +214,7 @@ function Server(opts, callback) { }, done); }, - // async.series: log startup information + // steed.series: log startup information function (done) { var logInfo = {}; @@ -580,7 +581,7 @@ Server.prototype.close = function(callback) { stuffToClose.push(that.persistence); } - async.each(stuffToClose, function(toClose, cb) { + steed.each(stuffToClose, function(toClose, cb) { toClose.close(cb, "server closed"); }, function() { that.ascoltatore.close(function () { diff --git a/package.json b/package.json index 9ae2a3b..d24ecf1 100644 --- a/package.json +++ b/package.json @@ -69,8 +69,8 @@ "underscore": "^1.7.0" }, "dependencies": { - "ascoltatori": "^2.0.0", - "async": "^2.0.0-rc.3", + "array-from": "^2.1.1", + "ascoltatori": "^3.0.0", "brfs": "~1.4.2", "bunyan": "^1.5.1", "clone": "^1.0.2", @@ -94,6 +94,7 @@ "retimer": "^1.0.1", "shortid": "^2.2.4", "st": "~1.1.0", + "steed": "^1.0.0", "uuid": "^2.0.1", "websocket-stream": "~3.1.0" }, diff --git a/test/abstract_server.js b/test/abstract_server.js index 50a2060..2fdbd36 100644 --- a/test/abstract_server.js +++ b/test/abstract_server.js @@ -1,4 +1,4 @@ -var async = require("async"); +var steed = require("steed"); var ascoltatori = require("ascoltatori"); module.exports = function(moscaSettings, createConnection) { @@ -23,7 +23,7 @@ module.exports = function(moscaSettings, createConnection) { instances.push(secondInstance); } - async.each(instances, function(instance, cb) { + steed.each(instances, function(instance, cb) { instance.close(cb); }, function() { setImmediate(done); @@ -120,9 +120,9 @@ module.exports = function(moscaSettings, createConnection) { }); it("should publish each subscribe to '$SYS/{broker-id}/new/subscribes'", function(done) { - var d = donner(2, done), - connectedClient = null, - publishedClientId = null; + var d = donner(2, done); + var connectedClient = null; + var publishedClientId = null; function verify() { if (connectedClient && publishedClientId) { @@ -139,7 +139,7 @@ module.exports = function(moscaSettings, createConnection) { } ]; - connectedClient = client; + connectedClient = client; instance.once("published", function(packet) { expect(packet.topic).to.be.equal("$SYS/" + instance.id + "/new/subscribes"); @@ -217,7 +217,7 @@ module.exports = function(moscaSettings, createConnection) { instances.push(serverOne); instances.push(serverTwo); - async.each(instances, function(instance, cb) { + steed.each(instances, function(instance, cb) { if (instance) { instance.close(cb); } else { @@ -244,7 +244,7 @@ module.exports = function(moscaSettings, createConnection) { settingsOne.backend = settingsTwo.backend = settings.backend; - async.series([ + steed.series([ function(cb) { serverOne = new mosca.Server(settingsOne, function(err, server) { serverOne.on('clientDisconnected', function(serverClient, reason) { @@ -382,7 +382,7 @@ module.exports = function(moscaSettings, createConnection) { var d = donner(2, done); var opts = buildOpts(), clientId = "123456789"; opts.clientId = clientId; - async.waterfall([ + steed.waterfall([ function(cb) { buildAndConnect(d, opts, function(client1) { cb(null, client1); @@ -890,7 +890,7 @@ module.exports = function(moscaSettings, createConnection) { client.on('connack', function() { client.disconnect(); client.disconnect(); - async.setImmediate(function() { + setImmediate(function() { client.stream.end(); }); }); @@ -918,7 +918,7 @@ module.exports = function(moscaSettings, createConnection) { it("should emit a ready and closed events", function(done) { var server = new mosca.Server(moscaSettings()); - async.series([ + steed.series([ function(cb) { server.on("ready", cb); @@ -1032,7 +1032,7 @@ module.exports = function(moscaSettings, createConnection) { it("should support unsubscribing a single client", function(done) { var d = donner(3, done); - async.waterfall([ + steed.waterfall([ function(cb) { buildAndConnect(d, function(client1) { @@ -1249,7 +1249,7 @@ module.exports = function(moscaSettings, createConnection) { it("should support will message", function(done) { - async.waterfall([ + steed.waterfall([ function(cb) { var client = createConnection(settings.port, settings.host); @@ -1654,7 +1654,7 @@ module.exports = function(moscaSettings, createConnection) { it("should support retained messages", function(done) { - async.waterfall([ + steed.waterfall([ function(cb) { var client = createConnection(settings.port, settings.host); @@ -1717,7 +1717,7 @@ module.exports = function(moscaSettings, createConnection) { it("should return only a single retained message", function(done) { - async.waterfall([ + steed.waterfall([ function(cb) { buildClient(cb, function(client) { @@ -1804,14 +1804,14 @@ module.exports = function(moscaSettings, createConnection) { opts.clientId = "mosca-unclean-clients-test"; opts.clean = false; - async.series([ + steed.series([ function(cb) { buildAndConnect(cb, opts, function(client, connack) { - + // sessionPresent must be false expect(connack.sessionPresent).to.be.eql(false); - + var subscriptions = [{ topic: "hello", qos: 1 @@ -1830,10 +1830,10 @@ module.exports = function(moscaSettings, createConnection) { function(cb) { buildAndConnect(cb, opts, function(client, connack) { - + // reconnection sessionPresent must be true expect(connack.sessionPresent).to.be.eql(true); - + client.publish({ topic: "hello", qos: 1, @@ -1858,7 +1858,7 @@ module.exports = function(moscaSettings, createConnection) { opts.clientId = "mosca-unclean-client-test"; opts.clean = false; - async.series([ + steed.series([ function(cb) { buildAndConnect(cb, opts, function(client, connack) { @@ -1921,7 +1921,7 @@ module.exports = function(moscaSettings, createConnection) { opts.clientId = "mosca-unclean-client-test"; - async.series([ + steed.series([ function(cb) { opts.clean = false; @@ -1970,7 +1970,7 @@ module.exports = function(moscaSettings, createConnection) { // reconnection sessionPresent must be false, it is clean expect(connack.sessionPresent).to.be.eql(false); - + client.on("publish", function(packet) { cb(new Error("unexpected publish")); }); @@ -2033,7 +2033,7 @@ module.exports = function(moscaSettings, createConnection) { }); } - async.waterfall([ + steed.waterfall([ step1, step2, step3, // two times! step1, step2, step3 @@ -2062,7 +2062,7 @@ module.exports = function(moscaSettings, createConnection) { opts.clean = false; opts.keepalive = 0; - async.series([ + steed.series([ function(cb) { buildAndConnect(cb, opts, function(client) { @@ -2122,7 +2122,7 @@ module.exports = function(moscaSettings, createConnection) { opts.clean = false; opts.keepalive = 0; - async.series([ + steed.series([ function(cb) { buildAndConnect(cb, opts, function(client) { diff --git a/test/authorizer.js b/test/authorizer.js index f2290d2..c057ca0 100644 --- a/test/authorizer.js +++ b/test/authorizer.js @@ -1,7 +1,7 @@ "use strict"; var hasher = require("pbkdf2-password")(); -var async = require("async"); +var steed = require("steed"); describe("mosca.Authorizer", function() { @@ -75,7 +75,7 @@ describe("mosca.Authorizer", function() { }); it("it should not authenticate a removed user", function(done) { - async.waterfall([ + steed.waterfall([ authorizer.addUser.bind(authorizer, "matteo", "collina"), authorizer.rmUser.bind(authorizer, "matteo"), instance.bind(null, client, "matteo", "collina") diff --git a/test/cli.js b/test/cli.js index b9031d3..117cf52 100644 --- a/test/cli.js +++ b/test/cli.js @@ -1,4 +1,4 @@ -var async = require("async"); +var steed = require("steed"); var tmp = require('tmp'); var fs = require("fs"); var mqtt = require("mqtt"); @@ -21,11 +21,11 @@ describe("mosca.cli", function() { }); afterEach(function(done) { - async.parallel(servers.map(function(s) { - return function(cb) { - s.close(cb); - }; - }), function() { + var count = 0; + steed.each(servers, function(s, cb) { + count++; + s.close(cb); + }, function() { done(); }); }); @@ -36,7 +36,7 @@ describe("mosca.cli", function() { servers.unshift(server); callback(server); } - async.setImmediate(done.bind(null, err)); + setImmediate(done.bind(null, err)); }); }; @@ -263,7 +263,7 @@ describe("mosca.cli", function() { it("should support authorizing an authorized client", function(done) { args.push("--credentials"); args.push("test/credentials.json"); - async.waterfall([ + steed.waterfall([ function(cb) { mosca.cli(args, cb); }, @@ -293,7 +293,7 @@ describe("mosca.cli", function() { it("should support negating an unauthorized client", function(done) { args.push("--credentials"); args.push("test/credentials.json"); - async.waterfall([ + steed.waterfall([ function(cb) { mosca.cli(args, cb); }, @@ -330,7 +330,7 @@ describe("mosca.cli", function() { var cloned = null; - async.waterfall([ + steed.waterfall([ function(cb) { tmp.file(cb); }, diff --git a/test/persistence/abstract.js b/test/persistence/abstract.js index 387de82..a0f6a45 100644 --- a/test/persistence/abstract.js +++ b/test/persistence/abstract.js @@ -1,6 +1,6 @@ "use strict"; -var async = require("async"); +var steed = require("steed"); var EventEmitter = require("events").EventEmitter; module.exports = function(create, buildOpts) { @@ -78,7 +78,7 @@ module.exports = function(create, buildOpts) { var instance = this.instance; - async.series([ + steed.series([ function(cb) { instance.storeRetained(packet, cb); }, @@ -111,7 +111,7 @@ module.exports = function(create, buildOpts) { var instance = this.instance; - async.parallel([ + steed.parallel([ function(cb) { instance.storeRetained(getPacket(), cb); }, function(cb) { instance.storeRetained(getPacket(), cb); }, function(cb) { instance.storeRetained(getPacket(), cb); }, @@ -144,7 +144,7 @@ module.exports = function(create, buildOpts) { var instance = this.instance; - async.series([ + steed.series([ instance.storeRetained.bind(instance, packet), instance.storeRetained.bind(instance, packet2), function(cb) { @@ -176,7 +176,7 @@ module.exports = function(create, buildOpts) { var instance = this.instance; - async.series([ + steed.series([ instance.storeRetained.bind(instance, packet), instance.storeRetained.bind(instance, packet2), function(cb) { @@ -207,7 +207,7 @@ module.exports = function(create, buildOpts) { var instance = this.instance; - async.series([ + steed.series([ function(cb) { instance.storeRetained(packet1, cb); }, @@ -245,7 +245,7 @@ module.exports = function(create, buildOpts) { var instance = this.instance; - async.series([ + steed.series([ function(cb) { instance.storeRetained(packet1, cb); }, diff --git a/test/persistence/levelup_spec.js b/test/persistence/levelup_spec.js index 049f8d1..32a4a36 100644 --- a/test/persistence/levelup_spec.js +++ b/test/persistence/levelup_spec.js @@ -2,7 +2,7 @@ var abstract = require("./abstract"); var LevelUp = require("../../").persistence.LevelUp; -var async = require("async"); +var steed = require("steed"); var tmpdir = require("osenv").tmpdir(); var path = require("path"); var rimraf = require("rimraf"); diff --git a/test/persistence/mongo_spec.js b/test/persistence/mongo_spec.js index 7c5c4cf..a3c0660 100644 --- a/test/persistence/mongo_spec.js +++ b/test/persistence/mongo_spec.js @@ -4,7 +4,7 @@ var abstract = require("./abstract"); var Mongo = require("../../").persistence.Mongo; var MongoClient = require('mongodb').MongoClient; var clean = require("mongo-clean"); -var async = require("async"); +var steed = require("steed"); describe("mosca.persistence.Mongo", function() { diff --git a/test/persistence/utils_spec.js b/test/persistence/utils_spec.js index 11dfac0..2661f51 100644 --- a/test/persistence/utils_spec.js +++ b/test/persistence/utils_spec.js @@ -3,7 +3,7 @@ var abstract = require("./abstract"); var utilities = require("../../lib/persistence/utils"); var topicPatterns = utilities.topicPatterns; -var async = require("async"); +var steed = require("steed"); describe("persistence utilities", function() { diff --git a/test/server.js b/test/server.js index be8189a..68da2e4 100644 --- a/test/server.js +++ b/test/server.js @@ -1,4 +1,4 @@ -var async = require("async"); +var steed = require("steed"); var ascoltatori = require("ascoltatori"); var abstractServerTests = require("./abstract_server"); var net = require("net"); @@ -530,7 +530,7 @@ describe("mosca.Server - MQTT backend", function() { instances = [secondInstance].concat(instances); } - async.parallel(instances.map(function(i) { + steed.parallel(instances.map(function(i) { return function(cb) { i.close(cb); }; @@ -597,7 +597,7 @@ describe("mosca.Server - MQTT backend", function() { var server = new mosca.Server(newSettings); - async.series([ + steed.series([ function(cb) { server.on("ready", cb); @@ -618,7 +618,7 @@ describe("mosca.Server - MQTT backend", function() { it("should support subscribing correctly to wildcards in a tree-based topology", function(done) { var d = donner(3, done); - async.waterfall([ + steed.waterfall([ function(cb) { settings.backend = { @@ -704,7 +704,7 @@ describe("mosca.Server - MQTT backend", function() { it("should not wrap messages with \"\" in a tree-based topology", function(done) { var d = donner(2, done); - async.waterfall([ + steed.waterfall([ function(cb) { buildAndConnect(d, function(client1) { @@ -773,7 +773,7 @@ describe("mosca.Server - MQTT backend", function() { var server = new mosca.Server(newSettings); - async.series([ + steed.series([ function(cb) { server.on("ready", cb); @@ -801,7 +801,7 @@ describe("mosca.Server - MQTT backend", function() { var server = new mosca.Server(newSettings); - async.series([ + steed.series([ function(cb) { server.on("ready", cb); diff --git a/test/server_mongo.js b/test/server_mongo.js index 7f5e77d..a9ebc66 100644 --- a/test/server_mongo.js +++ b/test/server_mongo.js @@ -1,5 +1,5 @@ var mqtt = require("mqtt"); -var async = require("async"); +var steed = require("steed"); var ascoltatori = require("ascoltatori"); var abstractServerTests = require("./abstract_server"); var MongoClient = require("mongodb").MongoClient; diff --git a/test/server_redis.js b/test/server_redis.js index 59f26ae..517e112 100644 --- a/test/server_redis.js +++ b/test/server_redis.js @@ -1,5 +1,5 @@ var mqtt = require("mqtt"); -var async = require("async"); +var steed = require("steed"); var ascoltatori = require("ascoltatori"); var abstractServerTests = require("./abstract_server"); var redis = require("ioredis"); diff --git a/test/server_websocket.js b/test/server_websocket.js index 473d77e..db8d378 100644 --- a/test/server_websocket.js +++ b/test/server_websocket.js @@ -1,4 +1,4 @@ -var async = require("async"); +var steed = require("steed"); var ascoltatori = require("ascoltatori"); var abstractServerTests = require("./abstract_server"); var createConnection = require("./helpers/createWebsocketConnection"); diff --git a/test/server_websocket_secure.js b/test/server_websocket_secure.js index 12706cb..4c7b317 100644 --- a/test/server_websocket_secure.js +++ b/test/server_websocket_secure.js @@ -1,4 +1,4 @@ -var async = require("async"); +var steed = require("steed"); var ascoltatori = require("ascoltatori"); var abstractServerTests = require("./abstract_server"); var request = require('supertest'); From 4cae9af28d53cf0b330d90f3eb7a4b07a981c60d Mon Sep 17 00:00:00 2001 From: Matteo Collina Date: Mon, 23 May 2016 22:11:56 +0200 Subject: [PATCH 20/27] Removed async-bench --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index d24ecf1..4f0d4a3 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,6 @@ "author": "Matteo Collina ", "license": "MIT", "devDependencies": { - "async_bench": "~0.5.1", "chai": "^3.5.0", "coveralls": "~2.11.1", "dox-foundation": "~0.5.4", From 12fb66f78e829774be098238bd18ac5f1761b8b9 Mon Sep 17 00:00:00 2001 From: Matteo Collina Date: Mon, 23 May 2016 22:34:19 +0200 Subject: [PATCH 21/27] Moved from bunyan to pino. --- README.md | 31 ++----------------- lib/cli.js | 5 ++- lib/options.js | 14 ++++----- lib/server.js | 7 ++--- package.json | 1 - test/cli.js | 53 -------------------------------- test/common.js | 7 ----- test/persistence/abstract.js | 45 ++++++++++++++------------- test/persistence/levelup_spec.js | 3 +- test/server.js | 3 +- test/server_mongo.js | 3 +- test/server_redis.js | 3 +- test/server_secure.js | 3 +- test/server_websocket.js | 3 +- test/server_websocket_secure.js | 3 +- 15 files changed, 45 insertions(+), 139 deletions(-) diff --git a/README.md b/README.md index 9e0c9cd..f0c214c 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,8 @@ Mosca   [![Build Status](https://travis-ci.org/mcollina/mosca.png ### Standalone ```bash -npm install mosca bunyan -g -mosca -v | bunyan +npm install mosca pino -g +mosca -v | pino ``` ### Embedded @@ -124,37 +124,10 @@ Chat with us on [Mosca's room](https://gitter.im/mcollina/mosca) on Gitter. * [MQTT protocol](http://mqtt.org) * [MQTT.js](http://github.com/adamvr/MQTT.js) - ## Authors [Matteo Collina](http://twitter.com/matteocollina) -## Contributors - - - - - - - - - - - - - - - - -
David HallsGitHub/davedoesdev
Andrea ReginatoGitHub/andreareginato
Chris WigginsGitHub/chriswiggins
Samir NaikGitHub/samirnaik
Leo SteinerGitHub/ldstein
John KokkinidisGitHub/SudoPlz
Morgan ChengGitHub/mocheng
- ## Logo [Sam Beck](http://two-thirty.tumblr.com) diff --git a/lib/cli.js b/lib/cli.js index 62739f5..635dbfe 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -30,7 +30,6 @@ var path = require("path"); var Authorizer = require("./authorizer"); var fs = require("fs"); var steed = require("steed")(); -var bunyan = require("bunyan"); var Server = require("./server"); var persistence = require("./persistence"); @@ -247,8 +246,8 @@ module.exports = function cli(argv, callback) { .option("--broker-id ", "the id of the broker in the $SYS/ namespace") .option("-c, --config ", "the config file to use (override every other option)") .option("-d, --db ", "the path were to store the database") - .option("-v, --verbose", "set the bunyan log to INFO") - .option("--very-verbose", "set the bunyan log to DEBUG"); + .option("-v, --verbose", "set the log level to INFO") + .option("--very-verbose", "set the log level to DEBUG"); function loadAuthorizerAndSave(cb) { runned = true; diff --git a/lib/options.js b/lib/options.js index 9ee6069..ca6009b 100755 --- a/lib/options.js +++ b/lib/options.js @@ -24,7 +24,7 @@ OTHER DEALINGS IN THE SOFTWARE. */ "use strict"; -var bunyan = require("bunyan"); +var pino = require("pino"); var extend = require("extend"); var clone = require("clone"); var jsonschema = require("jsonschema"); @@ -339,12 +339,12 @@ function defaultsLegacy() { maxInflightMessages: 1024, logger: { name: "mosca", - level: 40, + level: "warn", serializers: { client: serializers.clientSerializer, packet: serializers.packetSerializer, - req: bunyan.stdSerializers.req, - res: bunyan.stdSerializers.res + req: pino.stdSerializers.req, + res: pino.stdSerializers.res } } }; @@ -375,12 +375,12 @@ function defaultsModern() { maxInflightMessages: 1024, logger: { name: "mosca", - level: 40, + level: "warn", serializers: { client: serializers.clientSerializer, packet: serializers.packetSerializer, - req: bunyan.stdSerializers.req, - res: bunyan.stdSerializers.res + req: pino.stdSerializers.req, + res: pino.stdSerializers.res } } }; diff --git a/lib/server.js b/lib/server.js index 028ed33..fdc7897 100755 --- a/lib/server.js +++ b/lib/server.js @@ -29,7 +29,7 @@ var ws = require("websocket-stream"); var steed = require("steed")(); var ascoltatori = require("ascoltatori"); var EventEmitter = require("events").EventEmitter; -var bunyan = require("bunyan"); +var pino = require("pino"); var extend = require("extend"); var Client = require("./client"); var Stats = require("./stats"); @@ -55,8 +55,7 @@ var nop = function() {}; * that will power this server. * - `ascoltatore`, the ascoltatore to use (instead of `backend`). * - `maxInflightMessages`, the maximum number of inflight messages per client. - * - `logger`, the options for Bunyan. - * - `logger.childOf`, the parent Bunyan logger. + * - `logger`, the options for Pino. * - `persistence`, the options for the persistence. * A sub-key `factory` is used to specify what persistence * to use. @@ -148,7 +147,7 @@ function Server(opts, callback) { delete this.modernOpts.logger.name; this.logger = this.logger.child(this.modernOpts.logger); } else { - this.logger = bunyan.createLogger(this.modernOpts.logger); + this.logger = pino(this.modernOpts.logger); } if(this.modernOpts.stats) { diff --git a/package.json b/package.json index 4f0d4a3..55e980d 100644 --- a/package.json +++ b/package.json @@ -71,7 +71,6 @@ "array-from": "^2.1.1", "ascoltatori": "^3.0.0", "brfs": "~1.4.2", - "bunyan": "^1.5.1", "clone": "^1.0.2", "commander": "~2.9.0", "deepcopy": "^0.6.1", diff --git a/test/cli.js b/test/cli.js index 117cf52..2ed378d 100644 --- a/test/cli.js +++ b/test/cli.js @@ -50,45 +50,6 @@ describe("mosca.cli", function() { }); }); - it("should create a bunyan logger", function(done) { - args.push("-v"); - var s = startServer(done, function(server) { - expect(server.logger).to.exist; - }); - - if (s.logger) { - s.logger.streams.pop(); - } - }); - - it("should set the logging level to 40", function(done) { - startServer(done, function(server) { - expect(server.logger.level()).to.equal(40); - }); - }); - - it("should support a verbose option by setting the bunyan level to 30", function(done) { - args.push("-v"); - var s = startServer(done, function(server) { - expect(server.logger.level()).to.equal(30); - }); - - if (s.logger) { - s.logger.streams.pop(); - } - }); - - it("should support a very verbose option by setting the bunyan level to 20", function(done) { - args.push("--very-verbose"); - var s = startServer(done, function(server) { - expect(server.logger.level()).to.equal(20); - }); - - if (s.logger) { - s.logger.streams.pop(); - } - }); - it("should support a port flag", function(done) { args.push("-p"); args.push("2883"); @@ -153,20 +114,6 @@ describe("mosca.cli", function() { }); }); - it("should create necessary default options even if not specified in config file", function(done) { - args.push("-c"); - args.push(process.cwd() + "/test/sample_config.js"); - args.push("-v"); - - var s = startServer(done, function(server) { - expect(server.opts).to.have.deep.property("logger.name", "mosca"); - }); - - if (s.logger) { - s.logger.streams.pop(); - } - }); - it("should add an user to an authorization file", function(done) { args.push("adduser"); args.push("myuser"); diff --git a/test/common.js b/test/common.js index 1c529ab..91c95d2 100644 --- a/test/common.js +++ b/test/common.js @@ -45,13 +45,6 @@ global.rabbitSettings = function() { }; }; -var bunyan = require("bunyan"); - -global.globalLogger = bunyan.createLogger({ - name: "moscaTests", - level: 60 -}); - var sinonChai = require("sinon-chai"); chai.use(sinonChai); diff --git a/test/persistence/abstract.js b/test/persistence/abstract.js index a0f6a45..d3100b9 100644 --- a/test/persistence/abstract.js +++ b/test/persistence/abstract.js @@ -1,6 +1,7 @@ "use strict"; var steed = require("steed"); +var pino = require("pino"); var EventEmitter = require("events").EventEmitter; module.exports = function(create, buildOpts) { @@ -297,7 +298,7 @@ module.exports = function(create, buildOpts) { }; var client = { - logger: globalLogger, + logger: pino({ level: "error" }), forward: function(topic, payload, options, pattern) { expect(topic).to.eql(packet1.topic); expect(payload).to.eql(packet1.payload); @@ -323,7 +324,7 @@ module.exports = function(create, buildOpts) { var client = { id: "my client id - 42", clean: false, - logger: globalLogger, + logger: pino({ level: "error" }), subscriptions: { hello: { qos: 1 @@ -337,7 +338,7 @@ module.exports = function(create, buildOpts) { var client = { id: "my client id - 42", clean: false, - logger: globalLogger, + logger: pino({ level: "error" }), subscriptions: { hello: { qos: 1 @@ -355,7 +356,7 @@ module.exports = function(create, buildOpts) { var client = { id: "my client id - 42", clean: false, - logger: globalLogger, + logger: pino({ level: "error" }), subscriptions: { hello: { qos: 1 @@ -376,7 +377,7 @@ module.exports = function(create, buildOpts) { var client = { id: "my client id - 42", clean: true, - logger: globalLogger, + logger: pino({ level: "error" }), subscriptions: { hello: { qos: 1 @@ -397,7 +398,7 @@ module.exports = function(create, buildOpts) { var instance = this.instance; var client = { id: "my client id - 42", - logger: globalLogger, + logger: pino({ level: "error" }), subscriptions: { hello: { qos: 1 @@ -420,7 +421,7 @@ module.exports = function(create, buildOpts) { var client = { id: "my client id - 42", clean: true, - logger: globalLogger, + logger: pino({ level: "error" }), subscriptions: { hello: { qos: 1 @@ -439,7 +440,7 @@ module.exports = function(create, buildOpts) { var client = { id: "my client id - 42", clean: false, - logger: globalLogger, + logger: pino({ level: "error" }), subscriptions: { hello: { qos: 1 @@ -461,7 +462,7 @@ module.exports = function(create, buildOpts) { var client = { id: "my client id - 42", clean: false, - logger: globalLogger, + logger: pino({ level: "error" }), subscriptions: { hello: { qos: 1 @@ -488,7 +489,7 @@ module.exports = function(create, buildOpts) { var client = { id: "my client id - 42", clean: false, - logger: globalLogger, + logger: pino({ level: "error" }), subscriptions: { hello: { qos: 1 @@ -516,7 +517,7 @@ module.exports = function(create, buildOpts) { var client = { id: "my client id - 42", clean: false, - logger: globalLogger, + logger: pino({ level: "error" }), subscriptions: { hello: { qos: 1 @@ -539,7 +540,7 @@ module.exports = function(create, buildOpts) { var client = { id: "my client id - 42", clean: false, - logger: globalLogger, + logger: pino({ level: "error" }), subscriptions: { hello: { qos: 1 @@ -562,7 +563,7 @@ module.exports = function(create, buildOpts) { var client = { id: "my client id - 42", clean: false, - logger: globalLogger, + logger: pino({ level: "error" }), subscriptions: { hello: { qos: 0 @@ -583,7 +584,7 @@ module.exports = function(create, buildOpts) { var client = { id: "my client id - 42", clean: false, - logger: globalLogger, + logger: pino({ level: "error" }), subscriptions: { hello: { qos: 1 @@ -607,7 +608,7 @@ module.exports = function(create, buildOpts) { }); it("should not stream any offline packet", function(done) { - // ensure persistence engine call 'done' + // ensure persistence engine call "done" this.instance.streamOfflinePackets(client, function(err, packet) { done(new Error("this should never be called")); }, done); @@ -680,7 +681,7 @@ module.exports = function(create, buildOpts) { var client = { id: "my client id - 42", clean: false, - logger: globalLogger, + logger: pino({ level: "error" }), subscriptions: { hello: { qos: 1 @@ -705,7 +706,7 @@ module.exports = function(create, buildOpts) { var client = { id: "my client id - 42", clean: false, - logger: globalLogger, + logger: pino({ level: "error" }), subscriptions: { hello: 1 } @@ -728,7 +729,7 @@ module.exports = function(create, buildOpts) { var client = { id: "my client id - 42", clean: false, - logger: globalLogger, + logger: pino({ level: "error" }), subscriptions: { hello: 1 } @@ -752,7 +753,7 @@ module.exports = function(create, buildOpts) { var client = { id: "my client id - 42", clean: false, - logger: globalLogger, + logger: pino({ level: "error" }), subscriptions: { hello: { qos: 1 @@ -808,7 +809,7 @@ module.exports = function(create, buildOpts) { var client = { id: "my client id", clean: false, - logger: globalLogger, + logger: pino({ level: "error" }), subscriptions: { hello: { qos: 1 @@ -858,7 +859,7 @@ module.exports = function(create, buildOpts) { var client = { id: "my client id - 42", clean: false, - logger: globalLogger, + logger: pino({ level: "error" }), subscriptions: { "hello/#": { qos: 1 @@ -898,7 +899,7 @@ module.exports = function(create, buildOpts) { var client = { id: "my client id - 42", clean: false, - logger: globalLogger, + logger: pino({ level: "error" }), subscriptions: { hello: { qos: 1 diff --git a/test/persistence/levelup_spec.js b/test/persistence/levelup_spec.js index 32a4a36..781da4e 100644 --- a/test/persistence/levelup_spec.js +++ b/test/persistence/levelup_spec.js @@ -1,6 +1,7 @@ "use strict"; var abstract = require("./abstract"); +var pino = require("pino"); var LevelUp = require("../../").persistence.LevelUp; var steed = require("steed"); var tmpdir = require("osenv").tmpdir(); @@ -33,7 +34,7 @@ describe("mosca.persistence.LevelUp", function() { var client = { id: "my client id - 42", clean: false, - logger: globalLogger, + logger: pino({ level: "error" }), subscriptions: { "hello/#": { qos: 1 diff --git a/test/server.js b/test/server.js index 68da2e4..9c25681 100644 --- a/test/server.js +++ b/test/server.js @@ -13,8 +13,7 @@ var moscaSettings = function() { factory: mosca.persistence.Memory }, logger: { - childOf: globalLogger, - level: 60 + level: "error" } }; }; diff --git a/test/server_mongo.js b/test/server_mongo.js index a9ebc66..4525e75 100644 --- a/test/server_mongo.js +++ b/test/server_mongo.js @@ -35,8 +35,7 @@ describe("mosca.Server with mongo persistence", function() { publishNewClient: false, publishClientDisconnect: false, logger: { - childOf: globalLogger, - level: 60 + level: "error" }, backend : { type: "mongo" diff --git a/test/server_redis.js b/test/server_redis.js index 517e112..70cc31e 100644 --- a/test/server_redis.js +++ b/test/server_redis.js @@ -22,8 +22,7 @@ describe("mosca.Server with redis persistence", function() { stats: false, publishNewClient: false, logger: { - childOf: globalLogger, - level: 60 + level: "error" }, backend : { type: "redis" diff --git a/test/server_secure.js b/test/server_secure.js index 93b69d2..eeb967c 100644 --- a/test/server_secure.js +++ b/test/server_secure.js @@ -9,8 +9,7 @@ var moscaSettings = function() { var settings = { stats: false, logger: { - childOf: globalLogger.child({ port: port }), - level: 60 + level: "error" }, persistence: { factory: mosca.persistence.Memory diff --git a/test/server_websocket.js b/test/server_websocket.js index db8d378..09c5ade 100644 --- a/test/server_websocket.js +++ b/test/server_websocket.js @@ -8,8 +8,7 @@ var moscaSettings = function() { var settings = { stats: false, logger: { - childOf: globalLogger, - level: 60 + level: "error" }, persistence: { factory: mosca.persistence.Memory diff --git a/test/server_websocket_secure.js b/test/server_websocket_secure.js index 4c7b317..751adb7 100644 --- a/test/server_websocket_secure.js +++ b/test/server_websocket_secure.js @@ -10,8 +10,7 @@ var moscaSettings = function() { var settings = { stats: false, logger: { - childOf: globalLogger, - level: 60 + level: "error" }, persistence: { factory: mosca.persistence.Memory From a0ecc0e35fb1e2fc8f587aff4028a53c11bfe078 Mon Sep 17 00:00:00 2001 From: Matteo Collina Date: Mon, 23 May 2016 22:39:06 +0200 Subject: [PATCH 22/27] bumped qlobber and moment deps --- package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 55e980d..a57da63 100644 --- a/package.json +++ b/package.json @@ -82,13 +82,14 @@ "lru-cache": "~4.0.0", "memdown": "~1.1.1", "minimatch": "~3.0.0", - "moment": "~2.12.0", + "moment": "~2.13.0", "moving-average": "0.1.1", "mqtt": "^1.6.3", "mqtt-connection": "^2.1.1", "msgpack5": "^3.3.0", "pbkdf2-password": "^1.1.0", - "qlobber": "~0.6.0", + "pino": "^2.4.2", + "qlobber": "~0.7.0", "retimer": "^1.0.1", "shortid": "^2.2.4", "st": "~1.1.0", @@ -98,7 +99,6 @@ }, "optionalDependencies": { "leveldown": "~1.4.3", - "zmq": "~2.14.0", "amqp": "~0.2.4", "ioredis": "^1.15.1", "hiredis": "^0.4.1", From 631f5d6caaaade328a73e9d03780d1df4fd72e8a Mon Sep 17 00:00:00 2001 From: Behrad Date: Sun, 29 May 2016 12:35:46 +0430 Subject: [PATCH 23/27] replace KEYS with SCAN --- lib/persistence/redis.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/persistence/redis.js b/lib/persistence/redis.js index 677c6b6..2bef0e6 100644 --- a/lib/persistence/redis.js +++ b/lib/persistence/redis.js @@ -126,10 +126,16 @@ function RedisPersistence(options, callback) { if (that._explicitlyClosed()) { return; } - that._client.keys("client:sub:*", function(err, keys) { - if (err) { - return callback && callback(err, that); + var subsStream = that._client.scanStream({ + match: "client:sub:*" + }); + var keys = []; + subsStream.on('data', function(moreKeys){ + for( var i=0; i Date: Wed, 1 Jun 2016 00:16:20 +0430 Subject: [PATCH 24/27] update Qlobber version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 019e1ef..0191b07 100644 --- a/package.json +++ b/package.json @@ -91,7 +91,7 @@ "mqtt-connection": "^2.1.1", "msgpack5": "^3.3.0", "pbkdf2-password": "^1.1.0", - "qlobber": "~0.6.0", + "qlobber": "^0.7.0", "retimer": "^1.0.1", "shortid": "^2.2.4", "st": "~1.1.0", From 25e158fcdf43a0adc46e24a6dbe9e495eb890bfa Mon Sep 17 00:00:00 2001 From: Behrad Date: Wed, 1 Jun 2016 01:41:24 +0430 Subject: [PATCH 25/27] use pipeline to improve storeOfflinePacket performance for large matching list --- lib/persistence/redis.js | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/persistence/redis.js b/lib/persistence/redis.js index 2bef0e6..b4709c6 100644 --- a/lib/persistence/redis.js +++ b/lib/persistence/redis.js @@ -38,7 +38,7 @@ var defaults = { subscriptions: 60 * 60 * 1000, // TTL for packets is 1 hour - packets: 60 * 60 * 1000, + packets: 60 * 60 * 1000 }, storeMessagesQos0: false }; @@ -321,7 +321,6 @@ RedisPersistence.prototype.lookupSubscriptions = function(client, cb) { var subscriptions; var multi = this._client.multi(); - var that = this; multi.get(key); @@ -340,12 +339,15 @@ RedisPersistence.prototype.storeOfflinePacket = function(packet, done) { var that = this; var matches = this._subMatcher.match(packet.topic); + var pipeline = this._client.pipeline(); async.each(matches, function(client, cb) { - that._storePacket(client, packet, cb); - }, done); + that._storePacket(client, packet, pipeline, cb); + }, function(){ + pipeline.exec(done); + }); }; -RedisPersistence.prototype._storePacket = function(client, packet, cb) { +RedisPersistence.prototype._storePacket = function(client, packet, pipeline, cb) { if (this._explicitlyClosed()) { return cb && cb(new Error('Explicitly closed')); } @@ -353,12 +355,14 @@ RedisPersistence.prototype._storePacket = function(client, packet, cb) { var packetKey = "packets:" + client + ":" + packet.messageId, listKey = "packets:" + client; - this._client.multi() + pipeline.multi() .set(packetKey, JSON.stringify(packet)) .pexpire(packetKey, this._packetKeyTTL) .rpush(listKey, packetKey) .pexpire(listKey, this._listKeyTTL) - .exec(cb); + .exec(); + + cb(); }; var keyRegexp = /^([^:]+):(.+):([^:]+)$/; From 43a9874d37dd7fbed423a410223e8c1eaa146b05 Mon Sep 17 00:00:00 2001 From: Rob Fuller Date: Wed, 15 Jun 2016 15:58:08 +0100 Subject: [PATCH 26/27] added kafka example --- examples/kafka/Dockerfile | 25 +++++++ examples/kafka/README.md | 56 ++++++++++++++ examples/kafka/auth.json | 21 ++++++ examples/kafka/index.html | 98 ++++++++++++++++++++++++ examples/kafka/server.js | 153 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 353 insertions(+) create mode 100644 examples/kafka/Dockerfile create mode 100644 examples/kafka/README.md create mode 100644 examples/kafka/auth.json create mode 100644 examples/kafka/index.html create mode 100644 examples/kafka/server.js diff --git a/examples/kafka/Dockerfile b/examples/kafka/Dockerfile new file mode 100644 index 0000000..fdc0d48 --- /dev/null +++ b/examples/kafka/Dockerfile @@ -0,0 +1,25 @@ +FROM mhart/alpine-node:4 +MAINTAINER Robert Fuller + +RUN apk update && \ + apk add make gcc g++ python git + +RUN mkdir -p /usr/src/app +RUN mkdir -p /app/db + +WORKDIR /usr/src/app/ + +COPY ./ /usr/src/app/ + +RUN npm install --unsafe-perm --production +RUN npm install -g browserify uglify-js +RUN browserify -r mqtt -s mqtt | uglifyjs --screw-ie8 > public/mqtt.js + +COPY examples/kafka/server.js lib/mosca_kafka_server.js +COPY examples/kafka/auth.json auth.json +COPY examples/kafka/index.html public/ + +EXPOSE 80 +EXPOSE 1883 + +ENTRYPOINT ["node","lib/mosca_kafka_server.js"] diff --git a/examples/kafka/README.md b/examples/kafka/README.md new file mode 100644 index 0000000..afdd9c6 --- /dev/null +++ b/examples/kafka/README.md @@ -0,0 +1,56 @@ +Kafka MQTT Bridge Example +========================= + +Here is a Mosca example to expose an MQTT interface to [kafka](//kafka.apache.org). Try it right now in docker! + +Quickstart +---------- + +1. To try out the example with no changes, start a kafka server: + + docker run -d --env ADVERTISED_HOST=kafka01 --hostname=kafka01 --env ADVERTISED_PORT=9092 --name=kafka01 spotify/kafka + +2. Once the kafka is running, use the console producer to create some expected topics. + + for topic in spiddal-ctd \ + spiddal-fluorometer \ + airmar-rinville-1 \ + ais-rinville-1-geojson \ + spiddal-hydrophone + do docker exec -i -t kafka01 /bin/bash -c \ + "date | /opt/kafka_*/bin/kafka-console-producer.sh \ + --broker-list kafka01:9092 --topic $topic" + done + +3. Now start the kakfka mqtt bridge, linked to the kafka instance. + + docker run -d -p 2298:80 --link kafka01:kafka01 --link kafka01:kafka02 --link kafka01:kafka03 fullergalway/kafkamqtt + +4. Open up your browser and go to http://server:2298 for example [http://localhost:2298](http://localhost:2298) + +5. Finally, publish some data to your kafka topics by repeating step 2 above, and watch the data appear in your browser. + +Building +-------- + +Before building, you might like to modify [auth.json](auth.json) and [index.html](index.html) to reference your own topics. + + docker build -f examples/kafka/Dockerfile -t kafkamqtt . + + +Running +-------- + +To run the mosca mqtt server connected to your own kafka, provide the ip addresses when launching your docker container. (You'll need to add all three hosts (kafka01,kafka02,kafka03); repeat the ip address if you have fewer than three nodes in your cluster). + + docker run -d --name=kafkamqtt -p 2298:80 --add-host="kafka01:172.17.1.86" --add-host="kafka02:172.17.1.87" --add-host="kafka03:172.17.1.88" kafkamqtt + + +Credits +------- + +* [Matteo Collina](//twitter.com/matteocollina) +* [Robert Fuller](//github.com/fullergalway) +* [Adam Leadbetter](//twitter.com/adamleadbetter) +* [Damian Smyth](//ie.linkedin.com/in/damian-smyth-4b85563) +* [Eoin O'Grady](//ie.linkedin.com/in/eoin-o-grady-6177b) diff --git a/examples/kafka/auth.json b/examples/kafka/auth.json new file mode 100644 index 0000000..2e4b5a1 --- /dev/null +++ b/examples/kafka/auth.json @@ -0,0 +1,21 @@ +{ + "_README": "To generate a password for this authfile do: echo -n 'usernamePassword' | md5sum", + "*": { + "comment": "These topics available for all users, whether authorized or not", + "password": "", + "subscribe": [ + "airmar-rinville-1", + "spiddal-ctd", + "spiddal-fluorometer", + "spiddal-hydrophone", + "spiddal-adcp" ], + "publish": [] + }, + "fred": { + "password": "cc8355a2cc9fc3e0ed575db2f49268d1", + "subscribe": [ + "ais-rinville-1-geojson" + ], + "publish": [] + } +} diff --git a/examples/kafka/index.html b/examples/kafka/index.html new file mode 100644 index 0000000..424f288 --- /dev/null +++ b/examples/kafka/index.html @@ -0,0 +1,98 @@ + + + + + + Marine Institute MQTT live demo + + +
+

CTD

+ 

+   
Fluorometer

+   

+   
Weather

+   

+   
AIS

+   

+   
Hydrophone

+   

+   
+    
+  
+
diff --git a/examples/kafka/server.js b/examples/kafka/server.js
new file mode 100644
index 0000000..b538ac8
--- /dev/null
+++ b/examples/kafka/server.js
@@ -0,0 +1,153 @@
+//var mosca = require('mosca');
+var persistence = require("./persistence");
+var Server = require("./server");
+var crypto = require('crypto');
+var fs = require('fs');
+
+var backend = {
+    type: "kafka",
+    json: false,
+    connectionString: "kafka01:2181,kafka02:2181:kafka03:2181",
+    clientId: "mosca",
+    groupId: "mosca",
+    defaultEncoding: "utf8",
+    encodings:{
+          "spiddal-adcp": "buffer"
+    }
+  };
+
+//var SECURE_KEY = __dirname + '/../../test/secure/tls-key.pem';
+//var SECURE_CERT = __dirname + '/../../test/secure/tls-cert.pem';
+
+var moscaSettings = {
+    interfaces: [
+        { type: "mqtt", port: 1883 },
+        { type: "http", port: 80, bundle: true, static: "./public" }
+/*
+        { type: "mqtts", port: 8883, credentials: { keyPath: SECURE_KEY, certPath: SECURE_CERT } },
+        { type: "https", port: 3001, bundle: true, credentials: { keyPath: SECURE_KEY, certPath: SECURE_CERT } }
+*/
+    ],
+    id: "mosca",
+
+     /*
+     * avoid publishing to $SYS topics because
+     * it violates kafka topic naming convention
+     */
+    stats: false,
+    publishNewClient: false,
+    publishClientDisconnect: false,
+    publishSubscriptions: false,
+
+    logger: { name: 'MoscaServer', level: 'debug' },
+
+    persistence: { factory: persistence.LevelUp, path: "/app/db" },
+
+    backend: backend,
+};
+
+/*
+ * default authorization, if no auth.json file is present.
+ */
+var auth = {
+  "*":{ // this asterisk requires no username. Remove it only allowing authorized users.
+    password:"",
+    subscribe: [ 
+              "a_public_topic", "another_public_topic"
+            ],
+    publish: []
+   }
+};
+
+/*
+ * Read the auth.json file.
+ * TODO: auto reload the file when it changes,
+ * but maybe some complexities due to users already connected
+ * if permissions were to change.
+ */
+fs.readFile('auth.json', 'utf8', function (err, data) {
+    if (!err){
+       auth = JSON.parse(data);
+    }
+});
+
+/*
+ * user is authenticated against the auth.json file, which
+ * contains signature of password salted with username.
+ */
+var authenticate = function (client, username, password, callback) {
+    var authorized = false;
+    if(username === undefined && auth["*"] !== undefined){
+       authorized = true;
+    }else if(username !== undefined && password !== undefined){
+       var pw = 
+          crypto.createHash('md5').update(username).update(password.toString()).digest("hex");
+       if(auth[username] !== undefined && auth[username].password == pw){
+           client.user = username;
+           authorized = true;
+       }
+    }
+    callback(null,authorized);
+}
+/*
+ * publish and subscribe permissions defined in auth.json
+ */
+var authorizeSubscribe = function (client, topic, callback) {
+    var answer = false;
+    if(auth["*"] !== undefined && auth["*"].subscribe.indexOf(topic)>=0){
+       answer = true;
+    }else if(client.user !== undefined && auth[client.user].subscribe.indexOf(topic)>=0){
+          answer = true;
+    }
+    callback(null, answer);
+}
+
+var authorizePublish = function (client, topic, callback) {
+    var answer = false;
+    if(auth["*"] !== undefined && auth["*"].publish.indexOf(topic)>=0){
+       answer = true;
+    }else if(client.user !== undefined && auth[client.user].publish.indexOf(topic)>=0){
+          answer = true;
+    }
+    callback(null, answer);
+}
+
+var server = new Server(moscaSettings);
+
+server.on('ready', setup);
+
+function setup() {
+    server.authenticate = authenticate;
+    server.authorizePublish = authorizePublish;
+    server.authorizeSubscribe = authorizeSubscribe;
+    
+    console.log('Mosca server is up and running.');
+}
+
+server.on("error", function (err) {
+    console.log(err);
+});
+
+server.on('clientConnected', function (client) {
+    console.log('Client Connected \t:= ', client.id);
+});
+
+server.on('published', function (packet, client) {
+    console.log("Published :=", packet);
+});
+
+server.on('subscribed', function (topic, client) {
+    console.log("Subscribed :=", topic);
+});
+
+server.on('unsubscribed', function (topic, client) {
+    console.log('unsubscribed := ', topic);
+});
+
+server.on('clientDisconnecting', function (client) {
+    console.log('clientDisconnecting := ', client.id);
+});
+
+server.on('clientDisconnected', function (client) {
+    console.log('Client Disconnected     := ', client.id);
+});

From c6daf181a42e5062ecc93ea09974b8b976944f0e Mon Sep 17 00:00:00 2001
From: Behrad 
Date: Thu, 16 Jun 2016 19:09:56 +0430
Subject: [PATCH 27/27] Pass client subscriptions VIA publish so that each
 mosca instance doesn't fetch it

---
 lib/persistence/redis.js | 41 +++++++++++++++++++++++-----------------
 1 file changed, 24 insertions(+), 17 deletions(-)

diff --git a/lib/persistence/redis.js b/lib/persistence/redis.js
index 20838b9..b0905fd 100644
--- a/lib/persistence/redis.js
+++ b/lib/persistence/redis.js
@@ -84,7 +84,7 @@ function RedisPersistence(options, callback) {
   this._closing = false;
   this._closed = false;
 
-  var newSub = function(key, unsubs, retried, cb) {
+  var fetchAndUpdateLocalSub = function(key, unsubs, retried, cb) {
     that._client.get(key, function(err, result) {
       if (err) {
         if (cb) {
@@ -94,26 +94,15 @@ function RedisPersistence(options, callback) {
         }
       }
 
-      var xs = key.split(":");
-      var id = key.substr(xs[0].length + xs[1].length + 2);
       var subs = JSON.parse(result);
-
       if (!result || typeof subs !== 'object') {
         if (!retried) {
-          setTimeout(newSub.bind(null, key, unsubs, true, cb), 500);
+          setTimeout(fetchAndUpdateLocalSub.bind(null, key, unsubs, true, cb), 500);
         }
         return;
       }
 
-      Object.keys(subs).forEach(function(sub) {
-        that._subMatcher.add(sub, id);
-      });
-
-      if( unsubs ) {
-        unsubs.forEach(function(sub) {
-          that._subMatcher.remove(sub, id);
-        });
-      }
+      updateLocalSub(key, subs, unsubs);
 
       if (cb) {
         cb();
@@ -121,6 +110,21 @@ function RedisPersistence(options, callback) {
     });
   };
 
+  var updateLocalSub = function(key, subs, unsubs) {
+    var xs = key.split(":");
+    var id = key.substr(xs[0].length + xs[1].length + 2);
+
+    Object.keys(subs).forEach(function(sub) {
+      that._subMatcher.add(sub, id);
+    });
+
+    if( unsubs ) {
+      unsubs.forEach(function(unsub) {
+        that._subMatcher.remove(unsub, id);
+      });
+    }
+  };
+
   var that = this;
 
   this._pubSubClient.subscribe(this.options.channel, function(){
@@ -138,7 +142,7 @@ function RedisPersistence(options, callback) {
     });
     subsStream.on('end', function(){
       steed.each(keys, function(k,next){
-        newSub(k,null,false,next);
+        fetchAndUpdateLocalSub(k,null,false,next);
       }, function(err) {
         if (callback) {
           callback(err, that);
@@ -153,7 +157,7 @@ function RedisPersistence(options, callback) {
     }
     var parsed = JSON.parse(message);
     if (parsed.process !== that._id) {
-      newSub(parsed.key, parsed.unsubs);
+      updateLocalSub(parsed.key, parsed.subs, parsed.unsubs);
     }
   });
 }
@@ -258,9 +262,10 @@ RedisPersistence.prototype.storeSubscriptions = function(client, cb) {
   });
 
   this._client.get(clientSubKey, function(err, currentSubs){
+    var unsubs;
     if( !err && currentSubs ) {
       currentSubs = JSON.parse(currentSubs);
-      var unsubs  = Object.keys(currentSubs).filter(function (topic) {
+      unsubs = Object.keys(currentSubs).filter(function (topic) {
         return !subscriptions[topic];
       });
       unsubs.forEach(function (topic) {
@@ -271,6 +276,8 @@ RedisPersistence.prototype.storeSubscriptions = function(client, cb) {
       .set(clientSubKey, JSON.stringify(subscriptions))
       .publish(that.options.channel, JSON.stringify({
         key: clientSubKey,
+        subs: subscriptions,
+        unsubs: unsubs,
         process: that._id
       }))
       .pexpire(clientSubKey, that.options.ttl.subscriptions);