diff --git a/circle.yml b/circle.yml new file mode 100644 index 0000000..bad2338 --- /dev/null +++ b/circle.yml @@ -0,0 +1,6 @@ +dependencies: + override: + - nvm install 4.2.2 + - nvm alias default 4.2.2 + - npm install -g npm@2.14.7 + - npm install diff --git a/configs/.env b/configs/.env index 7724b2a..560ada8 100644 --- a/configs/.env +++ b/configs/.env @@ -1,20 +1,25 @@ -PORT=4000 -REDIS_HOST_KEYS="dockerHosts:" BUILD_WEIGHT=10 CONTAINER_WEIGHT=1 +DOCKER_CERT_PATH=/etc/ssl/docker HISTORY_WEIGHT=10 +IMAGE_BUILDER=runnable/image-builder +IMAGE_BUILDER_LABEL=image-builder-container LOG=true -IMAGE_BUILDER="runnable/image-builder" -IMAGE_BUILDER_LABEL="image-builder-container" -RUNNABLE_REGISTRY="registry.runnable.com" -ROLLBAR_KEY="f4888bcd48be45cabd42627dfed87bba" -ROLLBAR_OPTIONS_BRANCH="master" -NEWRELIC_NAME='mavis' -NEWRELIC_LEVEL='error' +NEWRELIC_LEVEL=error +NEWRELIC_NAME=mavis +PORT=4000 +REDIS_HOST_KEYS=dockerHosts: +ROLLBAR_KEY=f4888bcd48be45cabd42627dfed87bba +ROLLBAR_OPTIONS_BRANCH=master +RUNNABLE_REGISTRY=registry.runnable.com # Monitor-dog configuration -MONITOR_PREFIX="mavis" +MONITOR_PREFIX=mavis MONITOR_INTERVAL=60000 # Logging configuration LOG_LEVEL=trace + +# ponos +WORKER_MAX_RETRY_DELAY=30000 +WORKER_MIN_RETRY_DELAY=10000 \ No newline at end of file diff --git a/configs/.env.test b/configs/.env.test index 4fa9185..551b2bb 100644 --- a/configs/.env.test +++ b/configs/.env.test @@ -1,7 +1,11 @@ -PORT=65213 -REDIS_PORT=6379 -REDIS_IPADDRESS=127.0.0.1 +CONSUL_HOST=consul.com +CONSUL_PORT=8500 +DOCKER_CERT_PATH=./test/fixtures/certs LOG_LEVEL=fatal +PORT=65213 RABBITMQ_HOSTNAME=localhost RABBITMQ_PASSWORD=guest -RABBITMQ_USERNAME=guest \ No newline at end of file +RABBITMQ_USERNAME=guest +REDIS_IPADDRESS=127.0.0.1 +REDIS_PORT=6379 +SWARM_CONTAINER_NAME=swarm diff --git a/lib/models/consul.js b/lib/models/consul.js new file mode 100644 index 0000000..1fd7ad9 --- /dev/null +++ b/lib/models/consul.js @@ -0,0 +1,50 @@ +/** + * Consul API requests + * @module lib/models/Consul + */ +'use strict'; +require('loadenv')('mavis:env'); + +var url = require('url'); +var ErrorCat = require('error-cat'); +var error = new ErrorCat(); + +var log = require('../logger').child({ module: 'consul' }); + +module.exports = Consul; + +/** + * class used to talk to Consul + */ +function Consul () {} + +/** + * singleton consul client + * @type {Object} + */ +Consul._client = require('consul')({ + host: process.env.CONSUL_HOST, + port: process.env.CONSUL_PORT +}); + +/** + * checks dock host key in consul to see if it exist. + * will cb with error if dock key still exist + * @param {String} dockerUrl docker host to check for format: http://10.0.0.1:4242 + * @param {Function} cb (err) + */ +Consul.ensureDockRemoved = function (dockerUrl, cb) { + var host = url.parse(dockerUrl).host; + var logData = { host: host }; + log.info(logData, 'Consul.prototype.ensureDockRemoved'); + Consul._client.kv.get('swarm/docker/swarm/nodes/' + host, function(err, result) { + if (err) { return cb(err); } + // if we have a result that means the key still exist, cb with error + if (result) { + return cb(error.create(412, 'dock still exist', result)); + } + + log.trace(logData, 'ensureDockRemoved dock as been removed'); + return cb(null); + }); +}; diff --git a/lib/models/docker.js b/lib/models/docker.js new file mode 100644 index 0000000..6dc3b9c --- /dev/null +++ b/lib/models/docker.js @@ -0,0 +1,74 @@ +/** + * Docker API requests + * @module lib/models/docker + */ +'use strict'; +require('loadenv')('mavis:env'); + +var Dockerode = require('dockerode'); +var put = require('101/put'); +var fs = require('fs'); +var join = require('path').join; +var url = require('url'); + +var log = require('../logger').child({ module: 'docker' }); + +var certs = {}; + +module.exports = Docker; + +/** + * class used to talk to docker + * @param {string} dockerUrl format: http://hostname:hostport + */ +function Docker (dockerUrl) { + var parsedHost = url.parse(dockerUrl); + + this._client = new Dockerode(put({ + host: parsedHost.hostname, + port: parsedHost.port + }, certs)); + + this._logData = { + dockerUrl: dockerUrl + }; + log.trace(this._logData, 'Docker constructor'); +} + +/** + * loads certs for docker. does not throw if failed, just logs + * sync function as this should only happen once on startup + */ +Docker.loadCerts = function () { + // try/catch is a better pattern for this, since checking to see if it exists + // and then reading files can lead to race conditions (unlikely, but still) + try { + var certPath = process.env.DOCKER_CERT_PATH; + certs.ca = fs.readFileSync(join(certPath, '/ca.pem')); + certs.cert = fs.readFileSync(join(certPath, '/cert.pem')); + certs.key = fs.readFileSync(join(certPath, '/key.pem')); + log.info('Docker.loadCerts docker certificates loaded'); + } catch (err) { + log.warn({ err: err }, 'Docker.loadCerts cannot load certificates for docker'); + throw err; + } +}; + +/** + * stop swarm docker container + * @param {Function} cb (err) + */ +Docker.prototype.killSwarmContainer = function (cb) { + var self = this; + log.info(self._logData, 'Docker.prototype.killSwarmContainer'); + + self._client.getContainer(process.env.SWARM_CONTAINER_NAME).kill(function (err) { + if (err) { + log.error(put({ err: err }, self._logData), 'killSwarmContainer error killing container'); + return cb(err); + } + + log.trace(self._logData, 'killSwarmContainer killing container success'); + cb(null); + }); +}; diff --git a/lib/models/events.js b/lib/models/events.js index 0795602..6f89fd0 100644 --- a/lib/models/events.js +++ b/lib/models/events.js @@ -3,21 +3,20 @@ require('loadenv')('mavis:env'); var url = require('url'); var keypather = require('keypather')(); - -var dockData = require('../models/dockData.js'); -var log = require('../logger').child({ module: 'events:docker' }); - var TaskFatalError = require('ponos').TaskFatalError; var TaskError = require('ponos').TaskError; -var rabbitMQ = require('../rabbitmq.js'); +var dockData = require('../models/dockData.js'); +var Consul = require('../models/consul.js'); +var Docker = require('../models/docker.js'); +var rabbitMQ = require('../rabbitmq.js'); +var log = require('../logger').child({ module: 'events:docker' }); /** * Module used to handle runnable events */ var Events = module.exports = {}; - /** * Handles docker `die` events. * updates container build counts @@ -84,12 +83,45 @@ Events.handleUnhealthy = function (data, cb) { } dockData.deleteHost(data.host, function (err) { if (err) { - var taskErr = new TaskError('Failed to delete host', err); + var taskErr = new TaskError('handleUnhealthy', 'Failed to delete host', err); + return cb(taskErr); + } + var docker = new Docker(data.host); + docker.killSwarmContainer(function (err) { + if (err) { + var taskErr = new TaskError('handleUnhealthy', 'Failed to kill swarm container', err); + return cb(taskErr); + } + + rabbitMQ.getPublisher().publish('cluster-instance-provision', { + githubId: data.githubId + }); + rabbitMQ.getPublisher().publish('dock.wait-for-removal', { + dockerUrl: data.host + }); + cb(); + }); + }); +}; + +/** + * waits for dock to be removed form consul + * then publish dock.removed event + * @param {Object} data job data + * @param {String} data.dockerUrl url to check for format: http://10.0.0.1:4242 + * @param {Function} cb (err) + */ +Events.handleEnsureDockRemoved = function (data, cb) { + log.info({ data: data }, 'Events.handleEnsureDockRemoved'); + Consul.ensureDockRemoved(data.dockerUrl, function (err) { + if (err) { + var taskErr = new TaskError('dock.wait-for-removal', 'dock still exists', err); return cb(taskErr); } - rabbitMQ.getPublisher().publish('on-dock-removed', data); - rabbitMQ.getPublisher().publish('cluster-instance-provision', { - githubId: data.githubId + + log.trace({ data: data }, 'handleEnsureDockRemoved publishing dock.removed'); + rabbitMQ.getPublisher().publish('dock.removed', { + host: data.dockerUrl }); cb(); }); diff --git a/lib/models/worker-server.js b/lib/models/worker-server.js index 06e5be4..72c43eb 100644 --- a/lib/models/worker-server.js +++ b/lib/models/worker-server.js @@ -27,6 +27,7 @@ WorkerServer.listen = function (cb) { var tasks = { 'on-dock-unhealthy': require('../workers/on-dock-unhealthy.js'), + 'dock.wait-for-removal': require('../workers/dock.wait-for-removal.js'), 'container.life-cycle.died': require('../workers/container.life-cycle.died.js'), 'docker.events-stream.connected': require('../workers/docker.events-stream.connected.js'), 'docker.events-stream.disconnected': require('../workers/docker.events-stream.disconnected.js'), diff --git a/lib/rabbitmq.js b/lib/rabbitmq.js index cbf76d1..3b2f768 100644 --- a/lib/rabbitmq.js +++ b/lib/rabbitmq.js @@ -39,7 +39,8 @@ RabbitMQ.prototype.create = function (cb) { this._subscriber = new Hermes(put({ queues: [ - 'on-dock-unhealthy' + 'on-dock-unhealthy', + 'dock.wait-for-removal' ], subscribedEvents: [ 'container.life-cycle.died', @@ -51,9 +52,12 @@ RabbitMQ.prototype.create = function (cb) { this._publisher = new Hermes(put({ queues: [ - 'on-dock-removed', + 'dock.wait-for-removal', 'cluster-instance-provision' ], + publishedEvents: [ + 'dock.removed' + ] }, opts)) // connect publisher only since ponos is handling subscriber .connect(cb) diff --git a/lib/server.js b/lib/server.js index 862a8f5..28065bf 100644 --- a/lib/server.js +++ b/lib/server.js @@ -14,7 +14,7 @@ if (process.env.NEWRELIC_KEY) { require('newrelic'); } var Redis = require('./models/redis.js'); var rabbitMQ = require('./rabbitmq.js'); var WorkerServer = require('./models/worker-server.js'); - +var Docker = require('./models/docker.js'); module.exports = Server; @@ -36,6 +36,7 @@ Server.prototype.start = function (cb) { monitor.startSocketsMonitor(); docksMonitor.start(); Redis.connect(); + Docker.loadCerts(); this.server = app.listen(process.env.PORT, function(err) { if (err) { log.fatal({ err: err }, 'Error starting server. Exiting'); diff --git a/lib/workers/dock.wait-for-removal.js b/lib/workers/dock.wait-for-removal.js new file mode 100644 index 0000000..a90077e --- /dev/null +++ b/lib/workers/dock.wait-for-removal.js @@ -0,0 +1,28 @@ +/** + * Handles `dock.wait-for-removal` event + * @module lib/workers/dock.wait-for-removal + */ +'use strict'; + +var isString = require('101/is-string'); +var Promise = require('bluebird'); +var TaskFatalError = require('ponos').TaskFatalError; + +var Events = Promise.promisifyAll(require('../models/events.js')); +var log = require('../logger').child({ module: 'workers' }); + +module.exports = function (job) { + return Promise.resolve() + .then(function validateArguments () { + if (!isString(job.dockerUrl)) { + throw new TaskFatalError('missing dockerUrl'); + } + }) + .then(function () { + return Events.handleEnsureDockRemovedAsync(job); + }) + .catch(function (err) { + log.error({ err: err }, 'dock.wait-for-removal error'); + throw err; + }); +}; diff --git a/package.json b/package.json index f1dc80f..9272345 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,8 @@ "body-parser": "^1.13.3", "boom": "^2.6.1", "bunyan": "^1.4.0", + "consul": "^0.19.0", + "dockerode": "^2.2.6", "error-cat": "^1.4.0", "express": "^4.13.3", "express-bunyan-logger": "^1.1.1", @@ -38,6 +40,7 @@ "loadenv": "^1.1.0", "monitor-dog": "^1.4.1", "newrelic": "^1.22.0", + "nock": "^3.3.2", "node-dogstatsd": "0.0.6", "ponos": "^1.1.1", "redis": "^0.12.1", diff --git a/test/fixtures/certs/ca.pem b/test/fixtures/certs/ca.pem new file mode 100644 index 0000000..2aefada --- /dev/null +++ b/test/fixtures/certs/ca.pem @@ -0,0 +1 @@ +ca \ No newline at end of file diff --git a/test/fixtures/certs/cert.pem b/test/fixtures/certs/cert.pem new file mode 100644 index 0000000..8c22b35 --- /dev/null +++ b/test/fixtures/certs/cert.pem @@ -0,0 +1 @@ +cert \ No newline at end of file diff --git a/test/fixtures/certs/key.pem b/test/fixtures/certs/key.pem new file mode 100644 index 0000000..1aadabe --- /dev/null +++ b/test/fixtures/certs/key.pem @@ -0,0 +1 @@ +key \ No newline at end of file diff --git a/test/functional/workers/on-dock-unhealthy.js b/test/functional/workers/on-dock-unhealthy.js index 2964fad..213bdd8 100644 --- a/test/functional/workers/on-dock-unhealthy.js +++ b/test/functional/workers/on-dock-unhealthy.js @@ -2,88 +2,125 @@ require('loadenv')(); -var put = require('101/put'); var Lab = require('lab'); var lab = exports.lab = Lab.script(); var describe = lab.describe; var it = lab.it; -var after = lab.after; -var before = lab.before; var beforeEach = lab.beforeEach; +var afterEach = lab.afterEach; var Code = require('code'); var expect = Code.expect; var createCount = require('callback-count'); +var nock = require('nock'); +var redis = require('redis'); var Hermes = require('runnable-hermes'); var dockData = require('../../../lib/models/dockData.js'); var Server = require('../../../lib/server.js'); -var rabbitMQ = require('../../../lib/rabbitmq.js'); -var WorkerServer = require('../../../lib/models/worker-server.js'); -var redis = require('../../../lib/models/redis.js'); -describe('on-dock-unhealthy functional test', function () { +var server = new Server(); + +var publishedEvents = [ + 'container.life-cycle.died', + 'docker.events-stream.connected', + 'docker.events-stream.disconnected' +]; + +var subscribedEvents = [ + 'dock.removed' +]; + +var queues = [ + 'weave.start', + 'on-dock-unhealthy', + 'cluster-instance-provision' +]; + +var testPublisher = new Hermes({ + hostname: process.env.RABBITMQ_HOSTNAME, + password: process.env.RABBITMQ_PASSWORD, + port: process.env.RABBITMQ_PORT, + username: process.env.RABBITMQ_USERNAME, + publishedEvents: publishedEvents, + queues: queues, + name: 'testPublisher' +}); + +var testSubscriber = new Hermes({ + hostname: process.env.RABBITMQ_HOSTNAME, + password: process.env.RABBITMQ_PASSWORD, + port: process.env.RABBITMQ_PORT, + username: process.env.RABBITMQ_USERNAME, + subscribedEvents: subscribedEvents, + queues: queues, + name: 'testSubscriber' +}); + +var testRedis = redis.createClient( + process.env.REDIS_PORT, + process.env.REDIS_IPADDRESS); + + +describe('lib/workers/on-dock-unhealthy functional test', function () { var testHost = 'http://10.20.1.26:4242'; var testGihubId = 2194285; - var ctx = {}; - before(function (done) { - redis.connect(); - rabbitMQ.create(done); - }); - before(function (done) { - WorkerServer.listen(done); + beforeEach(function (done) { + // connect publisher so exchanges are generated before sever init + testPublisher.connect(done); }); - before(function (done) { - var opts = { - hostname: process.env.RABBITMQ_HOSTNAME, - password: process.env.RABBITMQ_PASSWORD, - port: process.env.RABBITMQ_PORT, - username: process.env.RABBITMQ_USERNAME, - name: 'mavis' - }; - ctx.rabbitClient = new Hermes(put({ - queues: [ - 'on-dock-unhealthy', - 'on-dock-removed', - 'cluster-instance-provision' - ], - }, opts)) - // connect publisher only since ponos is handling subscriber - .connect(done); + beforeEach(function (done) { + server.start(done); }); - after(function (done) { - ctx.rabbitClient.close(done); + beforeEach(function (done) { + // connect subscriber after server has started + testSubscriber.connect(done); }); lab.beforeEach(function (done) { - redis.client.flushall(done); + // nock docker call + nock(testHost) + .post('/containers/swarm/kill') + .reply(200); + + // nock consul call + nock('http://consul.com:8500') + .get('/v1/kv/swarm%2Fdocker%2Fswarm%2Fnodes%2F10.20.1.26%3A4242') + .reply(404); + + testRedis.flushall(done); }); beforeEach(function(done) { dockData.addHost(testHost, 'test,tags', done); }); - after(function (done) { - WorkerServer.stop(done); - }) - after(function (done) { - redis.disconnect(); - rabbitMQ.close(done); + afterEach(function (done) { + testSubscriber.close(done); + }); + + afterEach(function (done) { + testPublisher.close(done); + }); + + afterEach(function (done) { + server.stop(done); }); describe('on-docker-unhealthy event', function () { it('should remove host and publish two events', function (done) { var count = createCount(2, done); - ctx.rabbitClient.subscribe('cluster-instance-provision', function (data, cb) { + testSubscriber.subscribe('cluster-instance-provision', function (data, cb) { expect(data.githubId).to.equal(testGihubId); cb(); count.next(); }); - ctx.rabbitClient.subscribe('on-dock-removed', function (data, cb) { + + testSubscriber.subscribe('dock.removed', function (data, cb) { expect(data.host).to.equal(testHost); cb(); dockData.getAllDocks(function (err, data) { @@ -92,7 +129,7 @@ describe('on-dock-unhealthy functional test', function () { count.next(); }); }); - ctx.rabbitClient.publish('on-dock-unhealthy', { + testPublisher.publish('on-dock-unhealthy', { host: testHost, githubId: testGihubId }); diff --git a/test/unit/models/consul.js b/test/unit/models/consul.js new file mode 100644 index 0000000..7d6c18f --- /dev/null +++ b/test/unit/models/consul.js @@ -0,0 +1,72 @@ +'use strict'; + +require('loadenv')(); + +var Lab = require('lab'); +var lab = exports.lab = Lab.script(); +var describe = lab.describe; +var it = lab.it; +var afterEach = lab.afterEach; +var beforeEach = lab.beforeEach; +var Code = require('code'); +var expect = Code.expect; + +var sinon = require('sinon'); +var TaskError = require('ponos').TaskError; + +var Consul = require('../../../lib/models/consul.js'); + +describe('lib/models/consul unit test', function () { + describe('ensureDockRemoved', function () { + var dockerUrl = 'http://11.17.38.11:4242'; + + beforeEach(function (done) { + sinon.stub(Consul._client.kv, 'get'); + done(); + }); + + afterEach(function (done) { + Consul._client.kv.get.restore(); + done(); + }); + + it('should cb with error from Consul._client.kv.get', function (done) { + var error = new Error('starcraft'); + + Consul._client.kv.get.yieldsAsync(error); + + Consul.ensureDockRemoved(dockerUrl, function (err) { + expect(err).to.equal(error); + done(); + }); + }); + + it('should cb an err if result exists', function (done) { + // returning non falsey means the dock hasn't been removed + Consul._client.kv.get.yieldsAsync(null, {some: 'stuff'}); + + Consul.ensureDockRemoved(dockerUrl, function (err) { + expect(err).to.exist(); + done(); + }); + }); + + // result and error will both be null when the key does not exist in consul + it('should cb with no err and null if result is null', function (done) { + // returning null means the dock hasn't been removed + Consul._client.kv.get.yieldsAsync(null, null); + + Consul.ensureDockRemoved(dockerUrl, function (err) { + expect(err).to.not.exist(); + sinon.assert.calledOnce(Consul._client.kv.get); + + sinon.assert.calledWith( + Consul._client.kv.get, + 'swarm/docker/swarm/nodes/11.17.38.11:4242' + ); + + done(); + }); + }); + }); // end ensureDockRemoved +}); \ No newline at end of file diff --git a/test/unit/models/docker.js b/test/unit/models/docker.js new file mode 100644 index 0000000..3c1947c --- /dev/null +++ b/test/unit/models/docker.js @@ -0,0 +1,91 @@ +'use strict'; + +require('loadenv')(); + +var Lab = require('lab'); +var lab = exports.lab = Lab.script(); +var describe = lab.describe; +var it = lab.it; +var beforeEach = lab.beforeEach; +var Code = require('code'); +var expect = Code.expect; + +var Dockerode = require('dockerode'); +var sinon = require('sinon'); + +var Docker = require('../../../lib/models/docker.js'); + +describe('lib/models/docker unit test', function () { + describe('constructor', function () { + it('should create Docker', function (done) { + var docker = new Docker('http://10.0.0.1:4242'); + expect(docker._client).to.be.an.instanceof(Dockerode); + done(); + }); + }); // end constructor + + describe('staticMethods', function () { + describe('loadCerts', function () { + it('should throw if missing certs', function (done) { + process.env.DOCKER_CERT_PATH = 'fake/path'; + + expect(Docker.loadCerts).to.throw(); + done(); + }); + + it('should load certs', function (done) { + process.env.DOCKER_CERT_PATH = './test/fixtures/certs'; + + expect(Docker.loadCerts).to.not.throw(); + done(); + }); + }); // end loadCerts + }); // end staticMethods + + describe('prototype methods', function () { + var docker; + var dockerUrl = 'http://10.0.0.1:4242'; + + beforeEach(function (done) { + docker = new Docker(dockerUrl); + done(); + }); + + describe('killSwarmContainer', function () { + beforeEach(function (done) { + docker._client = { + getContainer: sinon.stub().returnsThis(), + kill: sinon.stub() + }; + done(); + }); + + it('should cb error if kill failed', function (done) { + var error = new Error('iceberg'); + docker._client.kill.yieldsAsync(error); + + docker.killSwarmContainer(function (err) { + expect(err).to.equal(error); + done(); + }); + }); + + it('should cb on success', function (done) { + docker._client.kill.yieldsAsync(); + + docker.killSwarmContainer(function (err) { + expect(err).to.not.exist(); + + sinon.assert.calledOnce(docker._client.getContainer); + sinon.assert.calledWith( + docker._client.getContainer, + 'swarm' + ); + sinon.assert.calledOnce(docker._client.kill); + + done(); + }); + }); + }); // end killSwarmContainer + }); // end prototype methods +}); \ No newline at end of file diff --git a/test/unit/models/events.js b/test/unit/models/events.js index 9960398..2f6d060 100644 --- a/test/unit/models/events.js +++ b/test/unit/models/events.js @@ -2,16 +2,24 @@ require('loadenv')('mavis:env'); -var async = require('async'); - var Lab = require('lab'); var lab = exports.lab = Lab.script(); +var Code = require('code'); +var expect = Code.expect; +var afterEach = lab.afterEach; +var beforeEach = lab.beforeEach; + var redis = require('../../../lib/models/redis.js'); var events = require('../../../lib/models/events.js'); var dockData = require('../../../lib/models/dockData.js'); +var RabbitMQ = require('../../../lib/rabbitmq.js'); +var Consul = require('../../../lib/models/consul.js'); + +var async = require('async'); +var sinon = require('sinon'); +var TaskError = require('ponos').TaskError; + var host = 'http://0.0.0.0:4242'; -var Code = require('code'); -var expect = Code.expect; function dataExpect1(data, numContainers, numBuilds, host) { @@ -32,7 +40,7 @@ function dataExpectNone (data) { expect(data.length).to.equal(0); } -lab.experiment('events.js unit test', function () { +lab.experiment('lib/models/events.js unit test', function () { lab.beforeEach(function (done) { redis.connect(); redis.client.flushall(done); @@ -558,4 +566,57 @@ lab.experiment('events.js unit test', function () { }); }); // handleDockDown }); // deamon + + lab.experiment('handleEnsureDockRemoved', function () { + var publishStub; + beforeEach(function (done) { + sinon.stub(Consul, 'ensureDockRemoved'); + publishStub = sinon.stub(); + sinon.stub(RabbitMQ, 'getPublisher').returns({ + publish: publishStub + }); + done(); + }); + + afterEach(function (done) { + Consul.ensureDockRemoved.restore(); + RabbitMQ.getPublisher.restore(); + done(); + }); + + lab.test('should cb err if ensureDockRemoved failed', function (done) { + Consul.ensureDockRemoved.yieldsAsync(new Error('dock not removed')); + + events.handleEnsureDockRemoved({}, function (err) { + expect(err).to.be.an.instanceOf(TaskError); + done(); + }); + }); + + lab.test('should publish dock.removed', function (done) { + var dockerUrl = 'http://10.0.102.2:4242'; + Consul.ensureDockRemoved.yieldsAsync(null); + + events.handleEnsureDockRemoved({ + dockerUrl: dockerUrl + }, function (err) { + expect(err).to.not.exist(); + + sinon.assert.calledOnce(Consul.ensureDockRemoved); + sinon.assert.calledWith( + Consul.ensureDockRemoved, + dockerUrl + ); + + sinon.assert.calledOnce(publishStub); + sinon.assert.calledWith( + publishStub, + 'dock.removed', + sinon.match({ host: dockerUrl }) + ); + + done(); + }); + }); + }); // end handleEnsureDockRemoved }); // docker events diff --git a/test/unit/models/redis.js b/test/unit/models/redis.js index e72a7aa..52084a9 100644 --- a/test/unit/models/redis.js +++ b/test/unit/models/redis.js @@ -14,7 +14,7 @@ var redis = require('redis'); var Redis = require('../../../lib/models/redis.js'); -describe('redis.js unit test', function () { +describe('lib/models/redis.js unit test', function () { describe('connect', function () { beforeEach(function (done) { sinon.stub(redis, 'createClient'); diff --git a/test/unit/models/worker-server.js b/test/unit/models/worker-server.js index 0726353..eade417 100644 --- a/test/unit/models/worker-server.js +++ b/test/unit/models/worker-server.js @@ -16,7 +16,7 @@ var ponos = require('ponos'); var RabbitMQ = require('../../../lib/rabbitmq.js'); var WorkerServer = require('../../../lib/models/worker-server.js'); -describe('WorkerServer unit test', function () { +describe('lib/models/worker-server unit test', function () { describe('listen', function () { beforeEach(function (done) { sinon.stub(RabbitMQ, 'getSubscriber'); diff --git a/test/unit/server.js b/test/unit/server.js index 6062049..e5ff2ca 100644 --- a/test/unit/server.js +++ b/test/unit/server.js @@ -18,6 +18,7 @@ var docksMonitor = require('../../lib/models/docks-monitor'); var RabbitMQ = require('../../lib/rabbitmq.js'); var Server = require('../../lib/server.js'); var app = require('../../lib/app'); +var Docker = require('../../lib/models/docker.js'); describe('server.js unit test', function () { @@ -29,6 +30,7 @@ describe('server.js unit test', function () { sinon.stub(app, 'listen'); sinon.stub(RabbitMQ, 'create'); sinon.stub(WorkerServer, 'listen'); + sinon.stub(Docker, 'loadCerts'); done(); }); @@ -39,6 +41,7 @@ describe('server.js unit test', function () { app.listen.restore(); RabbitMQ.create.restore(); WorkerServer.listen.restore(); + Docker.loadCerts.restore(); done(); }); @@ -46,6 +49,7 @@ describe('server.js unit test', function () { monitor.startSocketsMonitor.returns(); docksMonitor.start.returns(); Redis.connect.returns(); + Docker.loadCerts.returns(); app.listen.yieldsAsync(); RabbitMQ.create.yieldsAsync(); WorkerServer.listen.yieldsAsync(); diff --git a/test/unit/workers/container.life-cycle.died.js b/test/unit/workers/container.life-cycle.died.js index 9a22db8..59b104d 100644 --- a/test/unit/workers/container.life-cycle.died.js +++ b/test/unit/workers/container.life-cycle.died.js @@ -1,29 +1,22 @@ 'use strict'; require('loadenv')(); -var Promise = require('bluebird') var Lab = require('lab'); var lab = exports.lab = Lab.script(); var describe = lab.describe; var it = lab.it; -var afterEach = lab.afterEach; -var beforeEach = lab.beforeEach; var Code = require('code'); var expect = Code.expect; - var sinon = require('sinon'); var ponos = require('ponos'); var TaskFatalError = ponos.TaskFatalError; -var ErrorCat = require('error-cat'); -var RabbitMQ = require('../../../lib/rabbitmq.js'); var Events = require('../../../lib/models/events.js'); var dockData = require('../../../lib/models/dockData.js'); var containerLifeCycleDied = require('../../../lib/workers/container.life-cycle.died.js'); describe('container.life-cycle.died.js unit test', function () { - it('should throw error if invalid from', function (done) { sinon.stub(Events, '_hasValidFrom').returns(false); containerLifeCycleDied({}) @@ -72,9 +65,7 @@ describe('container.life-cycle.died.js unit test', function () { dockData.incKey.restore(); done(); }) - .catch(function (err) { - throw new Error('Should not happen'); - }); + .catch(done); }); it('should increment counter if build container', function (done) { @@ -100,8 +91,6 @@ describe('container.life-cycle.died.js unit test', function () { dockData.incKey.restore(); done(); }) - .catch(function (err) { - throw new Error('Should not happen'); - }); + .catch(done); }); }); // end container.life-cycle.died unit test diff --git a/test/unit/workers/dock.wait-for-removal.js b/test/unit/workers/dock.wait-for-removal.js new file mode 100644 index 0000000..0402888 --- /dev/null +++ b/test/unit/workers/dock.wait-for-removal.js @@ -0,0 +1,66 @@ +'use strict'; +require('loadenv')(); + +var Lab = require('lab'); +var lab = exports.lab = Lab.script(); +var describe = lab.describe; +var it = lab.it; +var afterEach = lab.afterEach; +var beforeEach = lab.beforeEach; +var Code = require('code'); +var expect = Code.expect; + +var sinon = require('sinon'); +var TaskFatalError = require('ponos').TaskFatalError; + +var Events = require('../../../lib/models/events.js'); +var ensureDockRemovedWorker = require('../../../lib/workers/dock.wait-for-removal.js'); + +describe('lib/workers/dock.wait-for-removal unit test', function () { + describe('run', function () { + beforeEach(function (done) { + sinon.stub(Events, 'handleEnsureDockRemovedAsync'); + done(); + }); + + afterEach(function (done) { + Events.handleEnsureDockRemovedAsync.restore(); + done(); + }); + + it('should throw error if handleEnsureDockRemovedAsync failed', function (done) { + var error = new Error('test'); + Events.handleEnsureDockRemovedAsync.throws(error); + ensureDockRemovedWorker({ + dockerUrl: '10.0.0.1:4224', + }) + .then(function () { + throw new Error('should have thrown'); + }) + .catch(function (err) { + expect(err).to.equal(error); + done(); + }); + }); + + it('should throw missing dockerUrl', function (done) { + ensureDockRemovedWorker({}) + .then(function () { + throw new Error('should have thrown'); + }) + .catch(function (err) { + expect(err).to.be.instanceOf(TaskFatalError); + done(); + }); + }); + + it('should be fine if no errors', function (done) { + Events.handleEnsureDockRemovedAsync.returns(); + ensureDockRemovedWorker({ + dockerUrl: '10.0.0.1:4224' + }) + .then(done) + .catch(done); + }); + }); // end run +}); // end docker.events-stream.connected unit test diff --git a/test/unit/workers/docker.events-stream.connected.js b/test/unit/workers/docker.events-stream.connected.js index ca6663c..9af89d8 100644 --- a/test/unit/workers/docker.events-stream.connected.js +++ b/test/unit/workers/docker.events-stream.connected.js @@ -1,29 +1,22 @@ 'use strict'; require('loadenv')(); -var Promise = require('bluebird') var Lab = require('lab'); var lab = exports.lab = Lab.script(); var describe = lab.describe; var it = lab.it; -var afterEach = lab.afterEach; -var beforeEach = lab.beforeEach; var Code = require('code'); var expect = Code.expect; - var sinon = require('sinon'); var ponos = require('ponos'); var TaskFatalError = ponos.TaskFatalError; -var ErrorCat = require('error-cat'); -var RabbitMQ = require('../../../lib/rabbitmq.js'); var Events = require('../../../lib/models/events.js'); var dockData = require('../../../lib/models/dockData.js'); var dockerEventsStreamConnected = require('../../../lib/workers/docker.events-stream.connected.js'); -describe('docker.events-stream.connected.js unit test', function () { - +describe('lib/workers/docker.events-stream.connected.js unit test', function () { it('should throw error if invalid host', function (done) { sinon.stub(Events, '_hasValidHost').returns(false); dockerEventsStreamConnected({}) @@ -55,8 +48,6 @@ describe('docker.events-stream.connected.js unit test', function () { dockData.addHost.restore(); done(); }) - .catch(function (err) { - throw new Error('Should not happen'); - }); + .catch(done); }); }); // end docker.events-stream.connected unit test diff --git a/test/unit/workers/docker.events-stream.disconnected.js b/test/unit/workers/docker.events-stream.disconnected.js index b4a24fd..1c1327b 100644 --- a/test/unit/workers/docker.events-stream.disconnected.js +++ b/test/unit/workers/docker.events-stream.disconnected.js @@ -1,28 +1,22 @@ 'use strict'; require('loadenv')(); -var Promise = require('bluebird') var Lab = require('lab'); var lab = exports.lab = Lab.script(); var describe = lab.describe; var it = lab.it; -var afterEach = lab.afterEach; -var beforeEach = lab.beforeEach; var Code = require('code'); var expect = Code.expect; - var sinon = require('sinon'); var ponos = require('ponos'); var TaskFatalError = ponos.TaskFatalError; -var ErrorCat = require('error-cat'); -var RabbitMQ = require('../../../lib/rabbitmq.js'); var Events = require('../../../lib/models/events.js'); var dockData = require('../../../lib/models/dockData.js'); var dockerEventsStreamDisconnected = require('../../../lib/workers/docker.events-stream.disconnected.js'); -describe('docker.events-stream.disconnected.js unit test', function () { +describe('lib/workers/docker.events-stream.disconnected.js unit test', function () { it('should throw error if invalid host', function (done) { sinon.stub(Events, '_hasValidHost').returns(false); @@ -54,8 +48,6 @@ describe('docker.events-stream.disconnected.js unit test', function () { dockData.deleteHost.restore(); done(); }) - .catch(function (err) { - throw new Error('Should not happen'); - }); + .catch(done); }); }); // end docker.events-stream.disconnected unit test diff --git a/test/unit/workers/on-dock-unhealthy.js b/test/unit/workers/on-dock-unhealthy.js index 27b6e28..1681936 100644 --- a/test/unit/workers/on-dock-unhealthy.js +++ b/test/unit/workers/on-dock-unhealthy.js @@ -16,86 +16,134 @@ var sinon = require('sinon'); var ponos = require('ponos'); var TaskFatalError = ponos.TaskFatalError; var TaskError = ponos.TaskError; -var Worker = require('../../../lib/workers/on-dock-unhealthy.js'); var dockData = require('../../../lib/models/dockData.js'); var rabbitMQ = require('../../../lib/rabbitmq.js'); var Events = require('../../../lib/models/events.js'); +var Docker = require('../../../lib/models/docker.js'); var onDockUnhealthy = require('../../../lib/workers/on-dock-unhealthy.js'); -describe('on-dock-unhealthy unit test', function () { +describe('lib/workers/on-dock-unhealthy unit test', function () { + describe('failed', function () { + beforeEach(function (done) { + sinon.spy(Events, '_hasValidHost'); + sinon.stub(dockData, 'deleteHost') + sinon.stub(Docker.prototype, 'killSwarmContainer').yieldsAsync(null); + done(); + }); - it('should throw error if invalid host', function (done) { - sinon.spy(Events, '_hasValidHost'); - onDockUnhealthy({}) + afterEach(function (done) { + Events._hasValidHost.restore(); + dockData.deleteHost.restore(); + Docker.prototype.killSwarmContainer.restore(); + done(); + }); + + it('should throw error if invalid host', function (done) { + onDockUnhealthy({}) + .then(function () { + throw new Error('Should not happen'); + }) + .catch(function (err) { + expect(err).to.be.instanceOf(TaskFatalError); + sinon.assert.calledOnce(Events._hasValidHost); + done(); + }); + }); + + it('should throw error delete host failed', function (done) { + dockData.deleteHost.yieldsAsync(new Error('Redis error')); + onDockUnhealthy({ + host: 'http://10.12.12.11:4242', + }) .then(function () { throw new Error('Should not happen'); }) .catch(function (err) { - expect(err).to.be.instanceOf(TaskFatalError); - expect(Events._hasValidHost.calledOnce).to.be.true(); - Events._hasValidHost.restore(); + expect(err).to.be.instanceOf(TaskError); + sinon.assert.calledOnce(Events._hasValidHost); + sinon.assert.calledOnce(dockData.deleteHost); + sinon.assert.calledWith( + dockData.deleteHost, + 'http://10.12.12.11:4242' + ); done(); }); - }); + }); - it('should throw error delete host failed', function (done) { - sinon.spy(Events, '_hasValidHost'); - sinon.stub(dockData, 'deleteHost').yieldsAsync(new Error('Redis error')); - onDockUnhealthy({ - host: 'http://10.12.12.11:4242', - }) - .then(function () { - throw new Error('Should not happen'); - }) - .catch(function (err) { - expect(err).to.be.instanceOf(TaskError); - expect(Events._hasValidHost.calledOnce).to.be.true(); - expect(dockData.deleteHost.called).to.be.true(); - expect(dockData.deleteHost.getCall(0).args[0]).to.equal('http://10.12.12.11:4242'); - dockData.deleteHost.restore(); - Events._hasValidHost.restore(); - done(); + it('should throw if killingSwarmContainer failed', function (done) { + var dockerUrl = 'http://10.12.12.11:4242'; + + dockData.deleteHost.yieldsAsync(null); + Docker.prototype.killSwarmContainer.yieldsAsync(new Error('Docker Error')); + + onDockUnhealthy({ + host: dockerUrl, + }) + .then(function () { + throw new Error('Should not happen'); + }) + .catch(function (err) { + expect(err).to.be.instanceOf(TaskError); + sinon.assert.calledOnce(Events._hasValidHost); + sinon.assert.calledOnce(dockData.deleteHost); + sinon.assert.calledWith( + dockData.deleteHost, + dockerUrl + ); + done(); + }); }); - }); + }); // end failed describe('success', function () { + beforeEach(function (done) { + sinon.spy(Events, '_hasValidHost'); + sinon.stub(dockData, 'deleteHost').yieldsAsync(null); + sinon.stub(Docker.prototype, 'killSwarmContainer').yieldsAsync(null); + done(); + }); + afterEach(function (done) { + Events._hasValidHost.restore(); + dockData.deleteHost.restore(); + Docker.prototype.killSwarmContainer.restore(); rabbitMQ._publisher = null; done(); }); - it('should delete host if valid', function (done) { - sinon.spy(Events, '_hasValidHost'); - sinon.stub(dockData, 'deleteHost').yieldsAsync(null); + + it('should emit provision and wait events', function (done) { + var dockerUrl = 'http://10.12.12.11:4242'; + rabbitMQ._publisher = { publish: function (name, data) {} }; sinon.spy(rabbitMQ._publisher, 'publish'); onDockUnhealthy({ - host: 'http://10.12.12.11:4242', + host: dockerUrl, githubId: 12312 }) .then(function () { - expect(Events._hasValidHost.calledOnce).to.be.true(); - Events._hasValidHost.restore(); + sinon.assert.calledOnce(Events._hasValidHost); - expect(dockData.deleteHost.called).to.be.true(); - expect(dockData.deleteHost.getCall(0).args[0]).to.equal('http://10.12.12.11:4242'); - dockData.deleteHost.restore(); + sinon.assert.calledOnce(dockData.deleteHost); + sinon.assert.calledWith( + dockData.deleteHost, + dockerUrl + ); + + sinon.assert.calledTwice(rabbitMQ._publisher.publish); + sinon.assert.calledWith( + rabbitMQ._publisher.publish.getCall(0), + 'cluster-instance-provision', + sinon.match({ githubId: 12312 }) + ); + sinon.assert.calledWith( + rabbitMQ._publisher.publish.getCall(1), + 'dock.wait-for-removal', + sinon.match({ dockerUrl: dockerUrl }) + ); - expect(rabbitMQ._publisher.publish.getCall(0).args[0]).to.equal('on-dock-removed'); - expect(rabbitMQ._publisher.publish.getCall(0).args[1]).to.deep.equal({ - host: 'http://10.12.12.11:4242', - githubId: 12312 - }); - expect(rabbitMQ._publisher.publish.getCall(1).args[0]).to.equal('cluster-instance-provision'); - expect(rabbitMQ._publisher.publish.getCall(1).args[1]).to.deep.equal({ - githubId: 12312 - }); - rabbitMQ._publisher.publish.restore(); done(); - }) - .catch(function (err) { - throw new Error('Should not happen'); }); }); });