Wait for dock removed

circle.yml

@@ -0,0 +1,6 @@
+dependencies:
+  override:
+    - nvm install 4.2.2
+    - nvm alias default 4.2.2
+    - npm install -g npm@2.14.7
+    - npm install

configs/.env

@@ -1,20 +1,25 @@
-PORT=4000
-REDIS_HOST_KEYS="dockerHosts:"
 BUILD_WEIGHT=10
 CONTAINER_WEIGHT=1
+DOCKER_CERT_PATH=/etc/ssl/docker
 HISTORY_WEIGHT=10
+IMAGE_BUILDER=runnable/image-builder
+IMAGE_BUILDER_LABEL=image-builder-container
 LOG=true
-IMAGE_BUILDER="runnable/image-builder"
-IMAGE_BUILDER_LABEL="image-builder-container"
-RUNNABLE_REGISTRY="registry.runnable.com"
-ROLLBAR_KEY="f4888bcd48be45cabd42627dfed87bba"
-ROLLBAR_OPTIONS_BRANCH="master"
-NEWRELIC_NAME='mavis'
-NEWRELIC_LEVEL='error'
+NEWRELIC_LEVEL=error
+NEWRELIC_NAME=mavis
+PORT=4000
+REDIS_HOST_KEYS=dockerHosts:
+ROLLBAR_KEY=f4888bcd48be45cabd42627dfed87bba
+ROLLBAR_OPTIONS_BRANCH=master
+RUNNABLE_REGISTRY=registry.runnable.com
 
 # Monitor-dog configuration
-MONITOR_PREFIX="mavis"
+MONITOR_PREFIX=mavis
 MONITOR_INTERVAL=60000
 
 # Logging configuration
 LOG_LEVEL=trace
+
+# ponos
+WORKER_MAX_RETRY_DELAY=30000
+WORKER_MIN_RETRY_DELAY=10000

configs/.env.test

@@ -1,7 +1,11 @@
-PORT=65213
-REDIS_PORT=6379
-REDIS_IPADDRESS=127.0.0.1
+CONSUL_HOST=consul.com
+CONSUL_PORT=8500
+DOCKER_CERT_PATH=./test/fixtures/certs
 LOG_LEVEL=fatal
+PORT=65213
 RABBITMQ_HOSTNAME=localhost
 RABBITMQ_PASSWORD=guest
-RABBITMQ_USERNAME=guest
+RABBITMQ_USERNAME=guest
+REDIS_IPADDRESS=127.0.0.1
+REDIS_PORT=6379
+SWARM_CONTAINER_NAME=swarm

lib/models/consul.js

@@ -0,0 +1,50 @@
+/**
+ * Consul API requests
+ * @module lib/models/Consul
+ */
+'use strict';
+require('loadenv')('mavis:env');
+
+var url = require('url');
+var ErrorCat = require('error-cat');
+var error = new ErrorCat();
+
+var log = require('../logger').child({ module: 'consul' });
+
+module.exports = Consul;
+
+/**
+ * class used to talk to Consul
+ */
+function Consul () {}
+
+/**
+ * singleton consul client
+ * @type {Object}
+ */
+Consul._client = require('consul')({
+  host: process.env.CONSUL_HOST,
+  port: process.env.CONSUL_PORT
+});
+
+/**
+ * checks dock host key in consul to see if it exist.
+ * will cb with error if dock key still exist
+ * @param {String} dockerUrl docker host to check for format: http://10.0.0.1:4242
+ * @param {Function} cb (err)
+ */
+Consul.ensureDockRemoved = function (dockerUrl, cb) {
+  var host = url.parse(dockerUrl).host;
+  var logData = { host: host };
+  log.info(logData, 'Consul.prototype.ensureDockRemoved');
+  Consul._client.kv.get('swarm/docker/swarm/nodes/' + host, function(err, result) {
+    if (err) { return cb(err); }
+    // if we have a result that means the key still exist, cb with error
+    if (result) {
+      return cb(error.create(412, 'dock still exist', result));
+    }
+
+    log.trace(logData, 'ensureDockRemoved dock as been removed');
+    return cb(null);
+  });
+};

lib/models/docker.js

@@ -0,0 +1,74 @@
+/**
+ * Docker API requests
+ * @module lib/models/docker
+ */
+'use strict';
+require('loadenv')('mavis:env');
+
+var Dockerode = require('dockerode');
+var put = require('101/put');
+var fs = require('fs');
+var join = require('path').join;
+var url = require('url');
+
+var log = require('../logger').child({ module: 'docker' });
+
+var certs = {};
+
+module.exports = Docker;
+
+/**
+ * class used to talk to docker
+ * @param {string} dockerUrl format: http://hostname:hostport
+ */
+function Docker (dockerUrl) {
+  var parsedHost = url.parse(dockerUrl);
+
+  this._client = new Dockerode(put({
+    host: parsedHost.hostname,
+    port: parsedHost.port
+  }, certs));
+
+  this._logData = {
+    dockerUrl: dockerUrl
+  };
+  log.trace(this._logData, 'Docker constructor');
+}
+
+/**
+ * loads certs for docker. does not throw if failed, just logs
+ * sync function as this should only happen once on startup
+ */
+Docker.loadCerts = function () {
+  // try/catch is a better pattern for this, since checking to see if it exists
+  // and then reading files can lead to race conditions (unlikely, but still)
+  try {
+    var certPath = process.env.DOCKER_CERT_PATH;
+    certs.ca = fs.readFileSync(join(certPath, '/ca.pem'));
+    certs.cert = fs.readFileSync(join(certPath, '/cert.pem'));
+    certs.key = fs.readFileSync(join(certPath, '/key.pem'));
+    log.info('Docker.loadCerts docker certificates loaded');
+  } catch (err) {
+    log.warn({ err: err }, 'Docker.loadCerts cannot load certificates for docker');
+    throw err;
+  }
+};
+
+/**
+ * stop swarm docker container
+ * @param  {Function} cb (err)
+ */
+Docker.prototype.killSwarmContainer = function (cb) {
+  var self = this;
+  log.info(self._logData, 'Docker.prototype.killSwarmContainer');
+
+  self._client.getContainer(process.env.SWARM_CONTAINER_NAME).kill(function (err) {
+    if (err) {
+      log.error(put({ err: err }, self._logData), 'killSwarmContainer error killing container');
+      return cb(err);
+    }
+
+    log.trace(self._logData, 'killSwarmContainer killing container success');
+    cb(null);
+  });
+};

lib/models/events.js

@@ -3,21 +3,20 @@
 require('loadenv')('mavis:env');
 var url = require('url');
 var keypather = require('keypather')();
-
-var dockData = require('../models/dockData.js');
-var log = require('../logger').child({ module: 'events:docker' });
-
 var TaskFatalError = require('ponos').TaskFatalError;
 var TaskError = require('ponos').TaskError;
-var rabbitMQ = require('../rabbitmq.js');
 
+var dockData = require('../models/dockData.js');
+var Consul = require('../models/consul.js');
+var Docker = require('../models/docker.js');
+var rabbitMQ = require('../rabbitmq.js');
+var log = require('../logger').child({ module: 'events:docker' });
 
 /**
  * Module used to handle runnable events
  */
 var Events = module.exports = {};
 
-
 /**
  * Handles docker `die` events.
  * updates container build counts
@@ -84,12 +83,45 @@ Events.handleUnhealthy = function (data, cb) {
   }
   dockData.deleteHost(data.host, function (err) {
     if (err) {
-      var taskErr = new TaskError('Failed to delete host', err);
+      var taskErr = new TaskError('handleUnhealthy', 'Failed to delete host', err);
+      return cb(taskErr);
+    }
+    var docker = new Docker(data.host);
+    docker.killSwarmContainer(function (err) {
+      if (err) {
+        var taskErr = new TaskError('handleUnhealthy', 'Failed to kill swarm container', err);
+        return cb(taskErr);
+      }
+
+      rabbitMQ.getPublisher().publish('cluster-instance-provision', {
+        githubId: data.githubId
+      });
+      rabbitMQ.getPublisher().publish('dock.wait-for-removal', {
+        dockerUrl: data.host
+      });
+      cb();
+    });
+  });
+};
+
+/**
+ * waits for dock to be removed form consul
+ * then publish dock.removed event
+ * @param  {Object}   data job data
+ * @param  {String}   data.dockerUrl url to check for format: http://10.0.0.1:4242
+ * @param  {Function} cb   (err)
+ */
+Events.handleEnsureDockRemoved = function (data, cb) {
+  log.info({ data: data }, 'Events.handleEnsureDockRemoved');
+  Consul.ensureDockRemoved(data.dockerUrl, function (err) {
+    if (err) {
+      var taskErr = new TaskError('dock.wait-for-removal', 'dock still exists', err);
       return cb(taskErr);
     }
-    rabbitMQ.getPublisher().publish('on-dock-removed', data);
-    rabbitMQ.getPublisher().publish('cluster-instance-provision', {
-      githubId: data.githubId
+
+    log.trace({ data: data }, 'handleEnsureDockRemoved publishing dock.removed');
+    rabbitMQ.getPublisher().publish('dock.removed', {
+      host: data.dockerUrl
     });
     cb();
   });

lib/models/worker-server.js

@@ -27,6 +27,7 @@ WorkerServer.listen = function (cb) {
 
   var tasks = {
     'on-dock-unhealthy': require('../workers/on-dock-unhealthy.js'),
+    'dock.wait-for-removal': require('../workers/dock.wait-for-removal.js'),
     'container.life-cycle.died': require('../workers/container.life-cycle.died.js'),
     'docker.events-stream.connected': require('../workers/docker.events-stream.connected.js'),
     'docker.events-stream.disconnected': require('../workers/docker.events-stream.disconnected.js'),

lib/rabbitmq.js

@@ -39,7 +39,8 @@ RabbitMQ.prototype.create = function (cb) {
 
   this._subscriber = new Hermes(put({
     queues: [
-      'on-dock-unhealthy'
+      'on-dock-unhealthy',
+      'dock.wait-for-removal'
     ],
     subscribedEvents: [
       'container.life-cycle.died',
@@ -51,9 +52,12 @@ RabbitMQ.prototype.create = function (cb) {
 
   this._publisher = new Hermes(put({
     queues: [
-      'on-dock-removed',
+      'dock.wait-for-removal',
       'cluster-instance-provision'
     ],
+    publishedEvents: [
+      'dock.removed'
+    ]
   }, opts))
   // connect publisher only since ponos is handling subscriber
   .connect(cb)

lib/server.js

@@ -14,7 +14,7 @@ if (process.env.NEWRELIC_KEY) { require('newrelic'); }
 var Redis = require('./models/redis.js');
 var rabbitMQ = require('./rabbitmq.js');
 var WorkerServer = require('./models/worker-server.js');
-
+var Docker = require('./models/docker.js');
 
 module.exports = Server;
 
@@ -36,6 +36,7 @@ Server.prototype.start = function (cb) {
   monitor.startSocketsMonitor();
   docksMonitor.start();
   Redis.connect();
+  Docker.loadCerts();
   this.server = app.listen(process.env.PORT, function(err) {
     if (err) {
       log.fatal({ err: err }, 'Error starting server. Exiting');

lib/workers/dock.wait-for-removal.js

@@ -0,0 +1,28 @@
+/**
+  * Handles `dock.wait-for-removal` event
+  * @module lib/workers/dock.wait-for-removal
+  */
+'use strict';
+
+var isString = require('101/is-string');
+var Promise = require('bluebird');
+var TaskFatalError = require('ponos').TaskFatalError;
+
+var Events = Promise.promisifyAll(require('../models/events.js'));
+var log = require('../logger').child({ module: 'workers' });
+
+module.exports = function (job) {
+  return Promise.resolve()
+    .then(function validateArguments () {
+      if (!isString(job.dockerUrl)) {
+        throw new TaskFatalError('missing dockerUrl');
+      }
+    })
+    .then(function () {
+      return Events.handleEnsureDockRemovedAsync(job);
+    })
+    .catch(function (err) {
+      log.error({ err: err }, 'dock.wait-for-removal error');
+      throw err;
+    });
+};

package.json

@@ -31,13 +31,16 @@
     "body-parser": "^1.13.3",
     "boom": "^2.6.1",
     "bunyan": "^1.4.0",
+    "consul": "^0.19.0",
+    "dockerode": "^2.2.6",
     "error-cat": "^1.4.0",
     "express": "^4.13.3",
     "express-bunyan-logger": "^1.1.1",
     "keypather": "^1.10.1",
     "loadenv": "^1.1.0",
     "monitor-dog": "^1.4.1",
     "newrelic": "^1.22.0",
+    "nock": "^3.3.2",
     "node-dogstatsd": "0.0.6",
     "ponos": "^1.1.1",
     "redis": "^0.12.1",

test/fixtures/certs/ca.pem

@@ -0,0 +1 @@
+ca

test/fixtures/certs/cert.pem

@@ -0,0 +1 @@
+cert

test/fixtures/certs/key.pem

@@ -0,0 +1 @@
+key