diff --git a/configs/.env.development b/configs/.env.development index 8b6ae3a99..13bf633fa 100644 --- a/configs/.env.development +++ b/configs/.env.development @@ -15,7 +15,6 @@ GITHUB_DEPLOY_KEY_TITLE=Runnable-development GITHUB_HOOK_SECRET=3V3RYTHINGisAW3S0ME! KRAIN_PORT=3100 LOG_SRC=true -MAVIS_HOST=http://mavis-staging-codenow.runnableapp.com MONGO=mongodb://127.0.0.1:27017/runnable2 NEO4J=http://localhost:7474 OPTIMUS_HOST=optimus-staging-CodeNow.runnableapp.com diff --git a/configs/.env.production b/configs/.env.production index 4a27cf6f1..8bd7d3aa7 100644 --- a/configs/.env.production +++ b/configs/.env.production @@ -14,7 +14,6 @@ GITHUB_DEPLOY_KEYS_POOL_SIZE=100 GITHUB_DEPLOY_KEY_TITLE=Runnable-io KRAIN_PORT=3100 LOG_LEVEL_STDOUT=trace -MAVIS_HOST=http://mavis.runnable.io:80 NEW_RELIC_TRACER_ENABLED=false REGISTRY_DOMAIN=registry.runnable.com S3_CONTEXT_RESOURCE_BUCKET=runnable.context.resources.production diff --git a/configs/.env.production-beta b/configs/.env.production-beta index 8825432c3..57ff56dcc 100644 --- a/configs/.env.production-beta +++ b/configs/.env.production-beta @@ -16,7 +16,6 @@ GITHUB_DEPLOY_KEYS_POOL_SIZE=100 GITHUB_DEPLOY_KEY_TITLE=Runnable-io-beta KRAIN_PORT=3100 LOG_LEVEL_STDOUT=trace -MAVIS_HOST=http://mavis.runnable-beta.com:80 NEW_RELIC_TRACER_ENABLED=false REGISTRY_DOMAIN=registry.runnable.com S3_CONTEXT_RESOURCE_BUCKET=runnable-beta.context.resources.production diff --git a/configs/.env.production-gamma b/configs/.env.production-gamma index 90b303eb9..bcdd87501 100644 --- a/configs/.env.production-gamma +++ b/configs/.env.production-gamma @@ -16,7 +16,6 @@ GITHUB_DEPLOY_KEYS_POOL_SIZE=100 GITHUB_DEPLOY_KEY_TITLE=Runnable-io-gamma KRAIN_PORT=3100 LOG_LEVEL_STDOUT=trace -MAVIS_HOST=http://mavis.runnable-gamma.com:80 NEW_RELIC_TRACER_ENABLED=false REGISTRY_DOMAIN=registry.runnable.com S3_CONTEXT_RESOURCE_BUCKET=runnable.context.resources.production-beta diff --git a/configs/.env.staging b/configs/.env.staging index 41cb68091..1cdbf1148 100644 --- a/configs/.env.staging +++ b/configs/.env.staging @@ -24,7 +24,6 @@ GITHUB_HOOK_SECRET=3V3RYTHINGisAW3S0ME! KRAIN_PORT=3100 LOGGLY_TOKEN=f673760d-e0b3-4a93-a15e-2862ea074f91 LOG_SRC=true -MAVIS_HOST=http://mavis-staging-codenow.runnableapp.com MIXPANEL_APP_ID=abed31a0844a1bf389205355edfda4c2 MONGO=mongodb://mongodb-staging-codenow.runnableapp.com/alpha NAVI_HOST=http://10.0.1.41:33744 diff --git a/configs/.env.test b/configs/.env.test index b60d5ad7b..695cbbb9f 100644 --- a/configs/.env.test +++ b/configs/.env.test @@ -27,7 +27,6 @@ IS_QUEUE_WORKER=true KRAIN_PORT=1234 LOG_LEVEL_STDOUT=none LOG_SRC=true -MAVIS_HOST=http://localhost:4000 MONGO=mongodb://127.0.0.1:27017/runnable_test123 NAVI_HOST=http://localhost:1234 NEO4J=http://localhost:7474 diff --git a/lib/error.js b/lib/error.js index 3b539cf2a..515c9976e 100644 --- a/lib/error.js +++ b/lib/error.js @@ -52,10 +52,6 @@ function logFn (err) { errorData = assign(errorData, pick(err.data.krain, ['uri', 'statusCode', 'info'])) logger.log.error(errorData, 'log krain error') } - if (err.data.mavis) { - errorData = assign(errorData, pick(err.data.mavis, ['uri', 'statusCode', 'info'])) - logger.log.error(errorData, 'log mavis error') - } if (err.data.s3) { errorData = assign(errorData, pick(err.data.s3, ['bucket', 'contextId', 'sourcePath', 'sourceUrl'])) diff --git a/lib/models/apis/mavis.js b/lib/models/apis/mavis.js deleted file mode 100644 index 7a5be22aa..000000000 --- a/lib/models/apis/mavis.js +++ /dev/null @@ -1,159 +0,0 @@ -/** - * Mavis is used to find a dock - * @module lib/models/apis/mavis - */ -'use strict' - -var ApiClient = require('simple-api-client') -var bluebird = require('bluebird') -var Boom = require('dat-middleware').Boom -var isObject = require('101/is-object') -var keypather = require('keypather')() -var put = require('101/put') -var util = require('util') - -var log = require('middlewares/logger')(__filename).log - -module.exports = Mavis - -function Mavis (opts) { - this.host = process.env.MAVIS_HOST - log.trace({ - tx: true, - opts: opts, - host: this.host - }, 'Mavis constructor') - ApiClient.call(this, this.host, opts) -} - -util.inherits(Mavis, ApiClient) - -Mavis.prototype.findDockForBuild = function (contextVersion, context, cb) { - var logData = { - tx: true, - contextVersion: contextVersion, - context: context - } - log.info(logData, 'Mavis.prototype.findDockForBuild') - if (!isObject(contextVersion)) { - log.warn(logData, 'findDockForBuild: !isObject contextVersion') - return cb(new Error('missing contextVersion')) - } - if (!isObject(context)) { - log.warn(logData, 'findDockForBuild: !isObject context') - return cb(new Error('missing context')) - } - // tag must be a string - var tags = context.owner.github + ',build' - var opts = { - type: 'container_build', - tags: tags - } - opts.prevDuration = contextVersion.duration || 0 - opts.prevImage = contextVersion.dockerTag || null - this.findDock(opts, cb) -} - -Mavis.prototype.findDockForContainer = function (contextVersion, cb) { - var logData = { - tx: true, - contextVersion: contextVersion - } - log.info(logData, 'Mavis.prototype.findDockForContainer') - if (!isObject(contextVersion)) { - log.warn(logData, 'findDockForContainer: !isObject contextVersion') - return cb(new Error('missing contextVersion')) - } else if (!keypather.get(contextVersion, 'owner.github')) { - log.warn(logData, 'findDockForContainer: contextVersion missing owner') - return cb(new Error('missing contextVersion owner')) - } - // tag must be a string - var tags = contextVersion.owner.github + ',run' - var opts = { - type: 'container_run', - tags: tags - } - // if dockerHost is not an address, its invalid - opts.prevDock = contextVersion.dockerHost || null - this.findDock(opts, cb) -} - -/** - * ask mavis for dock to run provided task on - * @param taskType: ['container_build', 'container_run'] - * @param prevDock: previous dock this image was run on - * @param cb: Callback - */ -Mavis.prototype.findDock = function (opts, cb) { - var logData = { - opts: opts, - tx: true - } - log.info(logData, 'Mavis.prototype.findDock') - var self = this - this.post('dock', { - json: opts - }, function (err, res) { - // For logging purposes - if (keypather.get(res, 'body')) { - logData.resBody = res.body - } - - if (err) { - log.error(put({ - err: err - }, logData), 'findDock: this.post error') - var boomErr = Boom.create(504, 'Unable to find dock', { - mavis: { - uri: self.host + '/dock' - }, - err: err - }) - cb(boomErr) - } else if (res.statusCode === 503 && opts.tags !== 'default') { - log.warn(logData, 'findDock: this.post elseif 503 error') - // if no docks and non default tag, try with default tag - opts.tags = 'default' - self.findDock(opts, cb) - } else if (res.statusCode >= 300) { - log.warn(logData, 'findDock: this.post elseif >=300 error') - cb(responseErr(res)) - } else { - log.trace(logData, 'findDock: success') - cb(null, res.body.dockHost) - } - }) -} - -function responseErr (res) { - log.info({tx: true, res: res}, 'models/apis/mavis responseErr') - var message - var code - if (res.statusCode >= 500) { - message = 'Unknown error from mavis' - code = res.statusCode === 500 - ? 502 - : res.statusCode - } else if (res.statusCode >= 400) { - message = 'Bad request error from mavis' - code = res.statusCode - } else if (res.statusCode >= 300) { - // mavis doesnt send redirects... so this is unexpected - message = 'Unexpected response from mavis' - code = 502 - } - if (res.body && res.body.message) { - message += ': ' + res.body.message - } - return Boom.create(code, message, { - mavis: { - uri: res.request.uri, - statusCode: res.statusCode, - info: res.body - } - }) -} - -// promisifyAll at the end of file -bluebird.promisifyAll(Mavis) -bluebird.promisifyAll(Mavis.prototype) diff --git a/lib/models/rabbitmq/index.js b/lib/models/rabbitmq/index.js index 217741c3c..5a6e5a51c 100644 --- a/lib/models/rabbitmq/index.js +++ b/lib/models/rabbitmq/index.js @@ -44,7 +44,8 @@ RabbitMQ.prototype.connect = function (cb) { username: process.env.RABBITMQ_USERNAME, subscribedEvents: [ 'container.network.attached', - 'container.network.attach-failed' + 'container.network.attach-failed', + 'dock.removed' ], publishedEvents: [ 'instance.created', @@ -60,7 +61,6 @@ RabbitMQ.prototype.connect = function (cb) { 'delete-instance-container', 'deploy-instance', 'metis-github-event', - 'on-dock-removed', 'on-image-builder-container-create', 'on-image-builder-container-die', 'on-instance-container-create', @@ -115,7 +115,7 @@ RabbitMQ.prototype.loadWorkers = function () { var ponosTasks = this.ponosTasks = { 'create-instance-container': require('workers/create-instance-container'), 'pull-instance-image': require('workers/pull-instance-image'), - 'on-dock-removed': require('workers/on-dock-removed') + 'dock.removed': require('workers/dock.removed') } // ponos: subscribe to queues var ponosServer = this.ponosServer = new ponos.Server({ diff --git a/lib/workers/on-dock-removed.js b/lib/workers/dock.removed.js similarity index 81% rename from lib/workers/on-dock-removed.js rename to lib/workers/dock.removed.js index e11c9b250..0164652c2 100644 --- a/lib/workers/on-dock-removed.js +++ b/lib/workers/dock.removed.js @@ -2,7 +2,7 @@ * Respond to dock-unhealthy event from docker-listener * - get running containers on dock * - redeploy those containers - * @module lib/workers/on-dock-removed + * @module lib/workers/dock.removed */ 'use strict' @@ -17,7 +17,7 @@ var joi = require('utils/joi') var TaskFatalError = require('ponos').TaskFatalError var log = require('middlewares/logger')(__filename).log -module.exports = OnDockRemovedWorker +module.exports = DockRemovedWorker /** * Main handler for docker unhealthy event @@ -27,7 +27,7 @@ module.exports = OnDockRemovedWorker * @param {Object} job - Job info * @returns {Promise} */ -function OnDockRemovedWorker (job) { +function DockRemovedWorker (job) { job = job || true // Do this so joi will trigger validation failure on empty job var logData = { tx: true, @@ -41,7 +41,7 @@ function OnDockRemovedWorker (job) { .catch(function (err) { throw new TaskFatalError(err) }) - .then(log.trace.bind(log, logData, 'OnDockRemovedWorker handle')) + .then(log.trace.bind(log, logData, 'DockRemovedWorker handle')) .then(function () { return Promise.fromCallback(function (cb) { ContextVersion.markDockRemovedByDockerHost(job.host, cb) @@ -58,16 +58,16 @@ function OnDockRemovedWorker (job) { }) }) .then(function (instances) { - log.trace(logData, 'OnDockRemovedWorker - Instances found ' + instances.length) + log.trace(logData, 'DockRemovedWorker - Instances found ' + instances.length) if (instances.length > 0) { return Promise.all([ - OnDockRemovedWorker._redeployContainers(instances), - OnDockRemovedWorker._updateFrontendInstances(instances) + DockRemovedWorker._redeployContainers(instances), + DockRemovedWorker._updateFrontendInstances(instances) ]) } }) .catch(function (e) { - log.trace(logData, 'OnDockRemovedWorker - ERROR! ' + e) + log.trace(logData, 'DockRemovedWorker - ERROR! ' + e) throw e }) } @@ -78,11 +78,11 @@ function OnDockRemovedWorker (job) { * @returns {Promise} * @private */ -OnDockRemovedWorker._redeployContainers = function (instances) { +DockRemovedWorker._redeployContainers = function (instances) { var runnableClient = new Runnable(process.env.FULL_API_DOMAIN, { requestDefaults: { headers: { - 'user-agent': 'worker-on-dock-removed' + 'user-agent': 'worker-dock.removed' } } }) @@ -107,7 +107,7 @@ OnDockRemovedWorker._redeployContainers = function (instances) { }) } -OnDockRemovedWorker._updateFrontendInstances = function (instances) { +DockRemovedWorker._updateFrontendInstances = function (instances) { return Promise.all( instances .map(function (instance) { diff --git a/lib/workers/pull-instance-image.js b/lib/workers/pull-instance-image.js index d4c48e041..b7da6808a 100644 --- a/lib/workers/pull-instance-image.js +++ b/lib/workers/pull-instance-image.js @@ -16,7 +16,6 @@ var TaskFatalError = require('ponos').TaskFatalError var Docker = require('models/apis/docker') var Instance = require('models/mongo/instance') var joi = require('utils/joi') -var Mavis = require('models/apis/mavis') var rabbitMQ = require('models/rabbitmq') var toJSON = require('utils/to-json') var toObjectId = require('utils/to-object-id') @@ -62,23 +61,13 @@ function pullInstanceImage (job) { return instance }) }) - .then(function findDockerHost (instance) { - log.info(logData, 'onInstanceImagePull.findDockerHost') - var mavis = new Mavis() - var cv = instance.contextVersion - return Promise.props({ - instance: instance, - dockerHost: mavis.findDockForContainerAsync(cv) - }) - }) - .then(function modifyInstanceImagePull (data) { + .then(function modifyInstanceImagePull (instance) { log.info(logData, 'onInstanceImagePull.modifyInstanceImagePull') - var instance = data.instance var cv = instance.contextVersion var dockerTag = cv.build.dockerTag var instancePromise = instance.modifyImagePullAsync(cv._id, { dockerTag: dockerTag, - dockerHost: data.dockerHost, + dockerHost: process.env.SWARM_HOST, sessionUser: { github: job.sessionUserGithubId }, @@ -93,14 +82,14 @@ function pullInstanceImage (job) { }) return Promise.props({ instance: instancePromise, - dockerHost: data.dockerHost, + dockerHost: process.env.SWARM_HOST, dockerTag: dockerTag }) }) .then(function pullImage (data) { log.info(logData, 'onInstanceImagePull.pullImage') var instance = data.instance - var docker = new Docker(data.dockerHost) + var docker = new Docker(process.env.SWARM_HOST) return docker.pullImageAsync(data.dockerTag) .then(function () { return data.instance // return instance diff --git a/package.json b/package.json index 04ba64b92..2f4274c8f 100644 --- a/package.json +++ b/package.json @@ -118,7 +118,6 @@ "jsdoc": "^3.2.2", "krain": "git+ssh://git@github.com:CodeNow/krain.git#v0.0.9", "lab": "5.8.0", - "mavis": "git+ssh://git@github.com:CodeNow/mavis#v0.3.0", "multiline": "^1.0.2", "nock": "^0.51.0", "node-inspector": "^0.9.2", diff --git a/scripts/sauron-restore.js b/scripts/sauron-restore.js deleted file mode 100644 index 3a1c4048f..000000000 --- a/scripts/sauron-restore.js +++ /dev/null @@ -1,80 +0,0 @@ -'use strict' -require('loadenv')() -var redis = require('models/redis') -var Instance = require('models/mongo/instance') -var mongoose = require('models/mongo/mongoose-control') -var createCount = require('callback-count') -var async = require('async') -var parseUrl = require('url').parse - -mongoose.start(function () { - restoreHosts(function (err) { - if (err) { throw err } - - console.log('done... disconnect from mongo') - mongoose.stop(function (err) { - if (err) { throw err } - console.log('DONE!') - process.exit(0) - }) - }) - - function restoreHosts (cb) { - Instance.find({ 'container.dockerContainer': { $exists: true } }, function (err, instances) { - if (err) { throw err } - - async.eachLimit(instances, 100, function (instance, cb) { - var hostIp = instance.network.hostIp - var networkIp = instance.network.networkIp - var dockerHostname = parseUrl(instance.container.dockerHost).hostname - var containerId = instance.container.dockerContainer - - // LOGS for DEBUGGING - // var count = createCount(4, cb) - // redis.hmget('weave:network', networkIp, function (err, val) { - // // NOTE: may not actually match - // if (dockerHostname !== val[0]) { - // console.log('INSTANCE', instance._id.toString()) - // console.log('EXPECT weave:network', networkIp, dockerHostname) - // console.log('ACTUAL weave:network', networkIp, val[0]) - // } - // count.next() - // }) - // redis.hmget('weave:network:container', hostIp, function (err, val) { - // // NOTE: may not actually match - // if (containerId !== val[0]) { - // console.log('INSTANCE', instance._id.toString()) - // console.log('EXPECT weave:network:container', hostIp, containerId) - // console.log('ACTUAL weave:network:container', hostIp, val[0]) - // } - // count.next() - // }) - // redis.hmget('weave:network:'+networkIp, hostIp, function (err, val) { - // // NOTE: may not actually match - // if(dockerHostname !== val[0]) { - // console.log('INSTANCE', instance._id.toString()) - // console.log('EXPECT weave:network:'+networkIp, hostIp, dockerHostname) - // console.log('ACTUAL weave:network:'+networkIp, hostIp, val[0]) - // } - // count.next() - // }) - // redis.hmget('weave:network:'+networkIp, networkIp, function (err, val) { - // if(networkIp !== val[0]) { - // console.log('INSTANCE', instance._id.toString()) - // console.log('EXPECT weave:network:'+networkIp, networkIp, networkIp) - // console.log('ACTUAL weave:network:'+networkIp, networkIp, val[0]) - // } - // count.next() - // }) - - // ACTUAL WRITE - var count = createCount(cb) - console.log('RESTORE', instance._id.toString()) - redis.hset('weave:network', networkIp, dockerHostname, count.inc().next) - redis.hset('weave:network:' + networkIp, networkIp, networkIp, count.inc().next) - redis.hset('weave:network:' + networkIp, hostIp, dockerHostname, count.inc().next) - redis.hset('weave:network:container', hostIp, containerId, count.inc().next) - }, cb) - }) - } -}) diff --git a/scripts/sauron-state/could-not-get-router-ip-fix.js b/scripts/sauron-state/could-not-get-router-ip-fix.js deleted file mode 100644 index b61459b02..000000000 --- a/scripts/sauron-state/could-not-get-router-ip-fix.js +++ /dev/null @@ -1,129 +0,0 @@ -'use strict' - -var mongoose = require('mongoose') -var redis = require('models/redis') -var Network = require('models/mongo/network') -var Instance = require('models/mongo/instance') -var createCount = require('callback-count') -var equals = require('101/equals') -var not = require('101/not') -var pluck = require('101/pluck') - -if (!process.env.NODE_PATH) { - throw new Error('NODE_PATH=./lib is required') -} -if (!process.env.MONGO) { - throw new Error('MONGO is required') -} -if (!process.env.SCRIPT_ORG_ID) { - throw new Error('SCRIPT_ORG_ID is required') -} -if (!process.env.ACTUALLY_RUN) { - console.log('DRY RUN!') -} -mongoose.connect(process.env.MONGO) - -/** - * Main - */ -main() -function main () { - var orgId = process.env.SCRIPT_ORG_ID - var count = createCount(2, handleIps) - var activeIps, allocatedIps, orgNetworkIp - // parallel - findActiveIpsForOrg(orgId, function (err, _activeIps) { - activeIps = _activeIps - count.next(err) - }) - findOrgNetworkIp(orgId, function (err, _orgNetworkIp) { - if (err) { count.next(err) } - orgNetworkIp = _orgNetworkIp - findAllocatedIpsForOrg(orgNetworkIp, function (err, _allocatedIps) { - if (err) { count.next(err) } - allocatedIps = _allocatedIps - count.next(err) - }) - }) - function handleIps (err) { - if (err) { throw err } - removeInactiveIps(orgNetworkIp, allocatedIps, activeIps, function (err) { - if (err) { throw err } - console.log('SCRIPT COMPLETE, all allocated inactive ips removed') - mongoose.disconnect() - }) - } -} - -/** - * Utils - */ - -/** - * find network host ips that are in use by org instances (mongo) - * @param {Number} orgId org github id - * @param {Function} cb callback - */ -function findActiveIpsForOrg (orgId, cb) { - var query = { 'owner.github': orgId } - var fields = { 'network.hostIp': 1 } - Instance.find(query, fields, function (err, instances) { - if (err) { return cb(err) } - cb(null, instances.map(pluck('network.hostIp'))) - }) -} - -/** - * find org network ip by org - * @param {Number} orgId org github id - * @param {Function} cb callback - */ -function findOrgNetworkIp (orgId, cb) { - Network.findOne({ 'owner.github': orgId }, function (err, network) { - if (err) { return cb(err) } - if (!network) { - err = new Error('network for org not found:' + orgId) - return cb(err) - } - cb(null, network.ip) - }) -} - -/** - * find network host ips allocated by sauron (redis) - * @param {String} orgNetworkIp org network ip - * @param {Function} cb callback - */ -function findAllocatedIpsForOrg (orgNetworkId, cb) { - var networkHashKey = 'weave:network:' + orgNetworkId - redis.hkeys(networkHashKey, cb) -} - -/** - * remove inactive allocated ips (redis) - * @param {String} orgNetworkIp org network ip - * @param {Array} allocatedIps array of allocated ips (from sauron, redis) - * @param {Array} activeIps array of active ips (instances, mongo) - * @param {Function} cb callback - */ -function removeInactiveIps (orgNetworkIp, allocatedIps, activeIps, cb) { - var inactiveIps = allocatedIps.filter(function (allocatedIp) { - var notEquals = not(equals) - return activeIps.every(notEquals(allocatedIp)) - }) - console.log('ALLOCATED IPS', allocatedIps) - console.log('ACTIVE IPS', activeIps) - var count = createCount(inactiveIps.length, cb) - var networkHashKey = 'weave:network:' + orgNetworkIp - if (inactiveIps.length === 0) { - return cb() - } - console.log('REMOVE IPS!!!', inactiveIps) - inactiveIps.forEach(function (inactiveIp) { - if (process.env.ACTUALLY_RUN) { - redis.hdel(networkHashKey, inactiveIp, count.next) - } else { - count.next() - } - }) -} diff --git a/scripts/smoke.js b/scripts/smoke.js deleted file mode 100644 index 2f4e42b5f..000000000 --- a/scripts/smoke.js +++ /dev/null @@ -1,82 +0,0 @@ -'use strict' - -var async = require('async') -var Runnable = require('runnable') -var user = new Runnable('localhost:3030') -var uuid = require('uuid') -// var createCount = require('callback-count') -var find = require('101/find') -var hasKeypaths = require('101/has-keypaths') - -var projectName = uuid() -var ctx = {} - -async.series([ - function (cb) { ctx.user = user.githubLogin('fc85cf8ce7d69de48cecd29a626dd8cfa6841a49', cb) }, - function (cb) { ctx.sourceContexts = ctx.user.fetchContexts({isSource: true}, cb) }, - function (cb) { ctx.sourceVersions = ctx.sourceContexts.models[0].fetchVersions({}, cb) }, - function (cb) { ctx.project = ctx.user.createProject({ name: projectName }, cb) }, - function (cb) { - ctx.env = ctx.project.newEnvironment(ctx.project.json().defaultEnvironment) - cb() - }, - function (cb) { ctx.build = ctx.env.createBuild({}, cb) }, - function (cb) { ctx.context = ctx.user.createContext({ name: projectName }, cb) }, - function (cb) { - ctx.contextVersion = ctx.context.createVersion({ - qs: { - toBuild: ctx.build.id() - }, - json: { - environment: ctx.env.id() - } - }, cb) - }, - function (cb) { ctx.contextVersion.addGithubRepo('bkendall/hairy-bear', cb) }, - function (cb) { - var icv = ctx.sourceVersions.models[0].json().infraCodeVersion - ctx.contextVersion.copyFilesFromSource(icv, cb) - }, - function (cb) { - ctx.files = ctx.contextVersion.rootDir.contents - ctx.files.fetch({path: '/'}, function (err) { - if (err) { return cb(err) } - ctx.dockerfile = find(ctx.files.models, hasKeypaths({'id()': '/Dockerfile'})) - cb() - }) - }, - function (cb) { - ctx.dockerfile.update({ - json: { - body: 'FROM dockerfile/nodejs\nADD ./hairy-bear /hb\n' + - 'WORKDIR /hb\nEXPOSE 80\nCMD ["node", "server.js"]' - } - }, cb) - }, - function (cb) { ctx.build.build({ message: uuid() }, cb) }, - function (cb) { - async.whilst( - function () { - return ctx.build && - !(ctx.build.json().completed || ctx.build.json().failed) - }, - function (cb) { ctx.build.fetch(cb) }, - cb) - }, - function (cb) { - ctx.instance = ctx.user.createInstance({ - json: { - build: ctx.build.id(), - name: uuid() - } - }, cb) - } -], function (err) { - if (err) { - console.error('err', err, err.stack) - process.exit(1) - } else { - console.log('done!') - process.exit(0) - } -}) diff --git a/scripts/update-docker.js b/scripts/update-docker.js deleted file mode 100644 index 9c9e0aa0b..000000000 --- a/scripts/update-docker.js +++ /dev/null @@ -1,229 +0,0 @@ -'use strict' -require('loadenv')() - -var async = require('async') -var request = require('request') -var fullUrl = 'http://' + process.env.TARGET_DOCK + ':4242' -var redis = require('models/redis') -var Runnable = require('runnable') -var user = new Runnable('localhost:80') -var saveKey = 'migrateDock:' + process.env.TARGET_DOCK -var MongoUser = require('models/mongo/user') -var Instance = require('models/mongo/instance') - -var mongoose = require('mongoose') -mongoose.connect(process.env.MONGO) - -var ERRORS = [] -// ensure env's -;['MONGO', 'MAVIS_HOST', 'TARGET_DOCK'].forEach(function (item) { - if (!process.env[item]) { - console.error('missing', item) - process.exit(1) - } -}) - -function login (cb) { - console.log('login') - var thisUser = user.githubLogin('f914c65e30f6519cfb4d10d0aa81e235dd9b3652', function (err) { - if (err) { return cb(err) } - MongoUser.updateById(thisUser.id(), { $set: { permissionLevel: 5 } }, cb) - }) -} - -// remove dock from mavis -function removeFromMavis (cb) { - console.log('removeFromMavis') - request({ - method: 'DELETE', - url: process.env.MAVIS_HOST + '/docks', - qs: { - host: fullUrl - } - }, function (err, res) { - if (err) { return cb(err) } - if (res.statusCode !== 200) { return cb(new Error('mavis delete failed')) } - cb() - }) -} - -// save list of running containers -// docker ps -// get context-version of containers and save in redis -var thisList = {} - -function saveList (cb) { - console.log('saveList') - Instance.find({ - 'container.dockerHost': fullUrl, - 'container.inspect.State.Running': true - }, function (err, instances) { - if (err) { return cb(err) } - thisList = instances - var multi = redis.multi() - multi.del(saveKey) - instances.forEach(function (item) { - multi.lpush(saveKey, item.shortHash) - }) - multi.exec(cb) - }) -} - -// stop all containers -function stopAllContainers (cb) { - console.log('stopAllContainers') - async.eachLimit(thisList, 10, function (instance, next) { - stopInstance(instance.shortHash, next) - }, cb) -} - -function stopInstance (shortHash, cb) { - var Instance = user.fetchInstance(shortHash, function (err) { - if (err) { - ERRORS.push({ - func: 'stopInstance:fetchInstance', - err: err.message, - shortHash: shortHash - }) - return cb() - } - Instance.stop(function (err) { - if (err) { - ERRORS.push({ - func: 'stopInstance:stop', - err: err.message, - shortHash: shortHash - }) - } - cb() - }) - }) -} - -function logErrs (cb) { - console.log('save errors', ERRORS) - cb() -} - -function saveAndKill (cb) { - async.series([ - login, - removeFromMavis, - saveList, - stopAllContainers, - logErrs - ], cb) -} - -// ////////////////////////////////////////////////// -// part 2 (seemless restart) -// ////////////////////////////////////////////////// -var ctx = {numStarted: 0} -function getAllContainers (cb) { - console.log('getAllContainers') - redis.lrange(saveKey, 0, -1, cb) -} - -function startAllContainers (instances, cb) { - console.log('instances') - ctx.retry = [] - async.eachLimit(instances, 10, function (shortHash, next) { - startInstance(shortHash, next) - }, cb) -} - -function startInstance (shortHash, cb) { - var Instance = user.fetchInstance(shortHash, function (err) { - if (err) { - ctx.retry.push(shortHash) - ERRORS.push({ - func: 'startInstance:fetchInstance', - err: err.message, - shortHash: shortHash - }) - return cb() - } - Instance.start(function (err) { - if (err) { - ctx.retry.push(shortHash) - ERRORS.push({ - func: 'startInstance:start', - err: err.message, - shortHash: shortHash - }) - } else { - ctx.numStarted++ - } - cb() - }) - }) -} - -function retryStart (cb) { - console.error('old errors', ERRORS) - ERRORS = [] - startAllContainers(ctx.retry, cb) -} - -// put back into mavis -function addToMavis (cb) { - console.log('addToMavis') - mavisRequst('PUT', {host: fullUrl}, function () { - mavisRequst('POST', {host: fullUrl, numContainer: ctx.numStarted}, cb) - }) -} - -function mavisRequst (type, data, cb) { - console.log('mavisRequst', type, data) - request({ - method: type, - url: process.env.MAVIS_HOST + '/docks', - qs: data - }, function (err, res) { - if (err || res.statusCode !== 200) { - ERRORS.push({ - func: 'addToMavis:PUT', - err: err && err.message || res.statusCode - }) - } - cb() - }) -} - -function restore (cb) { - async.waterfall([ - getAllContainers, - startAllContainers, - retryStart, - addToMavis - ], cb) -} - -// ///////////////////// -// / helpers - -function waitForYes (cb) { - console.log('curl -sSL https://get.docker.com/ubuntu/ | sudo sh') - console.log('update docker now, type yes to contine') - process.stdin.on('data', function read (chunk) { - if (~chunk.toString().indexOf('yes')) { - process.stdin.removeListener('data', read) - cb() - } else { - console.log('must type yes to continue') - } - }) -} - -function finish (err) { - console.log('DONE: err?', err) - console.log('all erros encounted', ERRORS) - process.exit() -} - -// program -async.series([ - saveAndKill, - waitForYes, - restore -], finish) diff --git a/test/functional/fixtures/api-control.js b/test/functional/fixtures/api-control.js index 98898a258..1f4917ee6 100644 --- a/test/functional/fixtures/api-control.js +++ b/test/functional/fixtures/api-control.js @@ -36,7 +36,8 @@ function ensureIndexes (cb) { // this create exchanges that is used by api var publishedEvents = [ 'container.network.attached', - 'container.network.attach-failed' + 'container.network.attach-failed', + 'dock.removed' ] var opts = { @@ -44,7 +45,7 @@ var opts = { password: process.env.RABBITMQ_PASSWORD, port: process.env.RABBITMQ_PORT, username: process.env.RABBITMQ_USERNAME, - name: '10.12.13.11.sauron' + name: 'mavis-sauron' } var rabbitPublisher = new Hermes(put({ publishedEvents: publishedEvents diff --git a/test/functional/fixtures/dock.js b/test/functional/fixtures/dock.js index 470de31c5..5b4f80ebc 100644 --- a/test/functional/fixtures/dock.js +++ b/test/functional/fixtures/dock.js @@ -1,9 +1,9 @@ + var async = require('async') var dockerModel = require('models/apis/docker') var createCount = require('callback-count') var docker = require('./docker') var redis = require('models/redis') -var mavisApp = require('mavis') var dockerModuleMock = require('./mocks/docker-model') var sinon = require('sinon') @@ -11,13 +11,12 @@ process.env.AUTO_RECONNECT = false // needed for test process.env.HOST_TAGS = 'default' // needed for test var dockerListener = require('docker-listener') -var url = require('url') var put = require('101/put') var Hermes = require('runnable-hermes') // Sauron mock listens for `container.life-cycle.started` event and -// publsihes `container.network.attached` +// publishes `container.network.attached` var sauronMock = { start: function (cb) { var publishedEvents = [ @@ -92,13 +91,9 @@ function startDock (done) { ctx.docker = docker.start(function (err) { if (err) { return count.next(err) } - ctx.mavis = mavisApp.listen(url.parse(process.env.MAVIS_HOST).port) - ctx.mavis.on('listening', function (err) { + dockerListener.start(process.env.DOCKER_LISTENER_PORT, function (err) { if (err) { return count.next(err) } - dockerListener.start(process.env.DOCKER_LISTENER_PORT, function (err) { - if (err) { return count.next(err) } - count.next() - }) + count.next() }) }) } @@ -107,9 +102,8 @@ function stopDock (done) { dockerModel.prototype.pullImage.restore() started = false var count = createCount(4, done) - ctx.mavis.close(count.next) sauronMock.stop(count.next) - redis.del(process.env.REDIS_HOST_KEYS, count.inc().next) + redis.del(process.env.REDIS_HOST_KEYS, count.next) dockerModuleMock.clean(count.next) dockerListener.stop(function (err) { if (err) { return count.next(err) } diff --git a/test/functional/fixtures/mocks/docker/build-logs.js b/test/functional/fixtures/mocks/docker/build-logs.js index 49f37ee0e..b6a626937 100644 --- a/test/functional/fixtures/mocks/docker/build-logs.js +++ b/test/functional/fixtures/mocks/docker/build-logs.js @@ -7,8 +7,7 @@ var createFrame = require('docker-frame') // This mock is currently used with docker container die event // for an image builder container. -// The docker host information is used from the instance document mongo (mavis). -// Mavis get's it's docker host info from docker listener. +// The docker host information is used from the instance document mongo. // Docker listener uses emits data with docker's external host module.exports = function (failure, error) { diff --git a/test/integration/workers/on-image-builder-container-die.js b/test/integration/workers/on-image-builder-container-die.js index 4b7f2d15b..3d47800fe 100644 --- a/test/integration/workers/on-image-builder-container-die.js +++ b/test/integration/workers/on-image-builder-container-die.js @@ -12,7 +12,6 @@ var createCount = require('callback-count') var rabbitMQ = require('models/rabbitmq') var sinon = require('sinon') var Docker = require('models/apis/docker') -var Mavis = require('models/apis/mavis') var dock = require('../../functional/fixtures/dock') var dockerMockEvents = require('../../functional/fixtures/docker-mock-events') var mongooseControl = require('models/mongo/mongoose-control.js') @@ -93,41 +92,37 @@ describe('OnImageBuilderContainerDie Integration Tests', function () { }) function createImageBuilder (err) { if (err) { return done(err) } - var mavis = new Mavis() - mavis.findDockForBuild(ctx.cv, ctx.cv, function (err, dockerHost) { + var docker = new Docker(process.env.SWARM_HOST) + ctx.cv.dockerHost = process.env.SWARM_HOST + var opts = { + manualBuild: true, + sessionUser: ctx.user, + ownerUsername: ctx.user.accounts.github.username, + contextVersion: ctx.cv, + network: { + hostIp: '1.1.1.1' + }, + tid: 1 + } + ctx.cv.populate('infraCodeVersion', function () { if (err) { return done(err) } - var docker = new Docker(dockerHost) - ctx.cv.dockerHost = dockerHost - var opts = { - manualBuild: true, - sessionUser: ctx.user, - ownerUsername: ctx.user.accounts.github.username, - contextVersion: ctx.cv, - network: { - hostIp: '1.1.1.1' - }, - tid: 1 - } - ctx.cv.populate('infraCodeVersion', function () { + ctx.cv.infraCodeVersion = { + context: ctx.cv.context + } // mock + docker.createImageBuilder(opts, function (err, container) { if (err) { return done(err) } - ctx.cv.infraCodeVersion = { - context: ctx.cv.context - } // mock - docker.createImageBuilder(opts, function (err, container) { + ctx.usedDockerContainer = container + ContextVersion.updateBy('_id', ctx.cv._id, { + $set: { + 'build.dockerContainer': container.id, + 'build.dockerTag': Docker.getDockerTag(opts.contextVersion) + } + }, {}, function (err) { if (err) { return done(err) } - ctx.usedDockerContainer = container - ContextVersion.updateBy('_id', ctx.cv._id, { - $set: { - 'build.dockerContainer': container.id, - 'build.dockerTag': Docker.getDockerTag(opts.contextVersion) - } - }, {}, function (err) { + ContextVersion.findById(ctx.cv._id, function (err, cv) { if (err) { return done(err) } - ContextVersion.findById(ctx.cv._id, function (err, cv) { - if (err) { return done(err) } - ctx.cv = cv - done() - }) + ctx.cv = cv + done() }) }) }) diff --git a/unit/models/apis/mavis.js b/unit/models/apis/mavis.js deleted file mode 100644 index a1ce3f37e..000000000 --- a/unit/models/apis/mavis.js +++ /dev/null @@ -1,190 +0,0 @@ -'use strict' - -require('loadenv')() - -var Lab = require('lab') -var lab = exports.lab = Lab.script() -var describe = lab.describe -var it = lab.it -var afterEach = lab.afterEach -var beforeEach = lab.beforeEach -var Code = require('code') -var expect = Code.expect - -var Mavis = require('models/apis/mavis') -var sinon = require('sinon') - -var path = require('path') -var moduleName = path.relative(process.cwd(), __filename) - -describe('mavis.js unit test: ' + moduleName, function () { - var ctx = {} - beforeEach(function (done) { - ctx.mavis = new Mavis() - done() - }) - describe('findDockForBuild', function () { - it('should error if missing contextVersion', function (done) { - ctx.mavis.findDockForBuild(null, null, function (err) { - expect(err).to.exist() - done() - }) - }) - it('should error if missing context', function (done) { - ctx.mavis.findDockForBuild({}, null, function (err) { - expect(err).to.exist() - done() - }) - }) - describe('should send correct inputs to findDock', function () { - var testOrgId = 23894759 - var testContext = { - owner: { github: testOrgId } - } - beforeEach(function (done) { - sinon.stub(ctx.mavis, 'findDock').yieldsAsync() - done() - }) - afterEach(function (done) { - ctx.mavis.findDock.restore() - done() - }) - it('no duration, no dockerTag', function (done) { - ctx.mavis.findDockForBuild({}, testContext, function () { - expect(ctx.mavis.findDock.calledWith({ - type: 'container_build', - tags: testOrgId + ',build', - prevDuration: 0, - prevImage: null - })).to.be.true() - done() - }) - }) - it('w/duration, w/dockerTag', function (done) { - var testDur = 1283956 - var testTag = 'runnatag' - ctx.mavis.findDockForBuild({ - duration: testDur, - dockerTag: testTag - }, testContext, function () { - expect(ctx.mavis.findDock.calledWith({ - type: 'container_build', - tags: testOrgId + ',build', - prevDuration: testDur, - prevImage: testTag - })).to.be.true() - done() - }) - }) - }) - }) - describe('findDockForContainer', function () { - it('should error if missing contextVersion', function (done) { - ctx.mavis.findDockForContainer(null, function (err) { - expect(err).to.exist() - done() - }) - }) - it('should error if missing contextVersion owner', function (done) { - ctx.mavis.findDockForContainer({}, function (err) { - expect(err).to.exist() - done() - }) - }) - describe('should send correct inputs to findDock', function () { - var testOrgId = 23894759 - var testContextVersion - beforeEach(function (done) { - testContextVersion = { - owner: { github: testOrgId } - } - sinon.stub(ctx.mavis, 'findDock').yieldsAsync() - done() - }) - afterEach(function (done) { - ctx.mavis.findDock.restore() - done() - }) - it('no dockerHost', function (done) { - ctx.mavis.findDockForContainer(testContextVersion, function () { - expect(ctx.mavis.findDock.calledWith({ - type: 'container_run', - tags: testOrgId + ',run', - prevDock: null - })).to.be.true() - done() - }) - }) - it('w/dockerHost', function (done) { - var testHost = 'godaddy' - testContextVersion.dockerHost = testHost - ctx.mavis.findDockForContainer(testContextVersion, function () { - expect(ctx.mavis.findDock.calledWith({ - type: 'container_run', - tags: testOrgId + ',run', - prevDock: testHost - })).to.be.true() - done() - }) - }) - }) - }) - describe('findDock', function () { - beforeEach(function (done) { - sinon.stub(ctx.mavis, 'post') - done() - }) - afterEach(function (done) { - ctx.mavis.post.restore() - done() - }) - it('should boom 504 if error making request', function (done) { - var testErr = 'Mugetsu' - ctx.mavis.post.yieldsAsync(testErr) - ctx.mavis.findDock({}, function (err) { - expect(err.output.statusCode).to.equal(504) - done() - }) - }) - it('should retry with default if statusCode 503 and not default', function (done) { - var testHost = 'ipage' - ctx.mavis.post.onFirstCall().yieldsAsync(null, { - statusCode: 503 - }) - ctx.mavis.post.onSecondCall().yieldsAsync(null, { - statusCode: 200, - body: { - dockHost: testHost - } - }) - ctx.mavis.findDock({tags: '2398457'}, function (err, host) { - expect(err).to.not.exist() - expect(host).to.equal(testHost) - done() - }) - }) - it('should boom error if statusCode > 300', function (done) { - ctx.mavis.post.yieldsAsync(null, { - statusCode: 401, - request: {uri: 'some test'}, - body: 'some body' - }) - ctx.mavis.findDock({}, function (err) { - expect(err.output.statusCode).to.equal(401) - done() - }) - }) - it('should cb host if 200', function (done) { - var testHost = 'web.com' - ctx.mavis.post.yieldsAsync(null, { - statusCode: 200, - body: {dockHost: testHost} - }) - ctx.mavis.findDock({}, function (err, host) { - expect(err).to.not.exist() - expect(host).to.equal(testHost) - done() - }) - }) - }) -}) diff --git a/unit/workers/on-dock-removed.js b/unit/workers/dock.removed.js similarity index 98% rename from unit/workers/on-dock-removed.js rename to unit/workers/dock.removed.js index c1ebffbe8..e64836feb 100644 --- a/unit/workers/on-dock-removed.js +++ b/unit/workers/dock.removed.js @@ -17,13 +17,13 @@ var sinon = require('sinon') var Instance = require('models/mongo/instance') var InstanceService = require('models/services/instance-service') var ContextVersion = require('models/mongo/context-version') -var Worker = require('workers/on-dock-removed') +var Worker = require('workers/dock.removed') var TaskFatalError = require('ponos').TaskFatalError var path = require('path') var moduleName = path.relative(process.cwd(), __filename) -describe('Worker: on-dock-removed unit test: ' + moduleName, function () { +describe('Worker: dock.removed unit test: ' + moduleName, function () { var testHost = 'goku' var testData = { host: testHost diff --git a/unit/workers/pull-instance-image.js b/unit/workers/pull-instance-image.js index 0204bc7e0..77b985ba5 100644 --- a/unit/workers/pull-instance-image.js +++ b/unit/workers/pull-instance-image.js @@ -11,7 +11,6 @@ var TaskFatalError = require('ponos').TaskFatalError var Docker = require('models/apis/docker') var Instance = require('models/mongo/instance') -var Mavis = require('models/apis/mavis') var PullInstanceImageWorker = require('workers/pull-instance-image') var rabbitMQ = require('models/rabbitmq') var toObjectId = require('utils/to-object-id') @@ -52,7 +51,6 @@ describe('pullInstanceImageWorker: ' + moduleName, function () { } ctx.mockInstance = newMockInstance(ctx.job) sinon.stub(Instance, 'findOneAsync') - sinon.stub(Mavis.prototype, 'findDockForContainerAsync') sinon.stub(Instance.prototype, 'modifyImagePullAsync') sinon.stub(Docker.prototype, 'pullImageAsync') sinon.stub(Instance.prototype, 'modifyUnsetImagePullAsync') @@ -62,7 +60,6 @@ describe('pullInstanceImageWorker: ' + moduleName, function () { afterEach(function (done) { Instance.findOneAsync.restore() Instance.prototype.modifyImagePullAsync.restore() - Mavis.prototype.findDockForContainerAsync.restore() Docker.prototype.pullImageAsync.restore() Instance.prototype.modifyUnsetImagePullAsync.restore() rabbitMQ.createInstanceContainer.restore() @@ -71,13 +68,10 @@ describe('pullInstanceImageWorker: ' + moduleName, function () { describe('success', function () { beforeEach(function (done) { - ctx.dockerHost = 'http://localhost:4243' Instance.findOneAsync .returns(Promise.resolve(ctx.mockInstance)) Instance.prototype.modifyImagePullAsync .returns(Promise.resolve(ctx.mockInstance)) - Mavis.prototype.findDockForContainerAsync - .returns(Promise.resolve(ctx.dockerHost)) Docker.prototype.pullImageAsync .returns(Promise.resolve()) Instance.prototype.modifyUnsetImagePullAsync @@ -94,15 +88,11 @@ describe('pullInstanceImageWorker: ' + moduleName, function () { _id: toObjectId(ctx.job.instanceId), build: toObjectId(ctx.job.buildId) }) - sinon.assert.calledWith( - Mavis.prototype.findDockForContainerAsync, - ctx.mockInstance.contextVersion - ) sinon.assert.calledWith( Instance.prototype.modifyImagePullAsync, ctx.mockInstance.contextVersion._id, { dockerTag: ctx.mockInstance.contextVersion.build.dockerTag, - dockerHost: ctx.dockerHost, + dockerHost: process.env.SWARM_HOST, sessionUser: { github: ctx.job.sessionUserGithubId }, @@ -133,7 +123,6 @@ describe('pullInstanceImageWorker: ' + moduleName, function () { describe('db state changed', function () { describe('instance.findOneAsync instance not found', function () { beforeEach(function (done) { - ctx.dockerHost = 'http://localhost:4243' Instance.findOneAsync .returns(Promise.resolve(null)) done() @@ -171,13 +160,10 @@ describe('pullInstanceImageWorker: ' + moduleName, function () { }) describe('instance.modifyUnsetImagePullAsync instance not found', function () { beforeEach(function (done) { - ctx.dockerHost = 'http://localhost:4243' Instance.findOneAsync .returns(Promise.resolve(ctx.mockInstance)) Instance.prototype.modifyImagePullAsync .returns(Promise.resolve(ctx.mockInstance)) - Mavis.prototype.findDockForContainerAsync - .returns(Promise.resolve(ctx.dockerHost)) Docker.prototype.pullImageAsync .returns(Promise.resolve()) Instance.prototype.modifyUnsetImagePullAsync @@ -223,32 +209,13 @@ describe('pullInstanceImageWorker: ' + moduleName, function () { PullInstanceImageWorker(ctx.job).asCallback(expectErr(ctx.err, done)) }) }) - describe('mavis.findDockForContainerAsync err', function () { - beforeEach(function (done) { - ctx.err = new Error() - Instance.findOneAsync - .returns(Promise.resolve(ctx.mockInstance)) - Instance.prototype.modifyImagePullAsync - .returns(Promise.resolve(ctx.mockInstance)) - Mavis.prototype.findDockForContainerAsync - .throws(ctx.err) - done() - }) - - it('should throw the err', function (done) { - PullInstanceImageWorker(ctx.job).asCallback(expectErr(ctx.err, done)) - }) - }) describe('docker.pullImageAsync err', function () { beforeEach(function (done) { ctx.err = new Error() - ctx.dockerHost = 'http://localhost:4243' Instance.findOneAsync .returns(Promise.resolve(ctx.mockInstance)) Instance.prototype.modifyImagePullAsync .returns(Promise.resolve(ctx.mockInstance)) - Mavis.prototype.findDockForContainerAsync - .returns(Promise.resolve(ctx.dockerHost)) Docker.prototype.pullImageAsync .throws(ctx.err) done() @@ -263,13 +230,10 @@ describe('pullInstanceImageWorker: ' + moduleName, function () { ctx.err = Boom.notFound('Follow pull image failed: image dockerTag: not found', { err: 'image dockerTag: not found' // err is a string }) - ctx.dockerHost = 'http://localhost:4243' Instance.findOneAsync .returns(Promise.resolve(ctx.mockInstance)) Instance.prototype.modifyImagePullAsync .returns(Promise.resolve(ctx.mockInstance)) - Mavis.prototype.findDockForContainerAsync - .returns(Promise.resolve(ctx.dockerHost)) Docker.prototype.pullImageAsync .throws(ctx.err) done()