Skip to content
This repository has been archived by the owner on Jan 7, 2018. It is now read-only.

Commit

Permalink
Reload and rewrite templates during process reload.
Browse files Browse the repository at this point in the history
This fixes templates not being rewritten during deploys, so template
changes were not being reflected.

See 18F/api.data.gov#161
  • Loading branch information
GUI committed Jan 26, 2015
1 parent a8424cf commit 963320e
Show file tree
Hide file tree
Showing 3 changed files with 168 additions and 147 deletions.
141 changes: 72 additions & 69 deletions lib/reload.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,80 +6,83 @@ module.exports = function(options, reloadCallback) {
async = require('async'),
execFile = require('child_process').execFile,
logger = require('./logger'),
processEnv = require('./process_env');
processEnv = require('./process_env'),
writeTemplates = require('./write_templates');

var configPath = processEnv.supervisordConfigPath();
var execOpts = {
env: processEnv.env(),
};
writeTemplates(function() {
var configPath = processEnv.supervisordConfigPath();
var execOpts = {
env: processEnv.env(),
};

logger.info('Begin reloading api-umbrella...');
execFile('supervisorctl', ['-c', configPath, 'update'], execOpts, function(error, stdout, stderr) {
if(error) {
logger.error('supervisorctl update error: ' + error.message + '\n\nSTDOUT: ' + stdout + '\n\nSTDERR:' + stderr);
return false;
}
logger.info('Begin reloading api-umbrella...');
execFile('supervisorctl', ['-c', configPath, 'update'], execOpts, function(error, stdout, stderr) {
if(error) {
logger.error('supervisorctl update error: ' + error.message + '\n\nSTDOUT: ' + stdout + '\n\nSTDERR:' + stderr);
return false;
}

var tasks = [];
var tasks = [];

if((_.isEmpty(options) || options.router) && config.get('service_router_enabled')) {
tasks = tasks.concat([
function(callback) {
logger.info('Reloading router-nginx...');
execFile('supervisorctl', ['-c', configPath, 'kill', 'HUP', 'router-nginx'], execOpts, callback);
},
function(callback) {
logger.info('Reloading gatekeeper...');
execFile('supervisorctl', ['-c', configPath, 'serialrestart', 'gatekeeper:*'], execOpts, callback);
},
function(callback) {
logger.info('Reloading config-reloader...');
execFile('supervisorctl', ['-c', configPath, 'restart', 'config-reloader'], execOpts, callback);
},
function(callback) {
logger.info('Reloading distributed-rate-limits-sync...');
execFile('supervisorctl', ['-c', configPath, 'restart', 'distributed-rate-limits-sync'], execOpts, callback);
},
function(callback) {
logger.info('Reloading log-processor...');
execFile('supervisorctl', ['-c', configPath, 'restart', 'log-processor'], execOpts, callback);
},
function(callback) {
logger.info('Reloading router-log-listener...');
execFile('supervisorctl', ['-c', configPath, 'restart', 'router-log-listener'], execOpts, callback);
},
]);
}
if((_.isEmpty(options) || options.router) && config.get('service_router_enabled')) {
tasks = tasks.concat([
function(callback) {
logger.info('Reloading router-nginx...');
execFile('supervisorctl', ['-c', configPath, 'kill', 'HUP', 'router-nginx'], execOpts, callback);
},
function(callback) {
logger.info('Reloading gatekeeper...');
execFile('supervisorctl', ['-c', configPath, 'serialrestart', 'gatekeeper:*'], execOpts, callback);
},
function(callback) {
logger.info('Reloading config-reloader...');
execFile('supervisorctl', ['-c', configPath, 'restart', 'config-reloader'], execOpts, callback);
},
function(callback) {
logger.info('Reloading distributed-rate-limits-sync...');
execFile('supervisorctl', ['-c', configPath, 'restart', 'distributed-rate-limits-sync'], execOpts, callback);
},
function(callback) {
logger.info('Reloading log-processor...');
execFile('supervisorctl', ['-c', configPath, 'restart', 'log-processor'], execOpts, callback);
},
function(callback) {
logger.info('Reloading router-log-listener...');
execFile('supervisorctl', ['-c', configPath, 'restart', 'router-log-listener'], execOpts, callback);
},
]);
}

if((_.isEmpty(options) || options.web) && config.get('service_web_enabled')) {
tasks = tasks.concat([
function(callback) {
logger.info('Reloading web-nginx...');
execFile('supervisorctl', ['-c', configPath, 'kill', 'HUP', 'web-nginx'], execOpts, callback);
},
function(callback) {
logger.info('Reloading web-puma...');
execFile('supervisorctl', ['-c', configPath, 'kill', 'USR2', 'web-puma'], execOpts, callback);
},
function(callback) {
logger.info('Reloading web-delayed-job...');
execFile('supervisorctl', ['-c', configPath, 'restart', 'web-delayed-job'], execOpts, callback);
},
]);
}
if((_.isEmpty(options) || options.web) && config.get('service_web_enabled')) {
tasks = tasks.concat([
function(callback) {
logger.info('Reloading web-nginx...');
execFile('supervisorctl', ['-c', configPath, 'kill', 'HUP', 'web-nginx'], execOpts, callback);
},
function(callback) {
logger.info('Reloading web-puma...');
execFile('supervisorctl', ['-c', configPath, 'kill', 'USR2', 'web-puma'], execOpts, callback);
},
function(callback) {
logger.info('Reloading web-delayed-job...');
execFile('supervisorctl', ['-c', configPath, 'restart', 'web-delayed-job'], execOpts, callback);
},
]);
}

if(tasks.length > 0) {
async.parallel(tasks, function(error) {
if(error) {
logger.error('Error reloading api-umbrella: ', error);
reloadCallback(error);
} else {
logger.info('Finished reloading api-umbrella');
reloadCallback();
}
});
} else {
reloadCallback();
}
if(tasks.length > 0) {
async.parallel(tasks, function(error) {
if(error) {
logger.error('Error reloading api-umbrella: ', error);
reloadCallback(error);
} else {
logger.info('Finished reloading api-umbrella');
reloadCallback();
}
});
} else {
reloadCallback();
}
}.bind(this));
}.bind(this));
};
80 changes: 2 additions & 78 deletions lib/router.js
Original file line number Diff line number Diff line change
Expand Up @@ -666,84 +666,8 @@ _.extend(Router.prototype, {
},

writeTemplates: function(callback) {
var gatekeeperHosts = _.times(config.get('gatekeeper.workers'), function(n) {
var port = parseInt(config.get('gatekeeper.starting_port'), 10) + n;
return {
port: port,
host: config.get('gatekeeper.host') + ':' + port,
process_name: 'gatekeeper' + (n + 1),
};
});

var templateConfig = _.extend({}, config.getAll(), {
api_umbrella_config_runtime_file: this.configLoader.runtimeFile,
api_umbrella_config_args: '--config ' + this.configLoader.runtimeFile,
gatekeeper_hosts: gatekeeperHosts,
gatekeeper_supervisor_process_names: _.pluck(gatekeeperHosts, 'process_name'),
test_env: (config.get('app_env') === 'test'),
development_env: (config.get('app_env') === 'development'),
primary_hosts: _.filter(config.get('hosts'), function(host) { return !host.secondary; }),
secondary_hosts: _.filter(config.get('hosts'), function(host) { return host.secondary; }),
has_default_host: (_.where(config.get('hosts'), { default: true }).length > 0),
supervisor_conditional_user: (config.get('user')) ? 'user=' + config.get('user') : '',
mongodb_yaml: yaml.safeDump(_.merge({
systemLog: {
path: path.join(config.get('log_dir'), 'mongod.log'),
},
storage: {
dbPath: path.join(config.get('db_dir'), 'mongodb'),
},
}, config.get('mongodb.embedded_server_config'))),
elasticsearch_yaml: yaml.safeDump(_.merge({
path: {
conf: path.join(config.get('etc_dir'), 'elasticsearch'),
data: path.join(config.get('db_dir'), 'elasticsearch'),
logs: path.join(config.get('log_dir')),
},
}, config.get('elasticsearch.embedded_server_config'))),
});

var templateRoot = path.resolve(__dirname, '../templates/etc');
glob(path.join(templateRoot, '**/*'), function(error, templatePaths) {
async.each(templatePaths, function(templatePath, eachCallback) {
if(fs.statSync(templatePath).isDirectory()) { return eachCallback(); }

var installPath = templatePath.replace(/\.hbs$/, '');
installPath = installPath.replace(templateRoot, '');
installPath = path.join(config.get('etc_dir'), installPath);

var content = '';

// For the api_backends template, write an empty file, since we don't
// have the necessary API backend information yet. This template gets
// managed by the config_reloader worker process after things are
// started.
if(!_.contains(installPath, 'nginx/api_backends.conf')) {
content = fs.readFileSync(templatePath).toString();
if(/\.hbs$/.test(templatePath)) {
var template = handlebars.compile(content);
content = template(templateConfig);
}
}

mkdirp.sync(path.dirname(installPath));
fs.writeFileSync(installPath, content);

// Since the api_backends file gets written by the separate
// config-reloader process, make sure it's writable if that process is
// running as the less-privileged user.
if(_.contains(installPath, 'nginx/api_backends.conf')) {
if(config.get('user')) {
var uid = posix.getpwnam(config.get('user')).uid;
var gid = posix.getgrnam(config.get('group')).gid;

fs.chownSync(installPath, uid, gid);
}
}

eachCallback();
}.bind(this), callback);
}.bind(this));
var writeTemplates = require('./write_templates');
writeTemplates(callback);
},

startSupervisor: function(callback) {
Expand Down
94 changes: 94 additions & 0 deletions lib/write_templates.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
'use strict';

module.exports = function(callback) {
var _ = require('lodash'),
async = require('async'),
config = require('api-umbrella-config').global(),
fs = require('fs'),
glob = require('glob'),
handlebars = require('handlebars'),
mkdirp = require('mkdirp'),
path = require('path'),
posix = require('posix'),
yaml = require('js-yaml');

var gatekeeperHosts = _.times(config.get('gatekeeper.workers'), function(n) {
var port = parseInt(config.get('gatekeeper.starting_port'), 10) + n;
return {
port: port,
host: config.get('gatekeeper.host') + ':' + port,
process_name: 'gatekeeper' + (n + 1),
};
});

var templateConfig = _.extend({}, config.getAll(), {
api_umbrella_config_runtime_file: config.path,
api_umbrella_config_args: '--config ' + config.path,
gatekeeper_hosts: gatekeeperHosts,
gatekeeper_supervisor_process_names: _.pluck(gatekeeperHosts, 'process_name'),
test_env: (config.get('app_env') === 'test'),
development_env: (config.get('app_env') === 'development'),
primary_hosts: _.filter(config.get('hosts'), function(host) { return !host.secondary; }),
secondary_hosts: _.filter(config.get('hosts'), function(host) { return host.secondary; }),
has_default_host: (_.where(config.get('hosts'), { default: true }).length > 0),
supervisor_conditional_user: (config.get('user')) ? 'user=' + config.get('user') : '',
mongodb_yaml: yaml.safeDump(_.merge({
systemLog: {
path: path.join(config.get('log_dir'), 'mongod.log'),
},
storage: {
dbPath: path.join(config.get('db_dir'), 'mongodb'),
},
}, config.get('mongodb.embedded_server_config'))),
elasticsearch_yaml: yaml.safeDump(_.merge({
path: {
conf: path.join(config.get('etc_dir'), 'elasticsearch'),
data: path.join(config.get('db_dir'), 'elasticsearch'),
logs: path.join(config.get('log_dir')),
},
}, config.get('elasticsearch.embedded_server_config'))),
});

var templateRoot = path.resolve(__dirname, '../templates/etc');
glob(path.join(templateRoot, '**/*'), function(error, templatePaths) {
async.each(templatePaths, function(templatePath, eachCallback) {
if(fs.statSync(templatePath).isDirectory()) { return eachCallback(); }

var installPath = templatePath.replace(/\.hbs$/, '');
installPath = installPath.replace(templateRoot, '');
installPath = path.join(config.get('etc_dir'), installPath);

mkdirp.sync(path.dirname(installPath));

// For the api_backends template, we don't have the necessary API backend
// information yet, so just make sure it exists and is writable. This
// template gets managed by the config_reloader worker process after
// things are started.
if(_.contains(installPath, 'nginx/api_backends.conf')) {
// Make sure the file exists.
fs.closeSync(fs.openSync(installPath, 'a'));

// Make sure it's writable in case the config-reloader process is
// running as the less-privileged user.
if(config.get('user')) {
var uid = posix.getpwnam(config.get('user')).uid;
var gid = posix.getgrnam(config.get('group')).gid;

fs.chownSync(installPath, uid, gid);
}

// All other templates get parsed and written.
} else {
var content = fs.readFileSync(templatePath).toString();
if(/\.hbs$/.test(templatePath)) {
var template = handlebars.compile(content);
content = template(templateConfig);
}

fs.writeFileSync(installPath, content);
}

eachCallback();
}.bind(this), callback);
}.bind(this));
};

0 comments on commit 963320e

Please sign in to comment.