Permalink
Browse files

release v2.2.0

  • Loading branch information...
apocas committed Aug 7, 2014
1 parent aa43f8b commit e4c9ebc7ee254700ab90563aafc85e20e9cb6aeb
Showing with 114 additions and 46 deletions.
  1. +5 −1 README.md
  2. +30 −22 lib/outkept.js
  3. +1 −2 lib/server.js
  4. +40 −0 loader.js
  5. +37 −20 main.js
  6. +1 −1 package.json
@@ -22,7 +22,11 @@
### Processes
* `Outkept` uses multiples processes, each subnet is crawled in a separated process, a process for server monitoring and a main controller process.
#### `Outkept` uses multiples processes.
* Each subnet is crawled in a separated process.
* Each 50 servers are managed and monitored by a process.
* A main controller process.
### SSH Connections
@@ -4,11 +4,13 @@ var Server = require('./server'),
outils = require('./utils'),
async = require('async');
var Outkept = function (passphrase, key) {
var Outkept = function (id, passphrase, key, ids) {
this.servers = [];
this.queue = [];
this.passphrase = passphrase;
this.key = key;
this.ids = ids;
this.id = id;
this.loadGardeners();
};
@@ -22,19 +24,19 @@ Outkept.prototype.loadGardeners = function () {
if (server.connected === false && server.started) {
server.conf.privateKey = self.key;
server.conf.passphrase = self.passphrase;
console.log(server.conf.host + ' retrying...');
console.log(self.id + ' - ' + server.conf.host + ' retrying...');
server.connect();
} else if(server.connected === true) {
vendors.mongo(function(db) {
var aux = server.extract();
aux.time = new Date().getTime() / 1000;
db.collection('readings').insert(aux, function(err, docs) {
if(err) console.log('Mongo error in outkept->save');
if(err) console.log(self.id + ' - Mongo error in outkept->save');
});
});
}
});
}, 2 * 60 * 1000);
}, 120000);
};
@@ -60,17 +62,17 @@ Outkept.prototype.createServer = function (hostname, sshport) {
});
s.on('alarmed', function (sensor) {
console.log(this.hostname + ' ALARM!');
console.log(self.id + ' - ' + this.hostname + ' ALARM!');
outils.sendTrigger('alarmed', sensor.conf.name, sensor.value, s);
});
s.on('warned', function (sensor) {
console.log(this.hostname + ' WARNING!');
console.log(self.id + ' - ' + this.hostname + ' WARNING!');
outils.sendTrigger('warned', sensor.conf.name, sensor.value, s);
});
s.on('fired', function (sensor) {
console.log(this.conf.host + ' sensor ' + sensor.conf.name + ' fired with value ' + sensor.value);
console.log(self.id + ' - ' + this.conf.host + ' sensor ' + sensor.conf.name + ' fired with value ' + sensor.value);
outils.sendTrigger('fired', sensor.conf.name, sensor.value, s);
});
@@ -85,17 +87,16 @@ Outkept.prototype.loadServers = function() {
var self = this;
vendors.mongo(function(db) {
db.collection('servers').find().toArray(function(err, replies) {
async.mapLimit(replies, 3, function(server, callback) {
if(server.address !== undefined && self.findServer(server.id) === undefined) {
console.log('Loading server: ' + server.id);
async.mapLimit(self.ids, 5, function(id, callback) {
db.collection('servers').findOne({id: id}, function(err, server) {
if(server && server.address !== undefined && self.findServer(server.id) === undefined) {
console.log(self.id + ' - Loading server: ' + server.id);
var s = self.createServer(server.address, server.port);
s.id = server.id;
s.hostname = server.hostname;
s.once('loaded', function() {
console.log('Server loaded: ' + this.id);
console.log(self.id + ' - Server loaded: ' + this.id);
self.servers.push(s);
callback();
});
@@ -105,20 +106,20 @@ Outkept.prototype.loadServers = function() {
});
s.once('offline', function(err) {
console.log(err);
console.log(self.id + ' - ' + this.id + ' - ' + err);
});
s.connect();
} else {
callback();
}
}, function(err, results) {
console.log('Servers loaded!');
setTimeout(function() {
self.loadServers();
}, 30000);
});
}, function(err, results) {
//console.log(self.id + ' - Servers loaded!');
setTimeout(function() {
self.loadServers();
}, 30000);
});
});
};
@@ -131,7 +132,14 @@ Outkept.prototype.findServer = function (id, cb) {
}
};
var main;
process.on('message', function(m) {
console.log('Starting...');
new Outkept(m.boot, m.key).start();
if(m.boot) {
console.log(m.id + ' - Starting...');
main = new Outkept(m.id, m.boot, m.key, m.ids);
main.start();
} else if(m.server) {
main.ids.push(m.server);
}
});
@@ -121,7 +121,7 @@ Server.prototype.loadSensors = function () {
aux.once('available', function () {
this.start();
console.log(self.id + ' Sensor loaded: ' + template.name);
//console.log(self.id + ' Sensor loaded: ' + template.name);
if(called === false) {
callback();
called = true;
@@ -156,7 +156,6 @@ Server.prototype.loadSensors = function () {
self.save();
this.latest = now;
}
vendors.mongopubsub.publish('updates', self);
self.emit('data', this);
});
@@ -0,0 +1,40 @@
var cp = require('child_process');
var Loader = function(id, passphrase, key, ids) {
this.id = id;
this.passphrase = passphrase;
this.key = key;
this.ids = ids;
};
Loader.prototype.load = function() {
var self = this;
this.proc = cp.fork('lib/outkept.js');
this.proc.send({'boot': this.passphrase, 'key': this.key, 'ids': this.ids, 'id': this.id});
this.proc.on('exit', function (code) {
console.log(self.id + ' - Loader has exited ' + code);
/*
//TODO: each loader has it's servers to mark as offline
vendors.mongo(function(db) {
db.collection('servers').update({}, {$set: {connected: false}}, { multi: true }, function() {
self.load();
});
});
*/
self.load();
});
};
Loader.prototype.loadServer = function(id) {
this.ids.push(id);
this.proc.send({'server': id});
};
Loader.prototype.kill = function() {
this.proc.removeAllListeners('exit');
this.proc.kill('SIGHUP');
};
module.exports = Loader;
57 main.js
@@ -1,16 +1,17 @@
var prompt = require('prompt'),
fs = require('fs'),
outils = require('./lib/utils'),
cp = require('child_process'),
vendors = require('./vendors'),
Runner = require('./lib/crawler/runner'),
config = require('./conf/config'),
Workers = require('./workers'),
Feeds = require('./lib/feeds/feeds');
Feeds = require('./lib/feeds/feeds'),
Loader = require('./loader');
var loaderMain,
var loadersMain = [],
workersLoader,
loaded = [],
runnerCrawlers,
shutting = false,
schema = {
@@ -23,34 +24,29 @@ var loaderMain,
process.on('exit', function() {
console.log('Killing everything...');
loaderMain.removeAllListeners('exit');
loaderMain.kill('SIGHUP');
for (var i = 0; i < loadersMain.length; i++) {
loadersMain[i].kill();
}
workersLoader.kill('SIGHUP');
runnerCrawlers.kill('SIGHUP');
});
prompt.start();
function loadMain(passphrase, key) {
loaderMain = cp.fork('lib/outkept.js');
loaderMain.send({ 'boot': passphrase, 'key': key});
loaderMain.on('exit', function (code) {
console.log('Main loader has exited ' + code);
vendors.mongo(function(db) {
db.collection('servers').update({}, {$set: {connected: false}}, { multi: true }, function() {
loadMain(passphrase, key);
});
});
});
function loadMain(passphrase, key, ids) {
var l = new Loader(loadersMain.length, passphrase, key, ids);
l.load();
loadersMain.push(l);
return l;
}
function loadFeeds() {
var feeds = new Feeds();
feeds.on('alert', function (feed, data) {
//console.log(feed.template.name + ' reported ' + data);
console.log('Feed ' + feed.template.name + ' reported ' + data);
outils.sendFeed(feed.template.name, data);
});
}
@@ -62,7 +58,28 @@ prompt.get(schema, function (err, result) {
var key = fs.readFileSync(config.crawler_key).toString('utf-8');
outils.secureDelete(config.crawler_key);
loadMain(passphrase, key);
var l = loadMain(passphrase, key, []);
vendors.mongo(function(db) {
setInterval(function() {
db.collection('servers').find().toArray(function(err, replies) {
var ids = [];
for (var i = 0; i < replies.length; i++) {
if(loaded.indexOf(replies[i].id) === -1) {
if(l.ids.length < 50) {
l.loadServer(replies[i].id);
loaded.push(replies[i].id);
} else {
l = loadMain(passphrase, key, []);
l.loadServer(replies[i].id);
loaded.push(replies[i].id);
}
}
}
});
}, 30000);
});
runnerCrawlers = new Runner();
runnerCrawlers.start(passphrase, key);
@@ -1,6 +1,6 @@
{
"name": "outkept",
"version": "2.0.0",
"version": "2.2.0",
"dependencies": {
"ssh2": "*",
"prompt": "*",

0 comments on commit e4c9ebc

Please sign in to comment.