Permalink
Browse files

Do buffering so we use less connections

  • Loading branch information...
1 parent 8a2d1e7 commit b22528444a95ea69aedafca06460d2ce7c6952ba @fygrave committed Apr 29, 2012
Showing with 16 additions and 8 deletions.
  1. +16 −8 lib/syslog/server_amqp_worker.js
@@ -23,6 +23,9 @@ var packetsStored = [0,0,0,0];
var diskUsage = [0,0];
var lost = 0;
+var messages = [];
+
+
function setup() {
@@ -44,7 +47,7 @@ function set_stats(exchange) {
setInterval(function() {
solrc.commit();
console.log("Commit to indexing");
- },20 *1000);
+ },60 *1000);
setInterval(function() {
// var stats = { packetsReceived: packetsReceived, packetsStored: packetsStored, overhead: packetsReceived - packetsStored, time: getDateInt() };
@@ -162,14 +165,19 @@ function storeMessage(msg, q) {
// //q.shift();
// });
- solrc.add(msg.body, function(err) {
- if (err) {
- console.log(err);
- console.log("solr error saving <<" + JSON.stringify(msg.body) + ">>");
- lost++;
- }
- q.shift();
+ messages.push(msg.body);
+ q.shift();
+ if (messages.length > 100) {
+ var o = messages;
+ messages = []; // race?
+ solrc.add(o, function(err) {
+ if (err) {
+ console.log(err);
+ console.log("solr error saving <<" + JSON.stringify(msg.body) + ">>");
+ lost++;
+ }
});
+ }
db.get('origins', msg.body.origin, function(err, orig, meta) {
if (err) {
if (err.code != "ECONNRESET") {

0 comments on commit b225284

Please sign in to comment.