Navigation Menu

Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Updated stats.js to delete counters #183

Merged
merged 8 commits into from Nov 29, 2012
Merged

Updated stats.js to delete counters #183

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
01da23d
Updated stats.js to delete counters
Oct 25, 2012
e3f7924
Update stats.js
Oct 30, 2012
71df1bd
Update test/graphite_tests.js
Oct 31, 2012
777ea8c
Update stats.js
Nov 1, 2012
1e66b4e
Tests for Delete Counters Config
Nov 13, 2012
24390ee
Updated the example config docs with the new option
Nov 14, 2012
c03f339
Changed stats.js to use delete
Nov 26, 2012
d0901b8
Fixed the tests
Nov 26, 2012
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions exampleConfig.js
View file
Open in desktop
Expand Up @@ -34,6 +34,7 @@ Optional Variables:
interval: how often to log frequent keys [ms, default: 0]
percent: percentage of frequent keys to log [%, default: 100]
log: location of log file for frequent keys [default: STDOUT]
deleteCounters: when flushing to graphite, send null instead of 0 [default: false]

console:
prettyprint: whether to prettyprint the console backend
Expand Down
11 changes: 10 additions & 1 deletion stats.js
View file
Open in desktop
Expand Up @@ -36,6 +36,9 @@ function loadBackend(config, name) {
}
};

// global for conf
var conf;

// Flush metrics to each backend.
function flushMetrics() {
var time_stamp = Math.round(new Date().getTime() / 1000);
Expand All @@ -51,8 +54,13 @@ function flushMetrics() {
// After all listeners, reset the stats
backendEvents.once('flush', function clear_metrics(ts, metrics) {
// Clear the counters
conf.deleteCounters = conf.deleteCounters || false;
for (key in metrics.counters) {
metrics.counters[key] = 0;
if (conf.deleteCounters) {
delete(metrics.counters[key]);
} else {
metrics.counters[key] = 0;
}
}

// Clear the timers
Expand Down Expand Up @@ -81,6 +89,7 @@ var stats = {
var l;

config.configFile(process.argv[2], function (config, oldConfig) {
conf = config;
if (! config.debug && debugInt) {
clearInterval(debugInt);
debugInt = false;
Expand Down
263 changes: 263 additions & 0 deletions test/graphite_delete_counters_tests.js
View file
Open in desktop
@@ -0,0 +1,263 @@
var fs = require('fs'),
net = require('net'),
temp = require('temp'),
spawn = require('child_process').spawn,
util = require('util'),
urlparse = require('url').parse,
_ = require('underscore'),
dgram = require('dgram'),
qsparse = require('querystring').parse,
http = require('http');


var writeconfig = function(text,worker,cb,obj){
temp.open({suffix: '-statsdconf.js'}, function(err, info) {
if (err) throw err;
fs.writeSync(info.fd, text);
fs.close(info.fd, function(err) {
if (err) throw err;
worker(info.path,cb,obj);
});
});
}

var array_contents_are_equal = function(first,second){
var intlen = _.intersection(first,second).length;
var unlen = _.union(first,second).length;
return (intlen == unlen) && (intlen == first.length);
}

var statsd_send = function(data,sock,host,port,cb){
send_data = new Buffer(data);
sock.send(send_data,0,send_data.length,port,host,function(err,bytes){
if (err) {
throw err;
}
cb();
});
}

// keep collecting data until a specified timeout period has elapsed
// this will let us capture all data chunks so we don't miss one
var collect_for = function(server,timeout,cb){
var received = [];
var in_flight = 0;
var timed_out = false;
var collector = function(req,res){
in_flight += 1;
var body = '';
req.on('data',function(data){ body += data; });
req.on('end',function(){
received = received.concat(body.split("\n"));
in_flight -= 1;
if((in_flight < 1) && timed_out){
server.removeListener('request',collector);
cb(received);
}
});
}

setTimeout(function (){
timed_out = true;
if((in_flight < 1)) {
server.removeListener('connection',collector);
cb(received);
}
},timeout);

server.on('connection',collector);
}

module.exports = {
setUp: function (callback) {
this.testport = 31337;
this.myflush = 200;
var configfile = "{graphService: \"graphite\"\n\
, batch: 200 \n\
, flushInterval: " + this.myflush + " \n\
, percentThreshold: 90\n\
, port: 8125\n\
, dumpMessages: false \n\
, debug: false\n\
, deleteCounters: true\n\
, graphitePort: " + this.testport + "\n\
, graphiteHost: \"127.0.0.1\"}";

this.acceptor = net.createServer();
this.acceptor.listen(this.testport);
this.sock = dgram.createSocket('udp4');

this.server_up = true;
this.ok_to_die = false;
this.exit_callback_callback = process.exit;

writeconfig(configfile,function(path,cb,obj){
obj.path = path;
obj.server = spawn('node',['stats.js', path]);
obj.exit_callback = function (code) {
obj.server_up = false;
if(!obj.ok_to_die){
console.log('node server unexpectedly quit with code: ' + code);
process.exit(1);
}
else {
obj.exit_callback_callback();
}
};
obj.server.on('exit', obj.exit_callback);
obj.server.stderr.on('data', function (data) {
console.log('stderr: ' + data.toString().replace(/\n$/,''));
});
/*
obj.server.stdout.on('data', function (data) {
console.log('stdout: ' + data.toString().replace(/\n$/,''));
});
*/
obj.server.stdout.on('data', function (data) {
// wait until server is up before we finish setUp
if (data.toString().match(/server is up/)) {
cb();
}
});

},callback,this);
},
tearDown: function (callback) {
this.sock.close();
this.acceptor.close();
this.ok_to_die = true;
if(this.server_up){
this.exit_callback_callback = callback;
this.server.kill();
} else {
callback();
}
},

send_well_formed_posts: function (test) {
test.expect(2);

// we should integrate a timeout into this
this.acceptor.once('connection',function(c){
var body = '';
c.on('data',function(d){ body += d; });
c.on('end',function(){
var rows = body.split("\n");
var entries = _.map(rows, function(x) {
var chunks = x.split(' ');
var data = {};
data[chunks[0]] = chunks[1];
return data;
});
test.ok(_.include(_.map(entries,function(x) { return _.keys(x)[0] }),'statsd.numStats'),'graphite output includes numStats');
test.equal(_.find(entries, function(x) { return _.keys(x)[0] == 'statsd.numStats' })['statsd.numStats'],2);
test.done();
});
});
},

send_malformed_post: function (test) {
test.expect(3);

var testvalue = 1;
var me = this;
this.acceptor.once('connection',function(c){
statsd_send('a_bad_test_value|z',me.sock,'127.0.0.1',8125,function(){
collect_for(me.acceptor,me.myflush*2,function(strings){
test.ok(strings.length > 0,'should receive some data');
var hashes = _.map(strings, function(x) {
var chunks = x.split(' ');
var data = {};
data[chunks[0]] = chunks[1];
return data;
});
var numstat_test = function(post){
var mykey = 'statsd.numStats';
return _.include(_.keys(post),mykey) && (post[mykey] == 2);
};
test.ok(_.any(hashes,numstat_test), 'statsd.numStats should be 0');

var bad_lines_seen_value_test = function(post){
var mykey = 'stats_counts.statsd.bad_lines_seen';
return _.include(_.keys(post),mykey) && isNaN(post[mykey]);
};
test.ok(_.any(hashes,bad_lines_seen_value_test), 'stats_counts.statsd.bad_lines_seen should be ' + testvalue);

test.done();
});
});
});
},

timers_are_valid: function (test) {
test.expect(3);

var testvalue = 100;
var me = this;
this.acceptor.once('connection',function(c){
statsd_send('a_test_value:' + testvalue + '|ms',me.sock,'127.0.0.1',8125,function(){
collect_for(me.acceptor,me.myflush*2,function(strings){
test.ok(strings.length > 0,'should receive some data');
var hashes = _.map(strings, function(x) {
var chunks = x.split(' ');
var data = {};
data[chunks[0]] = chunks[1];
return data;
});
var numstat_test = function(post){
var mykey = 'statsd.numStats';
return _.include(_.keys(post),mykey) && (post[mykey] == 2);
};
test.ok(_.any(hashes,numstat_test), 'statsd.numStats should be 1');

var testtimervalue_test = function(post){
var mykey = 'stats.timers.a_test_value.mean_90';
return _.include(_.keys(post),mykey) && (post[mykey] == testvalue);
};
test.ok(_.any(hashes,testtimervalue_test), 'stats.timers.a_test_value.mean should be ' + testvalue);

test.done();
});
});
});
},

counts_are_valid: function (test) {
test.expect(4);

var testvalue = 100;
var me = this;
this.acceptor.once('connection',function(c){
statsd_send('a_test_value:' + testvalue + '|c',me.sock,'127.0.0.1',8125,function(){
collect_for(me.acceptor,me.myflush*2,function(strings){
test.ok(strings.length > 0,'should receive some data');
var hashes = _.map(strings, function(x) {
var chunks = x.split(' ');
var data = {};
data[chunks[0]] = chunks[1];
return data;
});
var numstat_test = function(post){
var mykey = 'statsd.numStats';
return _.include(_.keys(post),mykey) && (post[mykey] == 2);
};
test.ok(_.any(hashes,numstat_test), 'statsd.numStats should be 1');

var testavgvalue_test = function(post){
var mykey = 'stats.a_test_value';
return _.include(_.keys(post),mykey) && (post[mykey] == (testvalue/(me.myflush / 1000)));
};
test.ok(_.any(hashes,testavgvalue_test), 'stats.a_test_value should be ' + (testvalue/(me.myflush / 1000)));

var testcountvalue_test = function(post){
var mykey = 'stats_counts.a_test_value';
return _.include(_.keys(post),mykey) && (post[mykey] == testvalue);
};
test.ok(_.any(hashes,testcountvalue_test), 'stats_counts.a_test_value should be ' + testvalue);

test.done();
});
});
});
}
}