Skip to content

Commit

Permalink
Adding support for sampling timers.
Browse files Browse the repository at this point in the history
  • Loading branch information
Ian Malpass committed Feb 1, 2013
1 parent 187697c commit cc12534
Show file tree
Hide file tree
Showing 4 changed files with 83 additions and 14 deletions.
5 changes: 4 additions & 1 deletion lib/process_metrics.js
View file
Expand Up @@ -6,6 +6,7 @@ var process_metrics = function (metrics, flushInterval, ts, flushCallback) {
var statsd_metrics = {};
var counters = metrics.counters;
var timers = metrics.timers;
var timer_counters = metrics.timer_counters;
var pctThreshold = metrics.pctThreshold;

for (key in counters) {
Expand Down Expand Up @@ -61,11 +62,13 @@ var process_metrics = function (metrics, flushInterval, ts, flushCallback) {
for (var i = 0; i < count; i++) {
sumOfDiffs += (values[i] - mean) * (values[i] - mean);
}

var stddev = Math.sqrt(sumOfDiffs / count);
current_timer_data["std"] = stddev;
current_timer_data["upper"] = max;
current_timer_data["lower"] = min;
current_timer_data["count"] = count;
current_timer_data["count"] = timer_counters[key];
current_timer_data["count_ps"] = timer_counters[key] / (flushInterval / 1000);
current_timer_data["sum"] = sum;
current_timer_data["mean"] = mean;

Expand Down
25 changes: 15 additions & 10 deletions stats.js
View file
Expand Up @@ -13,6 +13,7 @@ var dgram = require('dgram')
var keyCounter = {};
var counters = {};
var timers = {};
var timer_counters = {};
var gauges = {};
var sets = {};
var counter_rates = {};
Expand Down Expand Up @@ -48,6 +49,7 @@ function flushMetrics() {
counters: counters,
gauges: gauges,
timers: timers,
timer_counters: timer_counters,
sets: sets,
counter_rates: counter_rates,
timer_data: timer_data,
Expand All @@ -69,6 +71,7 @@ function flushMetrics() {
// Clear the timers
for (var key in metrics.timers) {
metrics.timers[key] = [];
metrics.timer_counters[key] = 0;
}

// Clear the sets
Expand Down Expand Up @@ -158,6 +161,16 @@ config.configFile(process.argv[2], function (config, oldConfig) {
for (var i = 0; i < bits.length; i++) {
var sampleRate = 1;
var fields = bits[i].split("|");
if (fields[2]) {
if (fields[2].match(/^@([\d\.]+)/)) {
sampleRate = Number(fields[2].match(/^@([\d\.]+)/)[1]);
} else {
l.log('Bad line: ' + fields + ' in msg "' + metrics[midx] +'"; has invalid sample rate');
counters[bad_lines_seen]++;
stats['messages']['bad_lines_seen']++;
continue;
}
}
if (fields[1] === undefined) {
l.log('Bad line: ' + fields + ' in msg "' + metrics[midx] +'"');
counters[bad_lines_seen]++;
Expand All @@ -167,8 +180,10 @@ config.configFile(process.argv[2], function (config, oldConfig) {
if (fields[1].trim() == "ms") {
if (! timers[key]) {
timers[key] = [];
timer_counters[key] = 0;
}
timers[key].push(Number(fields[0] || 0));
timer_counters[key] += (1 / sampleRate);
} else if (fields[1].trim() == "g") {
gauges[key] = Number(fields[0] || 0);
} else if (fields[1].trim() == "s") {
Expand All @@ -177,16 +192,6 @@ config.configFile(process.argv[2], function (config, oldConfig) {
}
sets[key].insert(fields[0] || '0');
} else {
if (fields[2]) {
if (fields[2].match(/^@([\d\.]+)/)) {
sampleRate = Number(fields[2].match(/^@([\d\.]+)/)[1]);
} else {
l.log('Bad line: ' + fields + ' in msg "' + metrics[midx] +'"; has invalid sample rate');
counters[bad_lines_seen]++;
stats['messages']['bad_lines_seen']++;
continue;
}
}
if (! counters[key]) {
counters[key] = 0;
}
Expand Down
35 changes: 34 additions & 1 deletion test/graphite_tests.js
View file
Expand Up @@ -190,7 +190,7 @@ module.exports = {
},

timers_are_valid: function (test) {
test.expect(3);
test.expect(5);

var testvalue = 100;
var me = this;
Expand All @@ -216,6 +216,39 @@ module.exports = {
};
test.ok(_.any(hashes,testtimervalue_test), 'stats.timers.a_test_value.mean should be ' + testvalue);

var count_test = function(post, metric){
var mykey = 'stats.timers.a_test_value.' + metric;
return _.first(_.filter(_.pluck(post, mykey), function (e) { return e }));
};
test.equals(count_test(hashes, 'count_ps'), 5, 'count_ps should be 5');
test.equals(count_test(hashes, 'count'), 1, 'count should be 1');

test.done();
});
});
});
},

sampled_timers_are_valid: function (test) {
test.expect(2);

var testvalue = 100;
var me = this;
this.acceptor.once('connection',function(c){
statsd_send('a_test_value:' + testvalue + '|ms|@0.1',me.sock,'127.0.0.1',8125,function(){
collect_for(me.acceptor,me.myflush*2,function(strings){
var hashes = _.map(strings, function(x) {
var chunks = x.split(' ');
var data = {};
data[chunks[0]] = chunks[1];
return data;
});
var count_test = function(post, metric){
var mykey = 'stats.timers.a_test_value.' + metric;
return _.first(_.filter(_.pluck(post, mykey), function (e) { return e }));
};
test.equals(count_test(hashes, 'count_ps'), 50, 'count_ps should be 50');
test.equals(count_test(hashes, 'count'), 10, 'count should be 10');
test.done();
});
});
Expand Down
32 changes: 30 additions & 2 deletions test/process_metrics_tests.js
View file
Expand Up @@ -7,13 +7,15 @@ module.exports = {
var counters = {};
var gauges = {};
var timers = {};
var timer_counters = {};
var sets = {};
var pctThreshold = null;

this.metrics = {
counters: counters,
gauges: gauges,
timers: timers,
timer_counters: timer_counters,
sets: sets,
pctThreshold: pctThreshold
}
Expand All @@ -36,40 +38,46 @@ module.exports = {
timers_handle_empty: function(test) {
test.expect(1);
this.metrics.timers['a'] = [];
this.metrics.timer_counters['a'] = 0;
pm.process_metrics(this.metrics, 100, this.time_stamp, function(){});
//potentially a cleaner way to check this
test.equal(undefined, this.metrics.counter_rates['a']);
test.done();
},
timers_single_time: function(test) {
test.expect(6);
test.expect(7);
this.metrics.timers['a'] = [100];
this.metrics.timer_counters['a'] = 1;
pm.process_metrics(this.metrics, 100, this.time_stamp, function(){});
timer_data = this.metrics.timer_data['a'];
test.equal(0, timer_data.std);
test.equal(100, timer_data.upper);
test.equal(100, timer_data.lower);
test.equal(1, timer_data.count);
test.equal(10, timer_data.count_ps);
test.equal(100, timer_data.sum);
test.equal(100, timer_data.mean);
test.done();
},
timers_multiple_times: function(test) {
test.expect(6);
test.expect(7);
this.metrics.timers['a'] = [100, 200, 300];
this.metrics.timer_counters['a'] = 3;
pm.process_metrics(this.metrics, 100, this.time_stamp, function(){});
timer_data = this.metrics.timer_data['a'];
test.equal(81.64965809277261, timer_data.std);
test.equal(300, timer_data.upper);
test.equal(100, timer_data.lower);
test.equal(3, timer_data.count);
test.equal(30, timer_data.count_ps);
test.equal(600, timer_data.sum);
test.equal(200, timer_data.mean);
test.done();
},
timers_single_time_single_percentile: function(test) {
test.expect(3);
this.metrics.timers['a'] = [100];
this.metrics.timer_counters['a'] = 1;
this.metrics.pctThreshold = [90];
pm.process_metrics(this.metrics, 100, this.time_stamp, function(){});
timer_data = this.metrics.timer_data['a'];
Expand All @@ -81,6 +89,7 @@ module.exports = {
timers_single_time_multiple_percentiles: function(test) {
test.expect(6);
this.metrics.timers['a'] = [100];
this.metrics.timer_counters['a'] = 1;
this.metrics.pctThreshold = [90, 80];
pm.process_metrics(this.metrics, 100, this.time_stamp, function(){});
timer_data = this.metrics.timer_data['a'];
Expand All @@ -95,6 +104,7 @@ module.exports = {
timers_multiple_times_single_percentiles: function(test) {
test.expect(3);
this.metrics.timers['a'] = [100, 200, 300];
this.metrics.timer_counters['a'] = 3;
this.metrics.pctThreshold = [90];
pm.process_metrics(this.metrics, 100, this.time_stamp, function(){});
timer_data = this.metrics.timer_data['a'];
Expand All @@ -106,9 +116,27 @@ module.exports = {
timers_multiple_times_multiple_percentiles: function(test) {
test.expect(6);
this.metrics.timers['a'] = [100, 200, 300];
this.metrics.timer_counters['a'] = 3;
this.metrics.pctThreshold = [90, 80];
pm.process_metrics(this.metrics, 100, this.time_stamp, function(){});
timer_data = this.metrics.timer_data['a'];
test.equal(200, timer_data.mean_90);
test.equal(300, timer_data.upper_90);
test.equal(600, timer_data.sum_90);
test.equal(150, timer_data.mean_80);
test.equal(200, timer_data.upper_80);
test.equal(300, timer_data.sum_80);
test.done();
},
timers_sampled_times: function(test) {
test.expect(8);
this.metrics.timers['a'] = [100, 200, 300];
this.metrics.timer_counters['a'] = 50;
this.metrics.pctThreshold = [90, 80];
pm.process_metrics(this.metrics, 100, this.time_stamp, function(){});
timer_data = this.metrics.timer_data['a'];
test.equal(50, timer_data.count);
test.equal(500, timer_data.count_ps);
test.equal(200, timer_data.mean_90);
test.equal(300, timer_data.upper_90);
test.equal(600, timer_data.sum_90);
Expand Down

0 comments on commit cc12534

Please sign in to comment.