Skip to content
This repository
Browse code

Merge branch 'master' of https://github.com/draco2003/statsd into ext…

…racted-metrics-processing
  • Loading branch information...
commit d2bfdf9552a097e1a882dacd20f76b4987b8874b 2 parents e37c952 + dc03d36
Daniel Schauenberg authored November 03, 2012
2  README.md
Source Rendered
@@ -270,6 +270,8 @@ metrics: {
270 270
     gauges: gauges,
271 271
     timers: timers,
272 272
     sets: sets,
  273
+    counter_rates: counter_rates,
  274
+    timer_data: timer_data,
273 275
     pctThreshold: pctThreshold
274 276
 }
275 277
   ```
4  backends/console.js
@@ -31,9 +31,11 @@ ConsoleBackend.prototype.flush = function(timestamp, metrics) {
31 31
   });
32 32
 
33 33
   var out = {
34  
-    counter: this.statsCache.counters,
  34
+    counters: this.statsCache.counters,
35 35
     timers: this.statsCache.timers,
36 36
     gauges: metrics.gauges,
  37
+    timer_data: metrics.timer_data,
  38
+    counter_rates: metrics.counter_rates,
37 39
     sets: function (vals) {
38 40
       var ret = {};
39 41
       for (val in vals) {
70  backends/graphite.js
@@ -55,88 +55,40 @@ var flush_stats = function graphite_flush(ts, metrics) {
55 55
   var statString = '';
56 56
   var numStats = 0;
57 57
   var key;
58  
-
  58
+  var timer_data_key;
59 59
   var counters = metrics.counters;
60 60
   var gauges = metrics.gauges;
61 61
   var timers = metrics.timers;
62 62
   var sets = metrics.sets;
63  
-  var pctThreshold = metrics.pctThreshold;
  63
+  var counter_rates = metrics.counter_rates;
  64
+  var timer_data = metrics.timer_data;
64 65
 
65 66
   for (key in counters) {
66  
-    var value = counters[key];
67  
-    var valuePerSecond = value / (flushInterval / 1000); // calculate "per second" rate
68  
-
69  
-    statString += 'stats.'        + key + ' ' + valuePerSecond + ' ' + ts + "\n";
70  
-    statString += 'stats_counts.' + key + ' ' + value          + ' ' + ts + "\n";
  67
+    statString += 'stats.'        + key + ' ' + counter_rates[key] + ' ' + ts + "\n";
  68
+    statString += 'stats_counts.' + key + ' ' + counters[key]      + ' ' + ts + "\n";
71 69
 
72 70
     numStats += 1;
73 71
   }
74 72
 
75  
-  for (key in timers) {
76  
-    if (timers[key].length > 0) {
77  
-      var values = timers[key].sort(function (a,b) { return a-b; });
78  
-      var count = values.length;
79  
-      var min = values[0];
80  
-      var max = values[count - 1];
81  
-
82  
-      var cumulativeValues = [min];
83  
-      for (var i = 1; i < count; i++) {
84  
-          cumulativeValues.push(values[i] + cumulativeValues[i-1]);
  73
+  for (key in timer_data) {
  74
+    if (Object.keys(timer_data).length > 0) {
  75
+      for (timer_data_key in timer_data[key]) {
  76
+         statString += 'stats.timers.' + key + '.' + timer_data_key + ' ' + timer_data[key][timer_data_key] + ' ' + ts + "\n";
85 77
       }
86 78
 
87  
-      var sum = min;
88  
-      var mean = min;
89  
-      var maxAtThreshold = max;
90  
-
91  
-      var message = "";
92  
-
93  
-      var key2;
94  
-
95  
-      for (key2 in pctThreshold) {
96  
-        var pct = pctThreshold[key2];
97  
-        if (count > 1) {
98  
-          var numInThreshold = Math.round(pct / 100 * count);
99  
-
100  
-          maxAtThreshold = values[numInThreshold - 1];
101  
-          sum = cumulativeValues[numInThreshold - 1];
102  
-          mean = sum / numInThreshold;
103  
-        }
104  
-
105  
-        var clean_pct = '' + pct;
106  
-        clean_pct.replace('.', '_');
107  
-        message += 'stats.timers.' + key + '.mean_'  + clean_pct + ' ' + mean           + ' ' + ts + "\n";
108  
-        message += 'stats.timers.' + key + '.upper_' + clean_pct + ' ' + maxAtThreshold + ' ' + ts + "\n";
109  
-        message += 'stats.timers.' + key + '.sum_' + clean_pct + ' ' + sum + ' ' + ts + "\n";
110  
-      }
111  
-
112  
-      sum = cumulativeValues[count-1];
113  
-      mean = sum / count;
114  
-
115  
-      var sumOfDiffs = 0;
116  
-      for (var i = 0; i < count; i++) {
117  
-         sumOfDiffs += (values[i] - mean) * (values[i] - mean);
118  
-      }
119  
-      var stddev = Math.sqrt(sumOfDiffs / count);
120  
-
121  
-      message += 'stats.timers.' + key + '.std ' + stddev  + ' ' + ts + "\n";
122  
-      message += 'stats.timers.' + key + '.upper ' + max   + ' ' + ts + "\n";
123  
-      message += 'stats.timers.' + key + '.lower ' + min   + ' ' + ts + "\n";
124  
-      message += 'stats.timers.' + key + '.count ' + count + ' ' + ts + "\n";
125  
-      message += 'stats.timers.' + key + '.sum ' + sum  + ' ' + ts + "\n";
126  
-      message += 'stats.timers.' + key + '.mean ' + mean + ' ' + ts + "\n";
127  
-      statString += message;
128  
-
129 79
       numStats += 1;
130 80
     }
131 81
   }
132 82
 
133 83
   for (key in gauges) {
134 84
     statString += 'stats.gauges.' + key + ' ' + gauges[key] + ' ' + ts + "\n";
  85
+
135 86
     numStats += 1;
136 87
   }
137 88
 
138 89
   for (key in sets) {
139 90
     statString += 'stats.sets.' + key + '.count ' + sets[key].values().length + ' ' + ts + "\n";
  91
+
140 92
     numStats += 1;
141 93
   }
142 94
 
85  lib/process_metrics.js
... ...
@@ -0,0 +1,85 @@
  1
+var process_metrics = function (metrics, flushInterval, ts, flushCallback) {
  2
+    var key;
  3
+    var counter_rates = {};
  4
+    var timer_data = {};
  5
+    var counters = metrics.counters;
  6
+    var timers = metrics.timers;
  7
+    var pctThreshold = metrics.pctThreshold;
  8
+
  9
+    for (key in counters) {
  10
+      var value = counters[key];
  11
+
  12
+      // calculate "per second" rate
  13
+      var valuePerSecond = value / (flushInterval / 1000);
  14
+      counter_rates[key] = valuePerSecond;
  15
+    }
  16
+
  17
+    for (key in timers) {
  18
+      if (timers[key].length > 0) {
  19
+        timer_data[key] = {};
  20
+        var current_timer_data = {};
  21
+
  22
+        var values = timers[key].sort(function (a,b) { return a-b; });
  23
+        var count = values.length;
  24
+        var min = values[0];
  25
+        var max = values[count - 1];
  26
+
  27
+        var cumulativeValues = [min];
  28
+        for (var i = 1; i < count; i++) {
  29
+            cumulativeValues.push(values[i] + cumulativeValues[i-1]);
  30
+        }
  31
+
  32
+        var sum = min;
  33
+        var mean = min;
  34
+        var maxAtThreshold = max;
  35
+
  36
+        var message = "";
  37
+
  38
+        var key2;
  39
+
  40
+        for (key2 in pctThreshold) {
  41
+          var pct = pctThreshold[key2];
  42
+          if (count > 1) {
  43
+            var numInThreshold = Math.round(pct / 100 * count);
  44
+
  45
+            maxAtThreshold = values[numInThreshold - 1];
  46
+            sum = cumulativeValues[numInThreshold - 1];
  47
+            mean = sum / numInThreshold;
  48
+          }
  49
+
  50
+          var clean_pct = '' + pct;
  51
+          clean_pct.replace('.', '_');
  52
+          current_timer_data["mean_" + clean_pct] = mean;
  53
+          current_timer_data["upper_" + clean_pct] = maxAtThreshold;
  54
+          current_timer_data["sum_" + clean_pct] = sum;
  55
+
  56
+        }
  57
+
  58
+        sum = cumulativeValues[count-1];
  59
+        mean = sum / count;
  60
+
  61
+        var sumOfDiffs = 0;
  62
+        for (var i = 0; i < count; i++) {
  63
+           sumOfDiffs += (values[i] - mean) * (values[i] - mean);
  64
+        }
  65
+        var stddev = Math.sqrt(sumOfDiffs / count);
  66
+        current_timer_data["std"] = stddev;
  67
+        current_timer_data["upper"] = max;
  68
+        current_timer_data["lower"] = min;
  69
+        current_timer_data["count"] = count;
  70
+        current_timer_data["sum"] = sum;
  71
+        current_timer_data["mean"] = mean;
  72
+
  73
+        timer_data[key] = current_timer_data;
  74
+
  75
+      }
  76
+    }
  77
+
  78
+    //add processed metrics to the metrics_hash
  79
+    metrics.counter_rates = counter_rates;
  80
+    metrics.timer_data = timer_data;
  81
+
  82
+    flushCallback(metrics);
  83
+  }
  84
+
  85
+exports.process_metrics = process_metrics
14  stats.js
@@ -6,6 +6,8 @@ var dgram  = require('dgram')
6 6
   , events = require('events')
7 7
   , logger = require('./lib/logger')
8 8
   , set = require('./lib/set')
  9
+  , pm = require('./lib/process_metrics')
  10
+
9 11
 
10 12
 // initialize data structures with defaults for statsd stats
11 13
 var keyCounter = {};
@@ -16,6 +18,8 @@ var counters = {
16 18
 var timers = {};
17 19
 var gauges = {};
18 20
 var sets = {};
  21
+var counter_rates = {};
  22
+var timer_data = {};
19 23
 var pctThreshold = null;
20 24
 var debugInt, flushInterval, keyFlushInt, server, mgmtServer;
21 25
 var startup_time = Math.round(new Date().getTime() / 1000);
@@ -45,6 +49,8 @@ function flushMetrics() {
45 49
     gauges: gauges,
46 50
     timers: timers,
47 51
     sets: sets,
  52
+    counter_rates: counter_rates,
  53
+    timer_data: timer_data,
48 54
     pctThreshold: pctThreshold
49 55
   }
50 56
 
@@ -66,14 +72,16 @@ function flushMetrics() {
66 72
     }
67 73
   });
68 74
 
69  
-  // Flush metrics to each backend.
70  
-  backendEvents.emit('flush', time_stamp, metrics_hash);
  75
+  pm.process_metrics(metrics_hash, flushInterval, time_stamp, function emitFlush(metrics) {
  76
+    backendEvents.emit('flush', time_stamp, metrics);
  77
+  });
  78
+
71 79
 };
72 80
 
73 81
 var stats = {
74 82
   messages: {
75 83
     last_msg_seen: startup_time,
76  
-    bad_lines_seen: 0,
  84
+    bad_lines_seen: 0
77 85
   }
78 86
 };
79 87
 
2  test/graphite_tests.js
@@ -240,7 +240,7 @@ module.exports = {
240 240
               var mykey = 'statsd.numStats';
241 241
               return _.include(_.keys(post),mykey) && (post[mykey] == 3);
242 242
             };
243  
-            test.ok(_.any(hashes,numstat_test), 'statsd.numStats should be 1');
  243
+            test.ok(_.any(hashes,numstat_test), 'statsd.numStats should be 3');
244 244
 
245 245
             var testavgvalue_test = function(post){
246 246
               var mykey = 'stats.a_test_value';
120  test/process_metrics_tests.js
... ...
@@ -0,0 +1,120 @@
  1
+var pm = require('../lib/process_metrics')
  2
+
  3
+module.exports = {
  4
+  setUp: function (callback) {
  5
+    this.time_stamp = Math.round(new Date().getTime() / 1000);
  6
+
  7
+    var counters = {};
  8
+    var gauges = {};
  9
+    var timers = {};
  10
+    var sets = {};
  11
+    var pctThreshold = null;
  12
+
  13
+    this.metrics = {
  14
+      counters: counters,
  15
+      gauges: gauges,
  16
+      timers: timers,
  17
+      sets: sets,
  18
+      pctThreshold: pctThreshold
  19
+    }
  20
+    callback();
  21
+  },
  22
+  counters_has_stats_count: function(test) {
  23
+    test.expect(1);
  24
+    this.metrics.counters['a'] = 2;
  25
+    pm.process_metrics(this.metrics, 1000, this.time_stamp, function(){});
  26
+    test.equal(2, this.metrics.counters['a']);
  27
+    test.done();
  28
+  },
  29
+  counters_has_correct_rate: function(test) {
  30
+    test.expect(1);
  31
+    this.metrics.counters['a'] = 2;
  32
+    pm.process_metrics(this.metrics, 100, this.time_stamp, function(){});
  33
+    test.equal(20, this.metrics.counter_rates['a']);
  34
+    test.done();
  35
+  },
  36
+  timers_handle_empty: function(test) {
  37
+    test.expect(1);
  38
+    this.metrics.timers['a'] = [];
  39
+    pm.process_metrics(this.metrics, 100, this.time_stamp, function(){});
  40
+    //potentially a cleaner way to check this
  41
+    test.equal(undefined, this.metrics.counter_rates['a']);
  42
+    test.done();
  43
+  },
  44
+  timers_single_time: function(test) {
  45
+    test.expect(6);
  46
+    this.metrics.timers['a'] = [100];
  47
+    pm.process_metrics(this.metrics, 100, this.time_stamp, function(){});
  48
+    timer_data = this.metrics.timer_data['a'];
  49
+    test.equal(0, timer_data.std);
  50
+    test.equal(100, timer_data.upper);
  51
+    test.equal(100, timer_data.lower);
  52
+    test.equal(1, timer_data.count);
  53
+    test.equal(100, timer_data.sum);
  54
+    test.equal(100, timer_data.mean);
  55
+    test.done();
  56
+  },
  57
+    timers_multiple_times: function(test) {
  58
+    test.expect(6);
  59
+    this.metrics.timers['a'] = [100, 200, 300];
  60
+    pm.process_metrics(this.metrics, 100, this.time_stamp, function(){});
  61
+    timer_data = this.metrics.timer_data['a'];
  62
+    test.equal(81.64965809277261, timer_data.std);
  63
+    test.equal(300, timer_data.upper);
  64
+    test.equal(100, timer_data.lower);
  65
+    test.equal(3, timer_data.count);
  66
+    test.equal(600, timer_data.sum);
  67
+    test.equal(200, timer_data.mean);
  68
+    test.done();
  69
+  },
  70
+    timers_single_time_single_percentile: function(test) {
  71
+    test.expect(3);
  72
+    this.metrics.timers['a'] = [100];
  73
+    this.metrics.pctThreshold = [90];
  74
+    pm.process_metrics(this.metrics, 100, this.time_stamp, function(){});
  75
+    timer_data = this.metrics.timer_data['a'];
  76
+    test.equal(100, timer_data.mean_90);
  77
+    test.equal(100, timer_data.upper_90);
  78
+    test.equal(100, timer_data.sum_90);
  79
+    test.done();
  80
+  },
  81
+    timers_single_time_multiple_percentiles: function(test) {
  82
+    test.expect(6);
  83
+    this.metrics.timers['a'] = [100];
  84
+    this.metrics.pctThreshold = [90, 80];
  85
+    pm.process_metrics(this.metrics, 100, this.time_stamp, function(){});
  86
+    timer_data = this.metrics.timer_data['a'];
  87
+    test.equal(100, timer_data.mean_90);
  88
+    test.equal(100, timer_data.upper_90);
  89
+    test.equal(100, timer_data.sum_90);
  90
+    test.equal(100, timer_data.mean_80);
  91
+    test.equal(100, timer_data.upper_80);
  92
+    test.equal(100, timer_data.sum_80);
  93
+    test.done();
  94
+  },
  95
+    timers_multiple_times_single_percentiles: function(test) {
  96
+    test.expect(3);
  97
+    this.metrics.timers['a'] = [100, 200, 300];
  98
+    this.metrics.pctThreshold = [90];
  99
+    pm.process_metrics(this.metrics, 100, this.time_stamp, function(){});
  100
+    timer_data = this.metrics.timer_data['a'];
  101
+    test.equal(200, timer_data.mean_90);
  102
+    test.equal(300, timer_data.upper_90);
  103
+    test.equal(600, timer_data.sum_90);
  104
+    test.done();
  105
+  },
  106
+    timers_multiple_times_multiple_percentiles: function(test) {
  107
+    test.expect(6);
  108
+    this.metrics.timers['a'] = [100, 200, 300];
  109
+    this.metrics.pctThreshold = [90, 80];
  110
+    pm.process_metrics(this.metrics, 100, this.time_stamp, function(){});
  111
+    timer_data = this.metrics.timer_data['a'];
  112
+    test.equal(200, timer_data.mean_90);
  113
+    test.equal(300, timer_data.upper_90);
  114
+    test.equal(600, timer_data.sum_90);
  115
+    test.equal(150, timer_data.mean_80);
  116
+    test.equal(200, timer_data.upper_80);
  117
+    test.equal(300, timer_data.sum_80);
  118
+    test.done();
  119
+  }
  120
+}

0 notes on commit d2bfdf9

Please sign in to comment.
Something went wrong with that request. Please try again.