From 99b8a159d1aa4074a6bf5bf181e38ded659c2f39 Mon Sep 17 00:00:00 2001 From: Jerry Chen Date: Thu, 26 May 2011 23:07:51 -0500 Subject: [PATCH 1/6] Documented some more config variables in exampleConfig.js and README --- README.md | 14 ++++++++++++-- exampleConfig.js | 19 ++++++++++++++++++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 71718061..b22f87f5 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Counting gorets:1|c -This is a simple counter. Add 1 to the "gorets" bucket. It stays in memory until the flush interval. +This is a simple counter. Add 1 to the "gorets" bucket. It stays in memory until the flush interval `config.flushInterval`. Timing @@ -31,7 +31,7 @@ Timing glork:320|ms -The glork took 320ms to complete this time. StatsD figures out 90th percentile, average (mean), lower and upper bounds for the flush interval. +The glork took 320ms to complete this time. StatsD figures out 90th percentile, average (mean), lower and upper bounds for the flush interval. The percentile threshold can be tweaked with `config.percentThreshold`. Sampling -------- @@ -40,6 +40,16 @@ Sampling Tells StatsD that this counter is being sent sampled every 1/10th of the time. +Debugging +--------- + +There are additional config variables available for debugging: + +* `debug` - log exceptions and periodically print out information on counters and timers +* `debugInterval` - interval for printing out information on counters and timers +* `dumpMessages` - print debug info on incoming messages + +For more information, check the `exampleConfig.js`. Guts ---- diff --git a/exampleConfig.js b/exampleConfig.js index 7c66c7cc..fe27ef17 100644 --- a/exampleConfig.js +++ b/exampleConfig.js @@ -1,6 +1,23 @@ +/* + +Required Variables: + + graphiteHost: hostname or IP of Graphite server + graphitePort: port of Graphite server + port: StatsD listening port [default: 8125] + +Optional Variables: + + debug: debug flag [default: false] + debugInterval: interval to print debug information [ms, default: 10000] + dumpMessages: log all incoming messages + flushInterval: interval (in ms) to flush to Graphite + percentThreshold: for time information, calculate the Nth percentile + [%, default: 90] + +*/ { graphitePort: 2003 , graphiteHost: "graphite.host.com" , port: 8125 } - From 3f8ae56425b034349096f5bd70811904ae16369b Mon Sep 17 00:00:00 2001 From: Garret Heaton Date: Thu, 10 Nov 2011 04:25:37 +0000 Subject: [PATCH 2/6] Node v0.6 compatibility --- config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.js b/config.js index a94ad472..d529fe5a 100644 --- a/config.js +++ b/config.js @@ -14,7 +14,7 @@ var Configurator = function (file) { if (err) { throw err; } old_config = self.config; - self.config = process.compile('config = ' + data, file); + self.config = eval('config = ' + fs.readFileSync(file)); self.emit('configChanged', self.config); }); }; From 7ac28e9e665d0fc89c30dc8f74862d85c1a4cdad Mon Sep 17 00:00:00 2001 From: Jesse Date: Tue, 27 Dec 2011 20:26:02 +0000 Subject: [PATCH 3/6] Make Graphite connection config variables optional. Leaving them out avoids connecting to Graphite at all. Use in combination with debug mode to develop and test statsd clients without a Graphite server. --- exampleConfig.js | 9 ++++++++- stats.js | 30 ++++++++++++++++-------------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/exampleConfig.js b/exampleConfig.js index fe27ef17..a902f314 100644 --- a/exampleConfig.js +++ b/exampleConfig.js @@ -2,9 +2,16 @@ Required Variables: + port: StatsD listening port [default: 8125] + +Graphite Required Variables: + +(Leave these unset to avoid sending stats to Graphite. + Set debug flag and leave these unset to run in 'dry' debug mode - + useful for testing statsd clients without a Graphite server.) + graphiteHost: hostname or IP of Graphite server graphitePort: port of Graphite server - port: StatsD listening port [default: 8125] Optional Variables: diff --git a/stats.js b/stats.js index 8cb80b25..709e8fcf 100644 --- a/stats.js +++ b/stats.js @@ -191,23 +191,25 @@ config.configFile(process.argv[2], function (config, oldConfig) { statString += 'statsd.numStats ' + numStats + ' ' + ts + "\n"; - try { - var graphite = net.createConnection(config.graphitePort, config.graphiteHost); - graphite.addListener('error', function(connectionException){ + if (config.graphiteHost) { + try { + var graphite = net.createConnection(config.graphitePort, config.graphiteHost); + graphite.addListener('error', function(connectionException){ + if (config.debug) { + sys.log(connectionException); + } + }); + graphite.on('connect', function() { + this.write(statString); + this.end(); + stats['graphite']['last_flush'] = Math.round(new Date().getTime() / 1000); + }); + } catch(e){ if (config.debug) { - sys.log(connectionException); + sys.log(e); } - }); - graphite.on('connect', function() { - this.write(statString); - this.end(); - stats['graphite']['last_flush'] = Math.round(new Date().getTime() / 1000); - }); - } catch(e){ - if (config.debug) { - sys.log(e); + stats['graphite']['last_exception'] = Math.round(new Date().getTime() / 1000); } - stats['graphite']['last_exception'] = Math.round(new Date().getTime() / 1000); } }, flushInterval); From 458594bbc65969ae7e4e231d0b61639daddd96fd Mon Sep 17 00:00:00 2001 From: Wei Hsu Date: Thu, 5 Jan 2012 12:54:15 -0800 Subject: [PATCH 4/6] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4dbb5430..1b774c81 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Concepts -------- * *buckets* - Each stat is in it's own "bucket". They are not predefined anywhere. Buckets can be named anything that will translate to Graphite (periods make folders, etc) + Each stat is in its own "bucket". They are not predefined anywhere. Buckets can be named anything that will translate to Graphite (periods make folders, etc) * *values* Each stat will have a value. How it is interpreted depends on modifiers From 766876b072122b3eccebaa2410079fe0d8bdef08 Mon Sep 17 00:00:00 2001 From: Mike Stipicevic Date: Tue, 10 Jan 2012 07:03:49 -0800 Subject: [PATCH 5/6] Add basic test framework Adds a node-unit based test framework designed to test a running statsd instance and test its output. Should facilitate later testing of features and prevent regressions. --- README.md | 6 ++ run_tests.sh | 12 +++ stats.js | 2 + test/graphite_tests.js | 227 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 247 insertions(+) create mode 100755 run_tests.sh create mode 100644 test/graphite_tests.js diff --git a/README.md b/README.md index 1b774c81..0c8b9ac5 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,12 @@ Installation and Configuration node stats.js /path/to/config +Tests +----- + +A test framework has been added using node-unit and some custom code to start and manipulate statsd. Please add tests under test/ for any new features or bug fixes encountered. Testing a live server can be tricky, attempts were made to eliminate race conditions but it may be possible to encounter a stuck state. If doing dev work, a `killall node` will kill any stray test servers in the background (don't do this on a production machine!). + +Tests can be executd with `./run_tests.sh`. Inspiration ----------- diff --git a/run_tests.sh b/run_tests.sh new file mode 100755 index 00000000..96581715 --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env node +try { + var reporter = require('nodeunit').reporters.default; +} +catch(e) { + console.log("Cannot find nodeunit module."); + console.log("Make sure to run 'npm install nodeunit'"); + process.exit(); +} + +process.chdir(__dirname); +reporter.run(['test/']); diff --git a/stats.js b/stats.js index 709e8fcf..09b84038 100644 --- a/stats.js +++ b/stats.js @@ -131,6 +131,8 @@ config.configFile(process.argv[2], function (config, oldConfig) { server.bind(config.port || 8125); mgmtServer.listen(config.mgmt_port || 8126); + sys.log("server is up"); + var flushInterval = Number(config.flushInterval || 10000); flushInt = setInterval(function () { diff --git a/test/graphite_tests.js b/test/graphite_tests.js new file mode 100644 index 00000000..5dc42f95 --- /dev/null +++ b/test/graphite_tests.js @@ -0,0 +1,227 @@ +var fs = require('fs'), + net = require('net'), + temp = require('temp'), + spawn = require('child_process').spawn, + sys = require('sys'), + urlparse = require('url').parse, + _ = require('underscore'), + dgram = require('dgram'), + qsparse = require('querystring').parse, + http = require('http'); + + +var writeconfig = function(text,worker,cb,obj){ + temp.open({suffix: '-statsdconf.js'}, function(err, info) { + if (err) throw err; + fs.write(info.fd, text); + fs.close(info.fd, function(err) { + if (err) throw err; + worker(info.path,cb,obj); + }); + }); +} + +var array_contents_are_equal = function(first,second){ + var intlen = _.intersection(first,second).length; + var unlen = _.union(first,second).length; + return (intlen == unlen) && (intlen == first.length); +} + +var statsd_send = function(data,sock,host,port,cb){ + send_data = new Buffer(data); + sock.send(send_data,0,send_data.length,port,host,function(err,bytes){ + if (err) { + throw err; + } + cb(); + }); +} + +// keep collecting data until a specified timeout period has elapsed +// this will let us capture all data chunks so we don't miss one +var collect_for = function(server,timeout,cb){ + var received = []; + var in_flight = 0; + var start_time = new Date().getTime(); + var collector = function(req,res){ + in_flight += 1; + var body = ''; + req.on('data',function(data){ body += data; }); + req.on('end',function(){ + received = received.concat(body.split("\n")); + in_flight -= 1; + if((in_flight < 1) && (new Date().getTime() > (start_time + timeout))){ + server.removeListener('request',collector); + cb(received); + } + }); + } + + setTimeout(function (){ + server.removeListener('connection',collector); + if((in_flight < 1)){ + cb(received); + } + },timeout); + + server.on('connection',collector); +} + +module.exports = { + setUp: function (callback) { + this.testport = 31337; + this.myflush = 200; + var configfile = "{graphService: \"graphite\"\n\ + , batch: 200 \n\ + , flushInterval: " + this.myflush + " \n\ + , port: 8125\n\ + , dumpMessages: false \n\ + , debug: false\n\ + , graphitePort: " + this.testport + "\n\ + , graphiteHost: \"127.0.0.1\"}"; + + this.acceptor = net.createServer(); + this.acceptor.listen(this.testport); + this.sock = dgram.createSocket('udp4'); + + this.server_up = true; + this.ok_to_die = false; + this.exit_callback_callback = process.exit; + + writeconfig(configfile,function(path,cb,obj){ + obj.path = path; + obj.server = spawn('node',['stats.js', path]); + obj.exit_callback = function (code) { + obj.server_up = false; + if(!obj.ok_to_die){ + console.log('node server unexpectedly quit with code: ' + code); + process.exit(1); + } + else { + obj.exit_callback_callback(); + } + }; + obj.server.on('exit', obj.exit_callback); + obj.server.stderr.on('data', function (data) { + console.log('stderr: ' + data.toString().replace(/\n$/,'')); + }); + /* + obj.server.stdout.on('data', function (data) { + console.log('stdout: ' + data.toString().replace(/\n$/,'')); + }); + */ + obj.server.stdout.on('data', function (data) { + // wait until server is up before we finish setUp + if (data.toString().match(/server is up/)) { + cb(); + } + }); + + },callback,this); + }, + tearDown: function (callback) { + this.sock.close(); + this.acceptor.close(); + this.ok_to_die = true; + if(this.server_up){ + this.exit_callback_callback = callback; + this.server.kill(); + } else { + callback(); + } + }, + + send_well_formed_posts: function (test) { + test.expect(2); + + // we should integrate a timeout into this + this.acceptor.once('connection',function(c){ + var body = ''; + c.on('data',function(d){ body += d; }); + c.on('end',function(){ + var rows = body.split("\n"); + var entries = _.map(rows, function(x) { + var chunks = x.split(' '); + var data = {}; + data[chunks[0]] = chunks[1]; + return data; + }); + test.ok(_.include(_.map(entries,function(x) { return _.keys(x)[0] }),'statsd.numStats'),'graphite output includes numStats'); + test.equal(_.find(entries, function(x) { return _.keys(x)[0] == 'statsd.numStats' })['statsd.numStats'],0); + test.done(); + }); + }); + }, + + timers_are_valid: function (test) { + test.expect(3); + + var testvalue = 100; + var me = this; + this.acceptor.once('connection',function(c){ + statsd_send('a_test_value:' + testvalue + '|ms',me.sock,'127.0.0.1',8125,function(){ + collect_for(me.acceptor,me.myflush*2,function(strings){ + test.ok(strings.length > 0,'should receive some data'); + var hashes = _.map(strings, function(x) { + var chunks = x.split(' '); + var data = {}; + data[chunks[0]] = chunks[1]; + return data; + }); + var numstat_test = function(post){ + var mykey = 'statsd.numStats'; + return _.include(_.keys(post),mykey) && (post[mykey] == 1); + }; + test.ok(_.any(hashes,numstat_test), 'statsd.numStats should be 1'); + + var testtimervalue_test = function(post){ + var mykey = 'stats.timers.a_test_value.mean'; + return _.include(_.keys(post),mykey) && (post[mykey] == testvalue); + }; + test.ok(_.any(hashes,testtimervalue_test), 'stats.timers.a_test_value.mean should be ' + testvalue); + + test.done(); + }); + }); + }); + }, + + counts_are_valid: function (test) { + test.expect(4); + + var testvalue = 100; + var me = this; + this.acceptor.once('connection',function(c){ + statsd_send('a_test_value:' + testvalue + '|c',me.sock,'127.0.0.1',8125,function(){ + collect_for(me.acceptor,me.myflush*2,function(strings){ + test.ok(strings.length > 0,'should receive some data'); + var hashes = _.map(strings, function(x) { + var chunks = x.split(' '); + var data = {}; + data[chunks[0]] = chunks[1]; + return data; + }); + var numstat_test = function(post){ + var mykey = 'statsd.numStats'; + return _.include(_.keys(post),mykey) && (post[mykey] == 1); + }; + test.ok(_.any(hashes,numstat_test), 'statsd.numStats should be 1'); + + var testavgvalue_test = function(post){ + var mykey = 'stats.a_test_value'; + return _.include(_.keys(post),mykey) && (post[mykey] == (testvalue/(me.myflush / 1000))); + }; + test.ok(_.any(hashes,testavgvalue_test), 'stats.a_test_value should be ' + (testvalue/(me.myflush / 1000))); + + var testcountvalue_test = function(post){ + var mykey = 'stats_counts.a_test_value'; + return _.include(_.keys(post),mykey) && (post[mykey] == testvalue); + }; + test.ok(_.any(hashes,testcountvalue_test), 'stats_counts.a_test_value should be ' + testvalue); + + test.done(); + }); + }); + }); + } +} From 60ea7f0ef45edffb43b7274b939c5bd9d07eaf06 Mon Sep 17 00:00:00 2001 From: leonmax Date: Tue, 24 Jan 2012 18:48:06 -0500 Subject: [PATCH 6/6] the 'data' doesn't have @sample_rate! 'sampled_data' does. --- python_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_example.py b/python_example.py index 19083627..bd6f93c2 100644 --- a/python_example.py +++ b/python_example.py @@ -72,7 +72,7 @@ def send(data, sample_rate=1): import random if random.random() <= sample_rate: for stat in data.keys(): - value = data[stat] + value = sampled_data[stat] sampled_data[stat] = "%s|@%s" %(value, sample_rate) else: sampled_data=data