Permalink
Browse files

Working example with statsd.

  • Loading branch information...
1 parent d5ce232 commit c0c151f7887cfedebc6938c9f32f7278570a465d @jbraeuer committed Feb 4, 2012
Showing with 149 additions and 1 deletion.
  1. +52 −1 Readme.markdown
  2. +61 −0 examples/indexer.conf
  3. +3 −0 examples/indexer.sh
  4. +27 −0 examples/shipper.conf
  5. +3 −0 examples/shipper.sh
  6. +3 −0 examples/web.sh
View
@@ -3,8 +3,59 @@ Running logstash from .debs on Ubuntu 11.10
# Install notes
## Install RabbitMQ
+
aptitude install rabbitmq-server rabbitmq-stomp
+
rabbitmqctl add_user logstash verysecret
+rabbitmqctl set_permissions -p / logstash ".*" ".*" ".*"
+
+## Install ElasticSearch
-# Install ElasticSearch
aptitude install elasticsearch
+
+# Run
+
+Use scripts in examples/ to run shipper, indexer and web
+
+# Integration with graphite via statsd
+
+## nodejs
+
+- nodejs 0.6
+- sudo add-apt-repository ppa:chris-lea/node.js
+
+## graphite
+
+- python2.7-carbon
+- edit /opt/graphite/conf/{storage-schemas,carbon}.conf
+- /opt/graphite/bin/carbon-cache.py start
+
+## statsd
+
+- https://github.com/etsy/statsd.git
+- build debian package
+
+- run statsd:
+ `/usr/bin/nodejs /usr/share/statsd/stats.js /etc/statsd/rdioConfig.js`
+
+- ABORT
+
+## statsd-c
+
+- https://github.com/jbuchbinder/statsd-c.git
+- graphite output not implemented
+
+- ABORT
+
+## ruby-statsd
+
+- git://github.com/fetep/ruby-statsd.git
+- has dependencies to eventmachine, etc
+
+fpm -s gem -t deb -S 19 -v 0.12.10 eventmachine
+fpm -s gem -t deb -S 19 -v 0.9.0 amq-protocol
+fpm -s gem -t deb -S 19 -v 0.9.1 amq-client
+fpm -s gem -t deb -S 19 -v 0.9.2 amqp
+fpm -s gem -t deb -S 19 -v 0.5 petef-statsd
+
+- ruby1.9.1 /var/lib/gems/1.9.1/bin/statsd -o stdout://
View
@@ -0,0 +1,61 @@
+input {
+ amqp {
+ # ship logs to the 'rawlogs' fanout queue.
+ type => "all"
+ host => "localhost"
+ exchange => "logstash_rawlogs"
+ name => "rawlogs_consumer"
+ user => "logstash"
+ password => "verysecret"
+ }
+}
+
+filter {
+ grok {
+ type => "syslog" # for logs of type "syslog"
+ pattern => "%{SYSLOGLINE}"
+ # You can specify multiple 'pattern' lines
+ }
+
+ grok {
+ type => "apache-access" # for logs of type 'apache-access'
+ pattern => "%{COMBINEDAPACHELOG}"
+ }
+
+ date {
+ type => "syslog"
+
+ # The 'timestamp' and 'timestamp8601' names are for fields in the
+ # logstash event. The 'SYSLOGLINE' grok pattern above includes a field
+ # named 'timestamp' that is set to the normal syslog timestamp if it
+ # exists in the event.
+ timestamp => "MMM d HH:mm:ss" # syslog 'day' value can be space-leading
+ timestamp => "MMM dd HH:mm:ss"
+ timestamp8601 => ISO8601 # Some syslogs use ISO8601 time format
+ }
+
+ date {
+ type => "apache-access"
+ timestamp => "dd/MMM/yyyy:HH:mm:ss Z"
+ }
+}
+
+output {
+ stdout {
+ debug => true
+ }
+
+ # If your elasticsearch server is discoverable with multicast, use this:
+ #elasticsearch { }
+
+ # If you can't discover using multicast, set the address explicitly
+ elasticsearch {
+ host => "localhost"
+ }
+
+ statsd {
+ # Count one hit every event by response
+ increment => "apache.response.%{response}"
+ count => [ "apache.bytes", "%{bytes}" ]
+ }
+}
View
@@ -0,0 +1,3 @@
+#! /bin/bash
+
+java -jar /usr/share/logstash/logstash-1.1.0-monolithic.jar agent -f indexer.conf
View
@@ -0,0 +1,27 @@
+input {
+ tcp {
+ type => "apache-access"
+ port => 3333
+ }
+ tcp {
+ type => "syslog"
+ port => 3334
+ }
+}
+
+output {
+ # Output events to stdout for debugging. Feel free to remove
+ # this output if you don't need it.
+ stdout {
+ debug => true
+ }
+
+ # Ship events to the amqp fanout exchange named 'rawlogs"
+ amqp {
+ host => "localhost"
+ user => "logstash"
+ password => "verysecret"
+ exchange_type => "fanout"
+ name => "logstash_rawlogs"
+ }
+}
View
@@ -0,0 +1,3 @@
+#! /bin/bash
+
+java -jar /usr/share/logstash/logstash-1.1.0-monolithic.jar agent -f shipper.conf
View
@@ -0,0 +1,3 @@
+#! /bin/bash
+
+java -jar /usr/share/logstash/logstash-1.1.0-monolithic.jar web --backend elasticsearch://localhost/

0 comments on commit c0c151f

Please sign in to comment.