Permalink
Browse files

- add graphite module

  • Loading branch information...
1 parent e446636 commit d5ac48ea4883e3515408b3bc361cabc19aeab92b @jordansissel jordansissel committed Nov 3, 2011
View
@@ -0,0 +1,79 @@
+[cache]
+LOCAL_DATA_DIR = /mnt/graphite/storage/whisper/
+
+# Specify the user to drop privileges to
+# If this is blank carbon runs as the user that invokes it
+# This user must have write access to the local data directory
+USER =
+
+# Limit the size of the cache to avoid swapping or becoming CPU bound.
+# Sorts and serving cache queries gets more expensive as the cache grows.
+# Use the value "inf" (infinity) for an unlimited cache size.
+MAX_CACHE_SIZE = inf
+
+# Limits the number of whisper update_many() calls per second, which effectively
+# means the number of write requests sent to the disk. This is intended to
+# prevent over-utilizing the disk and thus starving the rest of the system.
+# When the rate of required updates exceeds this, then carbon's caching will
+# take effect and increase the overall throughput accordingly.
+MAX_UPDATES_PER_SECOND = 1000
+
+# Softly limits the number of whisper files that get created each minute.
+# Setting this value low (like at 50) is a good way to ensure your graphite
+# system will not be adversely impacted when a bunch of new metrics are
+# sent to it. The trade off is that it will take much longer for those metrics'
+# database files to all get created and thus longer until the data becomes usable.
+# Setting this value high (like "inf" for infinity) will cause graphite to create
+# the files quickly but at the risk of slowing I/O down considerably for a while.
+MAX_CREATES_PER_MINUTE = inf
+
+LINE_RECEIVER_INTERFACE = 0.0.0.0
+LINE_RECEIVER_PORT = 2003
+
+PICKLE_RECEIVER_INTERFACE = 0.0.0.0
+PICKLE_RECEIVER_PORT = 2004
+
+CACHE_QUERY_INTERFACE = 0.0.0.0
+CACHE_QUERY_PORT = 7002
+
+# By default, carbon-cache will log every whisper update. This can be excessive and
+# degrade performance if logging on the same volume as the whisper data is stored.
+LOG_UPDATES = True
+
+
+# Enable AMQP if you want to receve metrics using an amqp broker
+# ENABLE_AMQP = False
+
+# Verbose means a line will be logged for every metric received
+# useful for testing
+# AMQP_VERBOSE = False
+
+# AMQP_HOST = localhost
+# AMQP_PORT = 5672
+# AMQP_VHOST = /
+# AMQP_USER = guest
+# AMQP_PASSWORD = guest
+# AMQP_EXCHANGE = graphite
+
+# Patterns for all of the metrics this machine will store. Read more at
+# http://en.wikipedia.org/wiki/Advanced_Message_Queuing_Protocol#Bindings
+#
+# Example: store all sales, linux servers, and utilization metrics
+# BIND_PATTERNS = sales.#, servers.linux.#, #.utilization
+#
+# Example: store everything
+# BIND_PATTERNS = #
+
+# NOTE: you cannot run both a cache and a relay on the same server
+# with the default configuration, you have to specify a distinict
+# interfaces and ports for the listeners.
+
+[relay]
+LINE_RECEIVER_INTERFACE = 0.0.0.0
+LINE_RECEIVER_PORT = 2003
+
+PICKLE_RECEIVER_INTERFACE = 0.0.0.0
+PICKLE_RECEIVER_PORT = 2004
+
+CACHE_SERVERS = server1, server2, server3
+MAX_QUEUE_SIZE = 10000
@@ -0,0 +1,19 @@
+# You may need to manually edit this file to fit your needs.
+# This configuration assumes the default installation prefix
+# of /opt/graphite/, if you installed graphite somewhere else
+# you will need to change all the occurances of /opt/graphite/
+# in this file to your chosen install location.
+
+import os
+import sys
+sys.path.insert(0, '/opt/graphite/webapp/')
+os.environ['DJANGO_SETTINGS_MODULE'] = 'graphite.settings'
+
+import django.core.handlers.wsgi
+
+_application = django.core.handlers.wsgi.WSGIHandler()
+
+def application(environ, start_response):
+ environ['PATH_INFO'] = environ['SCRIPT_NAME'] + environ['PATH_INFO']
+ return _application(environ, start_response)
+
@@ -0,0 +1,2 @@
+DATABASE_NAME = "/opt/graphite/storage/webapp/graphite.db"
+
@@ -0,0 +1,17 @@
+# Assume 60 second metrics by default
+[everything]
+priority = 150
+pattern = .*
+retentions = 60:1728000,300:5184000,3600:62208000
+
+# Host metrics are every 10 seconds
+[host]
+priority = 100
+pattern = host\..*\.system\..*
+retentions = 60:1728000,300:5184000,3600:62208000
+
+# App metrics are every 10 seconds
+[app]
+priority = 90
+pattern = host\..*\.app\..*
+retentions = 60:1728000,300:5184000,3600:62208000
@@ -0,0 +1,63 @@
+var net = require("net");
+var url = require("url");
+
+function to_dotted_notation(obj, parent_key, result) {
+ if (typeof(parent_key) == 'undefined') parent_key = "";
+ if (typeof(result) == 'undefined') result = {};
+
+ if (typeof(obj) == 'object') {
+ for (var i in obj) {
+ var key = parent_key ? (parent_key + "." + i) : i;
+ to_dotted_notation(obj[i], key, result);
+ }
+ } else if (typeof(obj) == 'number') {
+ result[parent_key] = obj;
+ }
+ return result;
+}
+
+var stdin = process.openStdin();
+var input = "";
+stdin.on("data", function(chunk) {
+ input += chunk;
+});
+stdin.on("end", function() {
+ data = JSON.parse(input);
+ results = to_dotted_notation(data);
+
+ /* TODO(sissel): validate args */
+ var targeturl = url.parse(process.argv[2]);
+ var host = targeturl.hostname;
+ var port = targeturl.port || 2003;
+ var prefix = targeturl.pathname.slice(1); /* trim leading '/' */
+
+ /* Only fetch matching keys */
+ var args = process.argv.slice(3); /* argv[0] == 'node', argv[1] is script name */
+
+ /* Create a regexp of (arg)|(arg)|(arg)... */
+ var pattern = args.map(function(arg) { return "(" + arg + ")" }).join("|");
+ var re = new RegExp(pattern);
+
+ var now = Math.floor((new Date()).getTime() / 1000);
+ var messages = []
+ for (var key in results) {
+ if (re.test(key)) {
+ var fullkey = key;
+ if (prefix) {
+ fullkey = prefix + "." + key;
+ }
+ messages.push([fullkey, results[key], now].join(" "));
+ }
+ }
+
+ var graphite = net.createConnection(port, host);
+ console.log("Sending to " + host + ":" + port);
+ graphite.on('connect', function() {
+ for (var i in messages) {
+ var m = messages[i].toLowerCase();
+ graphite.write(m + "\n");
+ console.log(m);
+ }
+ graphite.end();
+ });
+});
@@ -0,0 +1,18 @@
+define graphite::curljson($url, $filter=[], $metric_prefix="",
+ $interval=10) {
+ require ::graphite::tools
+
+ file {
+ "/usr/local/bin/curljson-$name.sh":
+ ensure => file,
+ content => template("graphite/tools/curl-json-to-graphite.sh.erb"),
+ notify => Supervisor::Program["curljson-$name"],
+ mode => 755;
+ }
+
+ supervisor::program {
+ "curljson-$name":
+ command => "/usr/local/bin/curljson-$name.sh",
+ user => "nobody";
+ }
+}
@@ -0,0 +1,23 @@
+class graphite::package {
+ include ::python::package::django
+ include ::python::package::django::tagging
+ include ::python::package::sqlite
+
+ python::package {
+ "python-whisper": ensure => latest;
+ "python-carbon": ensure => latest;
+ "python-graphite-web": ensure => latest;
+ "python-zope.interface": ensure => latest;
+ #"python-sqlite3": ensure => latest;
+
+ # Use our own version of cairo
+ "python-cairo": ensure => absent;
+ }
+
+ # We built these, not ubuntu native.
+ package {
+ "python-twisted": ensure => latest;
+ "python-pycairo": ensure => latest;
+ }
+}
+
@@ -0,0 +1,100 @@
+class graphite::server {
+ include ::graphite::package
+ include ::apache::mod::wsgi
+
+ user {
+ "graphite": ensure => present;
+ }
+
+ iptables::rule {
+ "allow http (for graphite)": ports => 80;
+ "allow https (for graphite)": ports => 443;
+ }
+
+ file {
+ "/mnt/graphite":
+ ensure => directory,
+ owner => "graphite";
+ "/mnt/graphite/storage":
+ ensure => directory,
+ owner => "graphite";
+ "/mnt/graphite/storage/whisper":
+ ensure => directory,
+ owner => "graphite",
+ recurse => "true";
+ "/mnt/graphite/storage/log":
+ ensure => directory,
+ owner => "graphite";
+ "/opt/graphite/conf/carbon.conf":
+ ensure => file,
+ owner => "graphite",
+ source => "puppet:///modules/graphite/carbon.conf",
+ require => Class["graphite::package"],
+ notify => Supervisor::Program["graphite-carbon"];
+ "/opt/graphite/conf/storage-schemas.conf":
+ ensure => file,
+ owner => "graphite",
+ source => "puppet:///modules/graphite/storage-schemas.conf",
+ require => Class["graphite::package"],
+ notify => Supervisor::Program["graphite-carbon"];
+ "/opt/graphite/webapp":
+ ensure => directory;
+ "/opt/graphite/webapp/graphite/graphite.wsgi":
+ ensure => file,
+ source => "puppet:///modules/graphite/graphite.wsgi";
+ "/opt/graphite":
+ ensure => directory,
+ owner => "graphite",
+ recurse => true;
+ "/opt/graphite/storage/whisper":
+ ensure => link,
+ force => true,
+ target => "/mnt/graphite/storage/whisper";
+ "/opt/graphite/storage/log":
+ ensure => link,
+ force => true,
+ target => "/mnt/graphite/storage/log";
+ "/opt/graphite/storage/log/webapp":
+ ensure => directory,
+ owner => "www-data";
+ [ "/opt/graphite/storage/log/webapp/exception.log",
+ "/opt/graphite/storage/log/webapp/info.log" ]:
+ ensure => file,
+ owner => "www-data";
+
+ "/opt/graphite/storage/webapp":
+ ensure => directory,
+ owner => "www-data";
+ "/opt/graphite/webapp/graphite/local_settings.py":
+ ensure => file,
+ source => "puppet:///modules/graphite/local_settings.py";
+
+ # graphite-web likes to have write access to this file, but it doesn't
+ # actually write to it? Confusing. -jordan
+ "/mnt/graphite/storage/index":
+ ensure => file,
+ owner => "www-data";
+ }
+
+ iptables::rule {
+ "allow metric pushes to graphite":
+ ports => [ 2003 ],
+ roles => [ "solr", "proxy", "frontend", "graphite", "monitor" ];
+ }
+
+ supervisor::program {
+ "grophite-web":
+ user => "graphite",
+ command => "python /opt/graphite/bin/run-graphite-devel-server.py /opt/graphite",
+ ensure => absent; # Don't need this anymore, remove after 2011/12/01
+ "graphite-carbon":
+ user => "graphite",
+ command => "python /opt/graphite/bin/carbon-cache.py --debug start",
+ require => [Class["graphite::package"], File["/mnt/graphite/storage/whisper"]];
+ }
+
+ apache::site {
+ "graphite":
+ content => template("graphite/graphite.httpd.conf.erb");
+ }
+}
@@ -0,0 +1,9 @@
+class graphite::tools {
+ include ::nodejs
+
+ file {
+ "/usr/local/bin/json-to-graphite.js":
+ source => "puppet:///modules/graphite/tools/json-to-graphite.js",
+ require => Class["nodejs"];
+ }
+}
@@ -0,0 +1,38 @@
+# This configuration assumes the default installation prefix
+# of /opt/graphite/, if you installed graphite somewhere else
+# you will need to change all the occurances of /opt/graphite/
+# in this file to your chosen install location.
+
+<VirtualHost *:80>
+ ServerName graphite.example.com
+ ServerAlias graphite.<%= deployment_hostname %>
+ ServerAlias graphite.<%= deployment %>.example.com
+ ServerAlias graphite
+
+ WSGIDaemonProcess graphite user=www-data group=www-data threads=25
+ WSGIProcessGroup graphite
+ WSGIScriptAlias / /opt/graphite/webapp/graphite/graphite.wsgi
+
+ <Directory /opt/graphite/webapp>
+ Order deny,allow
+ Allow from all
+ </Directory>
+
+ # NOTE: In order for the django admin site media to work you
+ # must change @DJANGO_ROOT@ to be the path to your django
+ # installation, which is probably something like:
+ # /usr/lib/python2.6/site-packages/django
+ Alias /media/ "/usr/lib/pymodules/python2.6/django/contrib/admin/media/"
+ <Directory /usr/lib/pymodules/python2.6/django/contrib/admin/media>
+ Order deny,allow
+ Allow from all
+ </Directory>
+
+ <Location "/content/">
+ SetHandler None
+ </Location>
+
+ <Location "/media/">
+ SetHandler None
+ </Location>
+</VirtualHost>
@@ -0,0 +1,11 @@
+#!/bin/sh
+#
+
+while true; do
+ curl "<%= url %>" \
+ | node /usr/local/bin/json-to-graphite.js \
+ graphite://monitor:2003/<%= metric_prefix %> \
+ <%= filter.join(" ") %>
+ sleep <%= interval %>
+done
+

0 comments on commit d5ac48e

Please sign in to comment.