Fix collectd metric processing

collectd-graphite-proxy.js

@@ -15,10 +15,34 @@
  * This code below will then convert the PUTVAL statements to graphite metrics
  * and ship them to 'monitor:2003'
  */
-
 var http = require("http");
 var net = require("net");
 var assert = require("assert");
+var fs = require('fs');
+
+var types = fs.readFileSync('/usr/share/collectd/types.db', encoding='utf8').split("\n");
+
+var typesObj = new Object;
+
+var type_comments_re = /^#/;
+var type_cut_re = /^([^\s]+)\s+(.*)/;
+
+for (var i in types) {
+  if (!type_comments_re.exec(types[i])) {
+    typeSet = type_cut_re.exec(types[i])
+    if (!typeSet) { continue; }
+    for (var t=0;t < typeSet.length;t++) {
+      var name = typeSet[1];
+      typesObj[name] = new Array();
+      var eachType = typeSet[2].split(", ")
+      for (var u=0; u < eachType.length; u++){
+        var theName = eachType[u].split(":")[0];
+        typesObj[name].push(theName);
+      }
+    }
+  }
+}
+
 
 try {
   var graphite_connection = net.createConnection(2003, host=process.argv[2]);
@@ -33,23 +57,78 @@ graphite_connection.on("error", function() {
 });
 
 var request_handler = function(request, response) {
-  var putval_re = /^PUTVAL ([^ ]+)(?: ([^ ]+=[^ ]+)?) ([0-9]+)(:(?:-?[0-9.]+)+)/;
+  var putval_re = /^PUTVAL ([^ ]+)(?: ([^ ]+=[^ ]+)?) ([0-9.]+)(:.*)/;
   request.addListener("data", function(chunk) {
-    metrics = chunk.toString().split("\n")
+    metrics = chunk.toString().split("\n");
     for (var i in metrics) {
       var m = putval_re.exec(metrics[i]);
       if (!m) {
         continue;
       }
-      var name = m[1];
-      var options = m[2];
-      var time = m[3];
-      var value = m[4].replace(/:/, "");
-
-      name = "collectd." + name.replace(/\./g, "_").replace(/\//g, ".");
-      message = [name, value, time].join(" ")
-      console.log(message)
-      graphite_connection.write(message + "\n");
+      var values = m[4].split(":");
+
+      for (var v in values) {
+
+        var name = m[1];
+        var options = m[2];
+        var time = m[3];
+
+        if ( v == 0 ) {
+          continue;
+        }
+
+        // Replace some chars for graphite, split into parts
+        var name_parts = name.replace(/\./g, "_").replace(/\//g, ".").split(".");
+
+        // Start to construct the new name
+        var rebuild = ["agents"]
+
+        var host = name_parts[0].split(/_/)[0]
+        rebuild = rebuild.concat(host)
+
+        // Pluigin names can contain an "instance" which is set apart by a dash
+        var plugin = name_parts[1].split("-")
+        rebuild = rebuild.concat(plugin[0])
+        if (plugin.length > 1) {
+          var plugin_instance = plugin.slice(1).join("-")
+          rebuild = rebuild.concat(plugin_instance)
+        }
+        plugin = plugin[0]
+
+        // Type names can also contain an "instance"
+        var type = name_parts[2].split("-")
+        if (type[0] != plugin) {
+          // If type and plugin are equal, delete one to clean up a bit
+          rebuild = rebuild.concat(type[0])
+        }
+        if (type.length > 1) {
+          var type_instance = type.slice(1).join("-")
+          rebuild = rebuild.concat(type_instance)
+        }
+        type = type[0]
+
+        // Put the name back together
+        name = rebuild.join(".")
+
+        if ( values.length > 2 ) {
+          var metric = name_parts[2];
+
+          //  If the metric contains a '-' (after removing the instance name)
+          //  then we want to remove it before looking up in the types.db
+          index = metric.search(/-/)
+          if (index > -1) {
+            metric = /^([\w]+)-(.*)$/.exec(metric);
+          } else {
+            // Kinda a hack
+            metric = [ "", metric]
+          }
+          name = name + "." + typesObj[metric[1]][v - 1];
+        }
+        message = [name, values[v], time].join(" ");
+        graphite_connection.write(message + "\n");
+
+      }
+
     }
   });
 

types.db

@@ -0,0 +1,187 @@
+absolute		value:ABSOLUTE:0:U
+apache_bytes		value:DERIVE:0:U
+apache_connections	value:GAUGE:0:65535
+apache_idle_workers	value:GAUGE:0:65535
+apache_requests		value:DERIVE:0:U
+apache_scoreboard	value:GAUGE:0:65535
+ath_nodes		value:GAUGE:0:65535
+ath_stat		value:DERIVE:0:U
+bitrate			value:GAUGE:0:4294967295
+bytes			value:GAUGE:0:U
+cache_operation		value:DERIVE:0:U
+cache_ratio		value:GAUGE:0:100
+cache_result		value:DERIVE:0:U
+cache_size		value:GAUGE:0:4294967295
+charge			value:GAUGE:0:U
+compression_ratio	value:GAUGE:0:2
+compression		uncompressed:DERIVE:0:U, compressed:DERIVE:0:U
+connections		value:DERIVE:0:U
+conntrack		value:GAUGE:0:4294967295
+contextswitch		value:DERIVE:0:U
+counter			value:COUNTER:U:U
+cpufreq			value:GAUGE:0:U
+cpu			value:DERIVE:0:U
+current_connections	value:GAUGE:0:U
+current_sessions	value:GAUGE:0:U
+current			value:GAUGE:U:U
+delay			value:GAUGE:-1000000:1000000
+derive			value:DERIVE:0:U
+df_complex		value:GAUGE:0:U
+df_inodes		value:GAUGE:0:U
+df			used:GAUGE:0:1125899906842623, free:GAUGE:0:1125899906842623
+disk_latency		read:GAUGE:0:U, write:GAUGE:0:U
+disk_merged		read:DERIVE:0:U, write:DERIVE:0:U
+disk_octets		read:DERIVE:0:U, write:DERIVE:0:U
+disk_ops_complex	value:DERIVE:0:U
+disk_ops		read:DERIVE:0:U, write:DERIVE:0:U
+disk_time		read:DERIVE:0:U, write:DERIVE:0:U
+dns_answer		value:DERIVE:0:U
+dns_notify		value:DERIVE:0:U
+dns_octets		queries:DERIVE:0:U, responses:DERIVE:0:U
+dns_opcode		value:DERIVE:0:U
+dns_qtype_cached	value:GAUGE:0:4294967295
+dns_qtype		value:DERIVE:0:U
+dns_query		value:DERIVE:0:U
+dns_question		value:DERIVE:0:U
+dns_rcode		value:DERIVE:0:U
+dns_reject		value:DERIVE:0:U
+dns_request		value:DERIVE:0:U
+dns_resolver		value:DERIVE:0:U
+dns_response		value:DERIVE:0:U
+dns_transfer		value:DERIVE:0:U
+dns_update		value:DERIVE:0:U
+dns_zops		value:DERIVE:0:U
+email_check		value:GAUGE:0:U
+email_count		value:GAUGE:0:U
+email_size		value:GAUGE:0:U
+entropy			value:GAUGE:0:4294967295
+fanspeed		value:GAUGE:0:U
+file_size		value:GAUGE:0:U
+files			value:GAUGE:0:U
+fork_rate		value:DERIVE:0:U
+frequency		value:GAUGE:0:U
+frequency_offset	value:GAUGE:-1000000:1000000
+fscache_stat		value:DERIVE:0:U
+gauge			value:GAUGE:U:U
+http_request_methods	value:DERIVE:0:U
+http_requests		value:DERIVE:0:U
+http_response_codes	value:DERIVE:0:U
+humidity		value:GAUGE:0:100
+if_collisions		value:DERIVE:0:U
+if_dropped		rx:DERIVE:0:U, tx:DERIVE:0:U
+if_errors		rx:DERIVE:0:U, tx:DERIVE:0:U
+if_multicast		value:DERIVE:0:U
+if_octets		rx:DERIVE:0:U, tx:DERIVE:0:U
+if_packets		rx:DERIVE:0:U, tx:DERIVE:0:U
+if_rx_errors		value:DERIVE:0:U
+if_tx_errors		value:DERIVE:0:U
+invocations		value:DERIVE:0:U
+io_octets		rx:DERIVE:0:U, tx:DERIVE:0:U
+io_packets		rx:DERIVE:0:U, tx:DERIVE:0:U
+ipt_bytes		value:DERIVE:0:U
+ipt_packets		value:DERIVE:0:U
+irq			value:DERIVE:0:U
+latency			value:GAUGE:0:65535
+links			value:GAUGE:0:U
+load			shortterm:GAUGE:0:100, midterm:GAUGE:0:100, longterm:GAUGE:0:100
+memcached_command	value:DERIVE:0:U
+memcached_connections	value:GAUGE:0:U
+memcached_items		value:GAUGE:0:U
+memcached_octets	rx:DERIVE:0:U, tx:DERIVE:0:U
+memcached_ops		value:DERIVE:0:U
+memory			value:GAUGE:0:281474976710656
+multimeter		value:GAUGE:U:U
+mysql_commands		value:DERIVE:0:U
+mysql_handler		value:DERIVE:0:U
+mysql_locks		value:DERIVE:0:U
+mysql_log_position	value:DERIVE:0:U
+mysql_octets		rx:DERIVE:0:U, tx:DERIVE:0:U
+nfs_procedure		value:DERIVE:0:U
+nginx_connections	value:GAUGE:0:U
+nginx_requests		value:DERIVE:0:U
+node_octets		rx:DERIVE:0:U, tx:DERIVE:0:U
+node_rssi		value:GAUGE:0:255
+node_stat		value:DERIVE:0:U
+node_tx_rate		value:GAUGE:0:127
+operations		value:DERIVE:0:U
+percent			value:GAUGE:0:100.1
+pg_blks			value:DERIVE:0:U
+pg_db_size		value:GAUGE:0:U
+pg_n_tup_c		value:DERIVE:0:U
+pg_n_tup_g		value:GAUGE:0:U
+pg_numbackends		value:GAUGE:0:U
+pg_scan			value:DERIVE:0:U
+pg_xact			value:DERIVE:0:U
+ping_droprate		value:GAUGE:0:100
+ping			value:GAUGE:0:65535
+ping_stddev		value:GAUGE:0:65535
+players			value:GAUGE:0:1000000
+power			value:GAUGE:0:U
+protocol_counter	value:DERIVE:0:U
+ps_code			value:GAUGE:0:9223372036854775807
+ps_count		processes:GAUGE:0:1000000, threads:GAUGE:0:1000000
+ps_cputime		user:DERIVE:0:U, syst:DERIVE:0:U
+ps_data			value:GAUGE:0:9223372036854775807
+ps_disk_octets		read:DERIVE:0:U, write:DERIVE:0:U
+ps_disk_ops		read:DERIVE:0:U, write:DERIVE:0:U
+ps_pagefaults		minflt:DERIVE:0:U, majflt:DERIVE:0:U
+ps_rss			value:GAUGE:0:9223372036854775807
+ps_stacksize		value:GAUGE:0:9223372036854775807
+ps_state		value:GAUGE:0:65535
+ps_vm			value:GAUGE:0:9223372036854775807
+queue_length		value:GAUGE:0:U
+records			value:GAUGE:0:U
+requests		value:GAUGE:0:U
+response_time		value:GAUGE:0:U
+route_etx		value:GAUGE:0:U
+route_metric		value:GAUGE:0:U
+routes			value:GAUGE:0:U
+serial_octets		rx:DERIVE:0:U, tx:DERIVE:0:U
+signal_noise		value:GAUGE:U:0
+signal_power		value:GAUGE:U:0
+signal_quality		value:GAUGE:0:U
+snr			value:GAUGE:0:U
+spam_check		value:GAUGE:0:U
+spam_score		value:GAUGE:U:U
+swap_io			value:DERIVE:0:U
+swap			value:GAUGE:0:1099511627776
+tcp_connections		value:GAUGE:0:4294967295
+temperature		value:GAUGE:-273.15:U
+threads			value:GAUGE:0:U
+time_dispersion		value:GAUGE:-1000000:1000000
+timeleft		value:GAUGE:0:3600
+time_offset		value:GAUGE:-1000000:1000000
+total_bytes		value:DERIVE:0:U
+total_connections	value:DERIVE:0:U
+total_operations	value:DERIVE:0:U
+total_requests		value:DERIVE:0:U
+total_sessions		value:DERIVE:0:U
+total_threads		value:DERIVE:0:U
+total_time_in_ms	value:DERIVE:0:U
+total_values		value:DERIVE:0:U
+uptime			value:GAUGE:0:4294967295
+users			value:GAUGE:0:65535
+vcpu			value:GAUGE:0:U
+virt_cpu_total		value:DERIVE:0:U
+virt_vcpu		value:DERIVE:0:U
+vmpage_action		value:DERIVE:0:U
+vmpage_faults		minflt:DERIVE:0:U, majflt:DERIVE:0:U
+vmpage_io		in:DERIVE:0:U, out:DERIVE:0:U
+vmpage_number		value:GAUGE:0:4294967295
+volatile_changes	value:GAUGE:0:U
+voltage_threshold	value:GAUGE:U:U, threshold:GAUGE:U:U
+voltage			value:GAUGE:U:U
+vs_memory		value:GAUGE:0:9223372036854775807
+vs_processes		value:GAUGE:0:65535
+vs_threads		value:GAUGE:0:65535
+#
+# Legacy types
+# (required for the v5 upgrade target)
+#
+arc_counts		demand_data:COUNTER:0:U, demand_metadata:COUNTER:0:U, prefetch_data:COUNTER:0:U, prefetch_metadata:COUNTER:0:U
+arc_l2_bytes		read:COUNTER:0:U, write:COUNTER:0:U
+arc_l2_size		value:GAUGE:0:U
+arc_ratio		value:GAUGE:0:U
+arc_size		current:GAUGE:0:U, target:GAUGE:0:U, minlimit:GAUGE:0:U, maxlimit:GAUGE:0:U
+mysql_qcache		hits:COUNTER:0:U, inserts:COUNTER:0:U, not_cached:COUNTER:0:U, lowmem_prunes:COUNTER:0:U, queries_in_cache:GAUGE:0:U
+mysql_threads		running:GAUGE:0:U, connected:GAUGE:0:U, cached:GAUGE:0:U, created:COUNTER:0:U