Implementing generic parser plugins and documentation

This constitutes a large change in how we will parse different data
formats going forward (for the plugins that support it)

This is working off @henrypfhu's changes.

DATA_FORMATS.md

@@ -0,0 +1,67 @@
+# Telegraf Data Formats
+
+There are many Telegraf plugins that are able to parse generic text data from
+a variety of sources, this includes parsing the stdout of executed scripts
+(`exec`) and parsing messages received from message brokers (`kafka_consumer`).
+
+Up until now, these plugins were statically configured to parse just a single
+data format. `exec` mostly only supported parsing JSON, and `kafka_consumer` only
+supported data in InfluxDB line-protocol.
+
+But now we are normalizing the parsing of various data formats across all
+plugins that can support it. You will be able to tell a plugin that supports
+different data formats by the presence of a `data_format` config option, for
+example, in the exec plugin:
+
+```
+[[inputs.exec]]
+  ### Commands array
+  commands = ["/tmp/test.sh", "/usr/bin/mycollector --foo=bar"]
+
+  ### measurement name suffix (for separating different commands)
+  name_suffix = "_mycollector"
+
+  ### Data format to consume. This can be "json", "influx" or "graphite" (line-protocol)
+  ### Each data format has it's own unique set of configuration options, read
+  ### more about them here:
+  ### https://github.com/influxdata/telegraf/blob/master/DATA_FORMATS.md
+  data_format = "json"
+```
+
+## Influx line-protocol Options:
+
+None!
+
+## JSON Options:
+
+```
+### List of tag names to extract from top-level of JSON server response
+tag_keys = [
+  "my_tag_1",
+  "my_tag_2"
+]
+```
+
+## Graphite Options:
+
+```
+### Below configuration will be used for data_format = "graphite", can be ignored for other data_format
+### If matching multiple measurement files, this string will be used to join the matched values.
+separator = "."
+
+### Each template line requires a template pattern.  It can have an optional
+### filter before the template and separated by spaces.  It can also have optional extra
+### tags following the template.  Multiple tags should be separated by commas and no spaces
+### similar to the line protocol format.  The can be only one default template.
+### Templates support below format:
+### 1. filter + template
+### 2. filter + template + extra tag
+### 3. filter + template with field key
+### 4. default template
+templates = [
+  "*.app env.service.resource.measurement",
+  "stats.* .host.measurement* region=us-west,agent=sensu",
+  "stats2.* .host.measurement.field",
+  "measurement*"
+]
+```

Godeps

@@ -2,10 +2,8 @@ git.eclipse.org/gitroot/paho/org.eclipse.paho.mqtt.golang.git dbd8d5c40a582eb9ad
 github.com/Shopify/sarama d37c73f2b2bce85f7fa16b6a550d26c5372892ef
 github.com/Sirupsen/logrus f7f79f729e0fbe2fcc061db48a9ba0263f588252
 github.com/amir/raidman 6a8e089bbe32e6b907feae5ba688841974b3c339
-github.com/armon/go-metrics 345426c77237ece5dab0e1605c3e4b35c3f54757
 github.com/aws/aws-sdk-go 87b1e60a50b09e4812dee560b33a238f67305804
 github.com/beorn7/perks b965b613227fddccbfffe13eae360ed3fa822f8d
-github.com/boltdb/bolt ee4a0888a9abe7eefe5a0992ca4cb06864839873
 github.com/cenkalti/backoff 4dc77674aceaabba2c7e3da25d4c823edfb73f99
 github.com/dancannon/gorethink 6f088135ff288deb9d5546f4c71919207f891a70
 github.com/davecgh/go-spew 5215b55f46b2b919f50a1df0eaa5886afe4e3b3d
@@ -14,16 +12,12 @@ github.com/eapache/queue ded5959c0d4e360646dc9e9908cff48666781367
 github.com/fsouza/go-dockerclient 7b651349f9479f5114913eefbfd3c4eeddd79ab4
 github.com/go-ini/ini afbd495e5aaea13597b5e14fe514ddeaa4d76fc3
 github.com/go-sql-driver/mysql 7c7f556282622f94213bc028b4d0a7b6151ba239
-github.com/gogo/protobuf e8904f58e872a473a5b91bc9bf3377d223555263
 github.com/golang/protobuf 6aaa8d47701fa6cf07e914ec01fde3d4a1fe79c3
 github.com/golang/snappy 723cc1e459b8eea2dea4583200fd60757d40097a
 github.com/gonuts/go-shellquote e842a11b24c6abfb3dd27af69a17f482e4b483c2
 github.com/gorilla/context 1c83b3eabd45b6d76072b66b746c20815fb2872d
 github.com/gorilla/mux 26a6070f849969ba72b72256e9f14cf519751690
 github.com/hailocab/go-hostpool e80d13ce29ede4452c43dea11e79b9bc8a15b478
-github.com/hashicorp/go-msgpack fa3f63826f7c23912c15263591e65d54d080b458
-github.com/hashicorp/raft 057b893fd996696719e98b6c44649ea14968c811
-github.com/hashicorp/raft-boltdb d1e82c1ec3f15ee991f7cc7ffd5b67ff6f5bbaee
 github.com/influxdata/config bae7cb98197d842374d3b8403905924094930f24
 github.com/influxdata/influxdb 697f48b4e62e514e701ffec39978b864a3c666e6
 github.com/influxdb/influxdb 697f48b4e62e514e701ffec39978b864a3c666e6
@@ -56,4 +50,4 @@ golang.org/x/text 6d3c22c4525a4da167968fa2479be5524d2e8bd0
 gopkg.in/dancannon/gorethink.v1 6f088135ff288deb9d5546f4c71919207f891a70
 gopkg.in/fatih/pool.v2 cba550ebf9bce999a02e963296d4bc7a486cb715
 gopkg.in/mgo.v2 03c9f3ee4c14c8e51ee521a6a7d0425658dd6f64
-gopkg.in/yaml.v2 f7716cbe52baa25d2e9b0d0da546fcf909fc16b4
+gopkg.in/yaml.v2 f7716cbe52baa25d2e9b0d0da546fcf909fc16b4

cmd/telegraf/telegraf.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/influxdata/telegraf/agent"
 	"github.com/influxdata/telegraf/internal/config"
+
 	_ "github.com/influxdata/telegraf/plugins/inputs/all"
 	_ "github.com/influxdata/telegraf/plugins/outputs/all"
 )

internal/internal.go

@@ -9,7 +9,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"strconv"
 	"strings"
 	"time"
 )
@@ -35,47 +34,6 @@ func (d *Duration) UnmarshalTOML(b []byte) error {
 
 var NotImplementedError = errors.New("not implemented yet")
 
-type JSONFlattener struct {
-	Fields map[string]interface{}
-}
-
-// FlattenJSON flattens nested maps/interfaces into a fields map
-func (f *JSONFlattener) FlattenJSON(
-	fieldname string,
-	v interface{},
-) error {
-	if f.Fields == nil {
-		f.Fields = make(map[string]interface{})
-	}
-	fieldname = strings.Trim(fieldname, "_")
-	switch t := v.(type) {
-	case map[string]interface{}:
-		for k, v := range t {
-			err := f.FlattenJSON(fieldname+"_"+k+"_", v)
-			if err != nil {
-				return err
-			}
-		}
-	case []interface{}:
-		for i, v := range t {
-			k := strconv.Itoa(i)
-			err := f.FlattenJSON(fieldname+"_"+k+"_", v)
-			if err != nil {
-				return nil
-			}
-		}
-	case float64:
-		f.Fields[fieldname] = t
-	case bool, string, nil:
-		// ignored types
-		return nil
-	default:
-		return fmt.Errorf("JSON Flattener: got unexpected type %T with value %v (%s)",
-			t, t, fieldname)
-	}
-	return nil
-}
-
 // ReadLines reads contents from a file and splits them by new lines.
 // A convenience wrapper to ReadLinesOffsetN(filename, 0, -1).
 func ReadLines(filename string) ([]string, error) {

plugins/inputs/elasticsearch/elasticsearch.go

@@ -10,8 +10,8 @@ import (
 	"time"
 
 	"github.com/influxdata/telegraf"
-	"github.com/influxdata/telegraf/internal"
 	"github.com/influxdata/telegraf/plugins/inputs"
+	jsonparser "github.com/influxdata/telegraf/plugins/parsers/json"
 )
 
 const statsPath = "/_nodes/stats"
@@ -168,7 +168,7 @@ func (e *Elasticsearch) gatherNodeStats(url string, acc telegraf.Accumulator) er
 
 		now := time.Now()
 		for p, s := range stats {
-			f := internal.JSONFlattener{}
+			f := jsonparser.JSONFlattener{}
 			err := f.FlattenJSON("", s)
 			if err != nil {
 				return err