adding a golang Template stage

docs/logentry/processing-log-lines.md

@@ -116,6 +116,10 @@ Extracting data (for use by other stages)
   * [regex](#regex) - use regex to extract data
   * [json](#json) - parse a JSON log and extract data
 
+Modifying extracted data
+
+  * [template](#template) - use Go templates to modify extracted data
+
 Filtering stages
 
   * [match](#match) - apply selectors to conditionally run stages based on labels
@@ -208,6 +212,76 @@ Would create the following `extracted` map:
 ```
 [Example in unit test](../../pkg/logentry/stages/json_test.go)
 
+#### template
+
+A template stage lets you manipulate the values in the `extracted` data map using [Go's template package](https://golang.org/pkg/text/template/).  This can be useful if you want to manipulate data extracted by regex or json stages before setting label values.  Maybe to replace all spaces with underscores or make everything lowercase, or append some values to the extracted data.
+
+You can set values in the extracted map for keys that did not previously exist.
+
+```yaml
+- template:
+    source:    ①
+    template:  ②
+```
+
+① `source` is **required** and is the key to the value in the `extracted` data map you wish to modify, this key does __not__ have to be present and will be added if missing.  
+② `template` is **required** and is a [Go template string](https://golang.org/pkg/text/template/)
+
+The value of the extracted data map is accessed by using `.Value` in your template
+
+In addition to normal template syntax, several functions have also been mapped to use directly or in a pipe configuration:
+
+```go
+"ToLower":    strings.ToLower,
+"ToUpper":    strings.ToUpper,
+"Replace":    strings.Replace,
+"Trim":       strings.Trim,
+"TrimLeft":   strings.TrimLeft,
+"TrimRight":  strings.TrimRight,
+"TrimPrefix": strings.TrimPrefix,
+"TrimSuffix": strings.TrimSuffix,
+"TrimSpace":  strings.TrimSpace,
+``` 
+
+##### Example
+
+```yaml
+- template:
+    source: app
+    template: '{{ .Value }}_some_suffix'
+```
+
+This would take the value of the `app` key in the `extracted` data map and append `_some_suffix` to it.  For example, if `app=loki` the new value for `app` in the map would be `loki_some_suffix`
+
+```yaml
+- template:
+    source: app
+    template: '{{ ToLower .Value }}'
+```
+
+This would take the value of `app` from `extracted` data and lowercase all the letters.  If `app=LOKI` the new value for `app` would be `loki`.  
+
+The template syntax passes paramters to functions using space delimiters, functions only taking a single argument can also use the pipe syntax:
+
+```yaml
+- template:
+    source: app
+    template: '{{ .Value | ToLower }}'
+```
+
+A more complicated function example:
+
+```yaml
+- template:
+    source: app
+    template: '{{ Replace .Value "loki" "bloki" 1 }}'
+```
+
+The arguments here as described for the [Replace function](https://golang.org/pkg/strings/#Replace), in this example we are saying to Replace in the string `.Value` (which is our extracted value for the `app` key) the occurrence of the string "loki" with the string "bloki" exactly 1 time.
+
+[More examples in unit test](../../pkg/logentry/stages/template_test.go)
+
+
 ### match
 
 A match stage will take the provided label `selector` and determine if a group of provided Stages will be executed or not based on labels

pkg/logentry/stages/labels_test.go

@@ -140,6 +140,14 @@ func TestLabelStage_Process(t *testing.T) {
 				"testLabel": "testValue",
 			},
 		},
+		"empty_extracted_data": {
+			LabelsConfig{
+				"testLabel": &sourceName,
+			},
+			map[string]interface{}{},
+			model.LabelSet{},
+			model.LabelSet{},
+		},
 	}
 	for name, test := range tests {
 		test := test

pkg/logentry/stages/match_test.go

@@ -119,6 +119,7 @@ func TestMatcher(t *testing.T) {
 		{"{foo=\"bar\",bar!=\"test\"}", map[string]string{"foo": "bar", "bar": "test"}, false, false},
 		{"{foo=\"bar\",bar=~\"te.*\"}", map[string]string{"foo": "bar", "bar": "test"}, true, false},
 		{"{foo=\"bar\",bar!~\"te.*\"}", map[string]string{"foo": "bar", "bar": "test"}, false, false},
+		{"{foo=\"\"}", map[string]string{}, true, false},
 	}
 
 	for _, tt := range tests {

pkg/logentry/stages/stage.go

@@ -19,6 +19,7 @@ const (
 	StageTypeDocker    = "docker"
 	StageTypeCRI       = "cri"
 	StageTypeMatch     = "match"
+	StageTypeTemplate  = "template"
 )
 
 // Stage takes an existing set of labels, timestamp and log entry and returns either a possibly mutated
@@ -86,6 +87,11 @@ func New(logger log.Logger, jobName *string, stageType string,
 		if err != nil {
 			return nil, err
 		}
+	case StageTypeTemplate:
+		s, err = newTemplateStage(logger, cfg)
+		if err != nil {
+			return nil, err
+		}
 	default:
 		return nil, errors.Errorf("Unknown stage type: %s", stageType)
 	}

pkg/logentry/stages/template.go

@@ -0,0 +1,125 @@
+package stages
+
+import (
+	"bytes"
+	"errors"
+	"reflect"
+	"strings"
+	"text/template"
+	"time"
+
+	"github.com/go-kit/kit/log"
+	"github.com/go-kit/kit/log/level"
+	"github.com/mitchellh/mapstructure"
+	"github.com/prometheus/common/model"
+)
+
+// Config Errors
+const (
+	ErrEmptyTemplateStageConfig = "template stage config cannot be empty"
+	ErrTemplateSourceRequired   = "template source value is required"
+)
+
+var (
+	functionMap = template.FuncMap{
+		"ToLower":    strings.ToLower,
+		"ToUpper":    strings.ToUpper,
+		"Replace":    strings.Replace,
+		"Trim":       strings.Trim,
+		"TrimLeft":   strings.TrimLeft,
+		"TrimRight":  strings.TrimRight,
+		"TrimPrefix": strings.TrimPrefix,
+		"TrimSuffix": strings.TrimSuffix,
+		"TrimSpace":  strings.TrimSpace,
+	}
+)
+
+// TemplateConfig configures template value extraction
+type TemplateConfig struct {
+	Source   string `mapstructure:"source"`
+	Template string `mapstructure:"template"`
+}
+
+// validateTemplateConfig validates the templateStage config
+func validateTemplateConfig(cfg *TemplateConfig) (*template.Template, error) {
+	if cfg == nil {
+		return nil, errors.New(ErrEmptyTemplateStageConfig)
+	}
+	if cfg.Source == "" {
+		return nil, errors.New(ErrTemplateSourceRequired)
+	}
+
+	return template.New("pipeline_template").Funcs(functionMap).Parse(cfg.Template)
+}
+
+// newTemplateStage creates a new templateStage
+func newTemplateStage(logger log.Logger, config interface{}) (*templateStage, error) {
+	cfg := &TemplateConfig{}
+	err := mapstructure.Decode(config, cfg)
+	if err != nil {
+		return nil, err
+	}
+	t, err := validateTemplateConfig(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return &templateStage{
+		cfgs:     cfg,
+		logger:   logger,
+		template: t,
+	}, nil
+}
+
+type templateData struct {
+	Value string
+}
+
+// templateStage will mutate the incoming entry and set it from extracted data
+type templateStage struct {
+	cfgs     *TemplateConfig
+	logger   log.Logger
+	template *template.Template
+}
+
+// Process implements Stage
+func (o *templateStage) Process(labels model.LabelSet, extracted map[string]interface{}, t *time.Time, entry *string) {
+	if o.cfgs == nil {
+		return
+	}
+	if v, ok := extracted[o.cfgs.Source]; ok {
+		s, err := getString(v)
+		if err != nil {
+			level.Debug(o.logger).Log("msg", "extracted template could not be converted to a string", "err", err, "type", reflect.TypeOf(v).String())
+			return
+		}
+		td := templateData{s}
+		buf := &bytes.Buffer{}
+		err = o.template.Execute(buf, td)
+		if err != nil {
+			level.Debug(o.logger).Log("msg", "failed to execute template on extracted value", "err", err, "value", v)
+			return
+		}
+		st := buf.String()
+		// If the template evaluates to an empty string, remove the key from the map
+		if st == "" {
+			delete(extracted, o.cfgs.Source)
+		} else {
+			extracted[o.cfgs.Source] = st
+		}
+
+	} else {
+		td := templateData{}
+		buf := &bytes.Buffer{}
+		err := o.template.Execute(buf, td)
+		if err != nil {
+			level.Debug(o.logger).Log("msg", "failed to execute template on extracted value", "err", err, "value", v)
+			return
+		}
+		st := buf.String()
+		// Do not set extracted data with empty values
+		if st != "" {
+			extracted[o.cfgs.Source] = st
+		}
+	}
+}