use templates for labels and values

config/config.go

@@ -48,12 +48,12 @@ func LoadConfigString(content []byte) (*v2.Config, string, error) {
 }
 
 func findVersion(content []byte) (int, string, error) {
-	versionExpr := regexp.MustCompile(`general:\s*config_version:[\t\f ]*(\S+)`)
+	versionExpr := regexp.MustCompile(`global:\s*config_version:[\t\f ]*(\S+)`)
 	versionInfo := versionExpr.FindStringSubmatch(string(content))
 	if len(versionInfo) == 2 {
 		version, err := strconv.Atoi(strings.TrimSpace(versionInfo[1]))
 		if err != nil {
-			return 0, "", fmt.Errorf("invalid 'general' configuration: '%v' is not a valid 'config_version'.", versionInfo[1])
+			return 0, "", fmt.Errorf("invalid 'global' configuration: '%v' is not a valid 'config_version'.", versionInfo[1])
 		}
 		return version, "", nil
 	} else { // no version found

config/config_test.go

@@ -20,7 +20,7 @@ import (
 )
 
 const exampleConfig = `
-general:
+global:
     config_version: 2
 input:
     type: file

config/v1/configV1.go

@@ -32,8 +32,7 @@ func Unmarshal(config []byte) (*v2.Config, error) {
 		Metrics: convertMetrics(*v1cfg.Metrics),
 		Server:  v1cfg.Server,
 	}
-	v2cfg.AddDefaults()
-	err = v2cfg.Validate()
+	err = v2.AddDefaultsAndValidate(v2cfg)
 	if err != nil {
 		return nil, err
 	}
@@ -51,25 +50,30 @@ func convertMetrics(v1metrics []*MetricConfig) *v2.MetricsConfig {
 			Name:       v1metric.Name,
 			Help:       v1metric.Help,
 			Match:      v1metric.Match,
-			Value:      v1metric.Value,
+			Value:      makeTemplate(v1metric.Value),
 			Cumulative: v1metric.Cumulative,
 			Buckets:    v1metric.Buckets,
 			Quantiles:  v1metric.Quantiles,
 		}
 		if len(v1metric.Labels) > 0 {
-			v2metrics[i].Labels = make([]v2.Label, len(v1metric.Labels))
-			for j, v1label := range v1metric.Labels {
-				v2metrics[i].Labels[j] = v2.Label{
-					GrokFieldName:   v1label.GrokFieldName,
-					PrometheusLabel: v1label.PrometheusLabel,
-				}
+			v2metrics[i].Labels = make(map[string]string, len(v1metric.Labels))
+			for _, v1label := range v1metric.Labels {
+				v2metrics[i].Labels[v1label.PrometheusLabel] = makeTemplate(v1label.GrokFieldName)
 			}
 		}
 	}
 	result := v2.MetricsConfig(v2metrics)
 	return &result
 }
 
+func makeTemplate(grokFieldName string) string {
+	if len(grokFieldName) > 0 {
+		return fmt.Sprintf("{{.%v}}", grokFieldName)
+	} else {
+		return ""
+	}
+}
+
 type Config struct {
 	// For sections that don't differ between v1 and v2, we reference v2 directly here.
 	Input   *v2.InputConfig  `yaml:",omitempty"`

config/v1/configV1_test.go

@@ -32,10 +32,10 @@ metrics:
       help: Dummy help message for counter.
       match: Some text here, then a %{DATE}.
       labels:
-          - grok_field_name: a
-            prometheus_label: b
-          - grok_field_name: c
-            prometheus_label: d
+          - grok_field_name: grok_field_a
+            prometheus_label: prom_label_a
+          - grok_field_name: grok_field_b
+            prometheus_label: prom_label_b
     - type: gauge
       name: test_gauge
       help: Dummy help message for gauge.
@@ -69,7 +69,7 @@ server:
 `
 
 const expected = `
-general:
+global:
     config_version: 2
 input:
     type: file
@@ -83,37 +83,32 @@ metrics:
       help: Dummy help message for counter.
       match: Some text here, then a %{DATE}.
       labels:
-          - grok_field_name: a
-            prometheus_label: b
-          - grok_field_name: c
-            prometheus_label: d
+        prom_label_a: '{{.grok_field_a}}'
+        prom_label_b: '{{.grok_field_b}}'
     - type: gauge
       name: test_gauge
       help: Dummy help message for gauge.
       match: '%{DATE} %{TIME} %{USER:user} %{NUMBER:val}'
-      value: val
+      value: '{{.val}}'
       cumulative: true
       labels:
-          - grok_field_name: user
-            prometheus_label: user
+        user: '{{.user}}'
     - type: histogram
       name: test_histogram
       help: Dummy help message for histogram.
       match: '%{DATE} %{TIME} %{USER:user} %{NUMBER:val}'
-      value: val
+      value: '{{.val}}'
       buckets: [1, 2, 3]
       labels:
-          - grok_field_name: user
-            prometheus_label: user
+        user: '{{.user}}'
     - type: summary
       name: test_summary
       help: Dummy help message for summary.
       match: '%{DATE} %{TIME} %{USER:user} %{NUMBER:val}'
-      value: val
+      value: '{{.val}}'
       quantiles: {0.5: 0.05, 0.9: 0.01, 0.99: 0.001}
       labels:
-          - grok_field_name: user
-            prometheus_label: user
+        user: '{{.user}}'
 server:
     protocol: https
     port: 1111

config/v2/configV2.go

@@ -17,16 +17,16 @@ package v2
 import (
 	"fmt"
 	"gopkg.in/yaml.v2"
+	"text/template"
 )
 
 func Unmarshal(config []byte) (*Config, error) {
 	cfg := &Config{}
 	err := yaml.Unmarshal(config, cfg)
 	if err != nil {
-		return nil, fmt.Errorf("invalid configuration: %v", err.Error())
+		return nil, fmt.Errorf("invalid configuration: %v. make sure to use 'single quotes' around strings with special characters (like match patterns or label templates), and make sure to use '-' only for lists (metrics) but not for maps (labels).", err.Error())
 	}
-	cfg.AddDefaults()
-	err = cfg.Validate()
+	err = AddDefaultsAndValidate(cfg)
 	if err != nil {
 		return nil, err
 	}
@@ -41,7 +41,7 @@ func (cfg *Config) String() string {
 	return string(out)
 }
 
-type GeneralConfig struct {
+type GlobalConfig struct {
 	ConfigVersion int `yaml:"config_version,omitempty"`
 }
 
@@ -56,21 +56,18 @@ type GrokConfig struct {
 	AdditionalPatterns []string `yaml:"additional_patterns,omitempty"`
 }
 
-type Label struct {
-	GrokFieldName   string `yaml:"grok_field_name,omitempty"`
-	PrometheusLabel string `yaml:"prometheus_label,omitempty"`
-}
-
 type MetricConfig struct {
-	Type       string              `yaml:",omitempty"`
-	Name       string              `yaml:",omitempty"`
-	Help       string              `yaml:",omitempty"`
-	Match      string              `yaml:",omitempty"`
-	Value      string              `yaml:",omitempty"`
-	Cumulative bool                `yaml:",omitempty"`
-	Buckets    []float64           `yaml:",flow,omitempty"`
-	Quantiles  map[float64]float64 `yaml:",flow,omitempty"`
-	Labels     []Label             `yaml:",omitempty"`
+	Type           string               `yaml:",omitempty"`
+	Name           string               `yaml:",omitempty"`
+	Help           string               `yaml:",omitempty"`
+	Match          string               `yaml:",omitempty"`
+	Value          string               `yaml:",omitempty"`
+	Cumulative     bool                 `yaml:",omitempty"`
+	Buckets        []float64            `yaml:",flow,omitempty"`
+	Quantiles      map[float64]float64  `yaml:",flow,omitempty"`
+	Labels         map[string]string    `yaml:",omitempty"`
+	LabelTemplates []*template.Template `yaml:"-"` // parsed version of Labels, will not be serialized to yaml.
+	ValueTemplate  *template.Template   `yaml:"-"` // parsed version of Value, will not be serialized to yaml.
 }
 
 type MetricsConfig []*MetricConfig
@@ -84,18 +81,18 @@ type ServerConfig struct {
 }
 
 type Config struct {
-	General *GeneralConfig `yaml:",omitempty"`
+	Global  *GlobalConfig  `yaml:",omitempty"`
 	Input   *InputConfig   `yaml:",omitempty"`
 	Grok    *GrokConfig    `yaml:",omitempty"`
 	Metrics *MetricsConfig `yaml:",omitempty"`
 	Server  *ServerConfig  `yaml:",omitempty"`
 }
 
-func (cfg *Config) AddDefaults() {
-	if cfg.General == nil {
-		cfg.General = &GeneralConfig{}
+func (cfg *Config) addDefaults() {
+	if cfg.Global == nil {
+		cfg.Global = &GlobalConfig{}
 	}
-	cfg.General.addDefaults()
+	cfg.Global.addDefaults()
 	if cfg.Input == nil {
 		cfg.Input = &InputConfig{}
 	}
@@ -115,7 +112,7 @@ func (cfg *Config) AddDefaults() {
 	cfg.Server.addDefaults()
 }
 
-func (c *GeneralConfig) addDefaults() {
+func (c *GlobalConfig) addDefaults() {
 	if c.ConfigVersion == 0 {
 		c.ConfigVersion = 2
 	}
@@ -140,7 +137,7 @@ func (c *ServerConfig) addDefaults() {
 	}
 }
 
-func (cfg *Config) Validate() error {
+func (cfg *Config) validate() error {
 	err := cfg.Input.validate()
 	if err != nil {
 		return err
@@ -238,27 +235,10 @@ func (c *MetricConfig) validate() error {
 	case !quantilesAllowed && len(c.Quantiles) > 0:
 		return fmt.Errorf("Invalid metric configuration: 'metrics.buckets' cannot be used for %v metrics.", c.Type)
 	}
-	// Labels are optionally supported for all metric types.
-	for _, label := range c.Labels {
-		err := label.validate()
-		if err != nil {
-			return err
-		}
-	}
+	// Labels and value are validated in InitTemplates()
 	return nil
 }
 
-func (l *Label) validate() error {
-	switch {
-	case l.GrokFieldName == "":
-		return fmt.Errorf("Invalid metrics configuration: 'metrics.label.grok_field_name' must not be empty.")
-	case l.PrometheusLabel == "":
-		return fmt.Errorf("Invalid metrics configuration: 'metrics.label.prometheus_label' must not be empty.")
-	default:
-		return nil
-	}
-}
-
 func (c *ServerConfig) validate() error {
 	switch {
 	case c.Protocol != "https" && c.Protocol != "http":
@@ -279,3 +259,41 @@ func (c *ServerConfig) validate() error {
 	}
 	return nil
 }
+
+// Made this public so it can be called when converting config v1 to config v2.
+func AddDefaultsAndValidate(cfg *Config) error {
+	var err error
+	cfg.addDefaults()
+	for _, metric := range []*MetricConfig(*cfg.Metrics) {
+		err = metric.InitTemplates()
+		if err != nil {
+			return err
+		}
+	}
+	return cfg.validate()
+}
+
+// Made this public so MetricConfig can be initialized in tests.
+func (metric *MetricConfig) InitTemplates() error {
+	var (
+		err   error
+		tmplt *template.Template
+		msg   = "invalid configuration: failed to read metric %v: error parsing %v template: %v: " +
+			"don't forget to put a . (dot) in front of grok fields, otherwise it will be interpreted as a function."
+	)
+	metric.LabelTemplates = make([]*template.Template, 0, len(metric.Labels))
+	for name, templateString := range metric.Labels {
+		tmplt, err = template.New(name).Parse(templateString)
+		if err != nil {
+			return fmt.Errorf(msg, fmt.Sprintf("label %v", metric.Name), name, err.Error())
+		}
+		metric.LabelTemplates = append(metric.LabelTemplates, tmplt)
+	}
+	if len(metric.Value) > 0 {
+		metric.ValueTemplate, err = template.New("__value__").Parse(metric.Value)
+		if err != nil {
+			return fmt.Errorf(msg, "value", metric.Name, err.Error())
+		}
+	}
+	return nil
+}