diff --git a/config/config.go b/config/config.go index 6a84ac06..0b18ad46 100644 --- a/config/config.go +++ b/config/config.go @@ -48,12 +48,12 @@ func LoadConfigString(content []byte) (*v2.Config, string, error) { } func findVersion(content []byte) (int, string, error) { - versionExpr := regexp.MustCompile(`general:\s*config_version:[\t\f ]*(\S+)`) + versionExpr := regexp.MustCompile(`global:\s*config_version:[\t\f ]*(\S+)`) versionInfo := versionExpr.FindStringSubmatch(string(content)) if len(versionInfo) == 2 { version, err := strconv.Atoi(strings.TrimSpace(versionInfo[1])) if err != nil { - return 0, "", fmt.Errorf("invalid 'general' configuration: '%v' is not a valid 'config_version'.", versionInfo[1]) + return 0, "", fmt.Errorf("invalid 'global' configuration: '%v' is not a valid 'config_version'.", versionInfo[1]) } return version, "", nil } else { // no version found diff --git a/config/config_test.go b/config/config_test.go index 42d5d578..801a13d8 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -20,7 +20,7 @@ import ( ) const exampleConfig = ` -general: +global: config_version: 2 input: type: file diff --git a/config/v1/configV1.go b/config/v1/configV1.go index 23933068..2f5a347c 100644 --- a/config/v1/configV1.go +++ b/config/v1/configV1.go @@ -32,8 +32,7 @@ func Unmarshal(config []byte) (*v2.Config, error) { Metrics: convertMetrics(*v1cfg.Metrics), Server: v1cfg.Server, } - v2cfg.AddDefaults() - err = v2cfg.Validate() + err = v2.AddDefaultsAndValidate(v2cfg) if err != nil { return nil, err } @@ -51,18 +50,15 @@ func convertMetrics(v1metrics []*MetricConfig) *v2.MetricsConfig { Name: v1metric.Name, Help: v1metric.Help, Match: v1metric.Match, - Value: v1metric.Value, + Value: makeTemplate(v1metric.Value), Cumulative: v1metric.Cumulative, Buckets: v1metric.Buckets, Quantiles: v1metric.Quantiles, } if len(v1metric.Labels) > 0 { - v2metrics[i].Labels = make([]v2.Label, len(v1metric.Labels)) - for j, v1label := range v1metric.Labels { - v2metrics[i].Labels[j] = v2.Label{ - GrokFieldName: v1label.GrokFieldName, - PrometheusLabel: v1label.PrometheusLabel, - } + v2metrics[i].Labels = make(map[string]string, len(v1metric.Labels)) + for _, v1label := range v1metric.Labels { + v2metrics[i].Labels[v1label.PrometheusLabel] = makeTemplate(v1label.GrokFieldName) } } } @@ -70,6 +66,14 @@ func convertMetrics(v1metrics []*MetricConfig) *v2.MetricsConfig { return &result } +func makeTemplate(grokFieldName string) string { + if len(grokFieldName) > 0 { + return fmt.Sprintf("{{.%v}}", grokFieldName) + } else { + return "" + } +} + type Config struct { // For sections that don't differ between v1 and v2, we reference v2 directly here. Input *v2.InputConfig `yaml:",omitempty"` diff --git a/config/v1/configV1_test.go b/config/v1/configV1_test.go index eb25d175..3ff9b256 100644 --- a/config/v1/configV1_test.go +++ b/config/v1/configV1_test.go @@ -32,10 +32,10 @@ metrics: help: Dummy help message for counter. match: Some text here, then a %{DATE}. labels: - - grok_field_name: a - prometheus_label: b - - grok_field_name: c - prometheus_label: d + - grok_field_name: grok_field_a + prometheus_label: prom_label_a + - grok_field_name: grok_field_b + prometheus_label: prom_label_b - type: gauge name: test_gauge help: Dummy help message for gauge. @@ -69,7 +69,7 @@ server: ` const expected = ` -general: +global: config_version: 2 input: type: file @@ -83,37 +83,32 @@ metrics: help: Dummy help message for counter. match: Some text here, then a %{DATE}. labels: - - grok_field_name: a - prometheus_label: b - - grok_field_name: c - prometheus_label: d + prom_label_a: '{{.grok_field_a}}' + prom_label_b: '{{.grok_field_b}}' - type: gauge name: test_gauge help: Dummy help message for gauge. match: '%{DATE} %{TIME} %{USER:user} %{NUMBER:val}' - value: val + value: '{{.val}}' cumulative: true labels: - - grok_field_name: user - prometheus_label: user + user: '{{.user}}' - type: histogram name: test_histogram help: Dummy help message for histogram. match: '%{DATE} %{TIME} %{USER:user} %{NUMBER:val}' - value: val + value: '{{.val}}' buckets: [1, 2, 3] labels: - - grok_field_name: user - prometheus_label: user + user: '{{.user}}' - type: summary name: test_summary help: Dummy help message for summary. match: '%{DATE} %{TIME} %{USER:user} %{NUMBER:val}' - value: val + value: '{{.val}}' quantiles: {0.5: 0.05, 0.9: 0.01, 0.99: 0.001} labels: - - grok_field_name: user - prometheus_label: user + user: '{{.user}}' server: protocol: https port: 1111 diff --git a/config/v2/configV2.go b/config/v2/configV2.go index 3c128352..7fbeb647 100644 --- a/config/v2/configV2.go +++ b/config/v2/configV2.go @@ -17,16 +17,16 @@ package v2 import ( "fmt" "gopkg.in/yaml.v2" + "text/template" ) func Unmarshal(config []byte) (*Config, error) { cfg := &Config{} err := yaml.Unmarshal(config, cfg) if err != nil { - return nil, fmt.Errorf("invalid configuration: %v", err.Error()) + return nil, fmt.Errorf("invalid configuration: %v. make sure to use 'single quotes' around strings with special characters (like match patterns or label templates), and make sure to use '-' only for lists (metrics) but not for maps (labels).", err.Error()) } - cfg.AddDefaults() - err = cfg.Validate() + err = AddDefaultsAndValidate(cfg) if err != nil { return nil, err } @@ -41,7 +41,7 @@ func (cfg *Config) String() string { return string(out) } -type GeneralConfig struct { +type GlobalConfig struct { ConfigVersion int `yaml:"config_version,omitempty"` } @@ -56,21 +56,18 @@ type GrokConfig struct { AdditionalPatterns []string `yaml:"additional_patterns,omitempty"` } -type Label struct { - GrokFieldName string `yaml:"grok_field_name,omitempty"` - PrometheusLabel string `yaml:"prometheus_label,omitempty"` -} - type MetricConfig struct { - Type string `yaml:",omitempty"` - Name string `yaml:",omitempty"` - Help string `yaml:",omitempty"` - Match string `yaml:",omitempty"` - Value string `yaml:",omitempty"` - Cumulative bool `yaml:",omitempty"` - Buckets []float64 `yaml:",flow,omitempty"` - Quantiles map[float64]float64 `yaml:",flow,omitempty"` - Labels []Label `yaml:",omitempty"` + Type string `yaml:",omitempty"` + Name string `yaml:",omitempty"` + Help string `yaml:",omitempty"` + Match string `yaml:",omitempty"` + Value string `yaml:",omitempty"` + Cumulative bool `yaml:",omitempty"` + Buckets []float64 `yaml:",flow,omitempty"` + Quantiles map[float64]float64 `yaml:",flow,omitempty"` + Labels map[string]string `yaml:",omitempty"` + LabelTemplates []*template.Template `yaml:"-"` // parsed version of Labels, will not be serialized to yaml. + ValueTemplate *template.Template `yaml:"-"` // parsed version of Value, will not be serialized to yaml. } type MetricsConfig []*MetricConfig @@ -84,18 +81,18 @@ type ServerConfig struct { } type Config struct { - General *GeneralConfig `yaml:",omitempty"` + Global *GlobalConfig `yaml:",omitempty"` Input *InputConfig `yaml:",omitempty"` Grok *GrokConfig `yaml:",omitempty"` Metrics *MetricsConfig `yaml:",omitempty"` Server *ServerConfig `yaml:",omitempty"` } -func (cfg *Config) AddDefaults() { - if cfg.General == nil { - cfg.General = &GeneralConfig{} +func (cfg *Config) addDefaults() { + if cfg.Global == nil { + cfg.Global = &GlobalConfig{} } - cfg.General.addDefaults() + cfg.Global.addDefaults() if cfg.Input == nil { cfg.Input = &InputConfig{} } @@ -115,7 +112,7 @@ func (cfg *Config) AddDefaults() { cfg.Server.addDefaults() } -func (c *GeneralConfig) addDefaults() { +func (c *GlobalConfig) addDefaults() { if c.ConfigVersion == 0 { c.ConfigVersion = 2 } @@ -140,7 +137,7 @@ func (c *ServerConfig) addDefaults() { } } -func (cfg *Config) Validate() error { +func (cfg *Config) validate() error { err := cfg.Input.validate() if err != nil { return err @@ -238,27 +235,10 @@ func (c *MetricConfig) validate() error { case !quantilesAllowed && len(c.Quantiles) > 0: return fmt.Errorf("Invalid metric configuration: 'metrics.buckets' cannot be used for %v metrics.", c.Type) } - // Labels are optionally supported for all metric types. - for _, label := range c.Labels { - err := label.validate() - if err != nil { - return err - } - } + // Labels and value are validated in InitTemplates() return nil } -func (l *Label) validate() error { - switch { - case l.GrokFieldName == "": - return fmt.Errorf("Invalid metrics configuration: 'metrics.label.grok_field_name' must not be empty.") - case l.PrometheusLabel == "": - return fmt.Errorf("Invalid metrics configuration: 'metrics.label.prometheus_label' must not be empty.") - default: - return nil - } -} - func (c *ServerConfig) validate() error { switch { case c.Protocol != "https" && c.Protocol != "http": @@ -279,3 +259,41 @@ func (c *ServerConfig) validate() error { } return nil } + +// Made this public so it can be called when converting config v1 to config v2. +func AddDefaultsAndValidate(cfg *Config) error { + var err error + cfg.addDefaults() + for _, metric := range []*MetricConfig(*cfg.Metrics) { + err = metric.InitTemplates() + if err != nil { + return err + } + } + return cfg.validate() +} + +// Made this public so MetricConfig can be initialized in tests. +func (metric *MetricConfig) InitTemplates() error { + var ( + err error + tmplt *template.Template + msg = "invalid configuration: failed to read metric %v: error parsing %v template: %v: " + + "don't forget to put a . (dot) in front of grok fields, otherwise it will be interpreted as a function." + ) + metric.LabelTemplates = make([]*template.Template, 0, len(metric.Labels)) + for name, templateString := range metric.Labels { + tmplt, err = template.New(name).Parse(templateString) + if err != nil { + return fmt.Errorf(msg, fmt.Sprintf("label %v", metric.Name), name, err.Error()) + } + metric.LabelTemplates = append(metric.LabelTemplates, tmplt) + } + if len(metric.Value) > 0 { + metric.ValueTemplate, err = template.New("__value__").Parse(metric.Value) + if err != nil { + return fmt.Errorf(msg, "value", metric.Name, err.Error()) + } + } + return nil +} diff --git a/config/v2/configV2_test.go b/config/v2/configV2_test.go index f9df11e9..785b1bc3 100644 --- a/config/v2/configV2_test.go +++ b/config/v2/configV2_test.go @@ -20,7 +20,7 @@ import ( ) const counter_config = ` -general: +global: config_version: 2 input: type: file @@ -34,17 +34,15 @@ metrics: help: Dummy help message. match: Some text here, then a %{DATE}. labels: - - grok_field_name: a - prometheus_label: b - - grok_field_name: c - prometheus_label: d + label_a: '{{.some_grok_field_a}}' + label_b: '{{.some_grok_field_b}}' server: protocol: https port: 1111 ` const gauge_config = ` -general: +global: config_version: 2 input: type: stdin @@ -54,8 +52,8 @@ metrics: - type: gauge name: test_histogram help: Dummy help message. - match: Some text here, then a %{DATE}. - value: val + match: Some %{NUMBER:val} here, then a %{DATE}. + value: '{{.val}}' cumulative: true server: protocol: http @@ -64,7 +62,7 @@ server: ` const histogram_config = ` -general: +global: config_version: 2 input: type: stdin @@ -74,8 +72,8 @@ metrics: - type: histogram name: test_histogram help: Dummy help message. - match: Some text here, then a %{DATE}. - value: val + match: Some %{NUMBER:val} here, then a %{DATE}. + value: '{{.val}}' buckets: $BUCKETS server: protocol: http @@ -83,7 +81,7 @@ server: ` const summary_config = ` -general: +global: config_version: 2 input: type: stdin @@ -93,8 +91,8 @@ metrics: - type: summary name: test_summary help: Dummy help message. - match: Some text here, then a %{DATE}. - value: val + match: Some %{NUMBER:val} here, then a %{DATE}. + value: '{{.val}}' quantiles: $QUANTILES server: protocol: http @@ -110,7 +108,7 @@ func TestGaugeValidConfig(t *testing.T) { } func TestGaugeInvalidConfig(t *testing.T) { - invalidCfg := strings.Replace(gauge_config, " value: val\n", "", 1) + invalidCfg := strings.Replace(gauge_config, " value: '{{.val}}'\n", "", 1) _, err := Unmarshal([]byte(invalidCfg)) if err == nil || !strings.Contains(err.Error(), "'metrics.value' must not be empty") { t.Fatal("Expected error message saying that value is missing.") @@ -174,6 +172,14 @@ func TestSummaryInvalidConfig(t *testing.T) { } } +func TestValueInvalidTemplate(t *testing.T) { + invalidCfg := strings.Replace(gauge_config, "value: '{{.val}}'", "value: '{{val}}'", 1) + _, err := Unmarshal([]byte(invalidCfg)) + if err == nil { + t.Fatal("Expected error, because using {{val}} instead of {{.val}}.") + } +} + func loadOrFail(t *testing.T, cfgString string) *Config { cfg, err := Unmarshal([]byte(cfgString)) if err != nil { diff --git a/exporter/grok.go b/exporter/grok.go index 0e052c1d..d8ee5f00 100644 --- a/exporter/grok.go +++ b/exporter/grok.go @@ -35,14 +35,18 @@ func Compile(pattern string, patterns *Patterns, libonig *OnigurumaLib) (*Onigur } func VerifyFieldNames(m *v2.MetricConfig, regex *OnigurumaRegexp) error { - for _, label := range m.Labels { - if !regex.HasCaptureGroup(label.GrokFieldName) { - return fmt.Errorf("grok field %v not found in match pattern", label.GrokFieldName) + for _, template := range m.LabelTemplates { + for _, grokFieldName := range referencedGrokFields(template) { + if !regex.HasCaptureGroup(grokFieldName) { + return fmt.Errorf("%v: error in label %v: grok field %v not found in match pattern", m.Name, template.Name(), grokFieldName) + } } } - if m.Value != "" { - if !regex.HasCaptureGroup(m.Value) { - return fmt.Errorf("grok field %v not found in match pattern", m.Value) + if len(m.Value) > 0 { + for _, grokFieldName := range referencedGrokFields(m.ValueTemplate) { + if !regex.HasCaptureGroup(grokFieldName) { + return fmt.Errorf("%v: grok field %v not found in match pattern", m.Name, grokFieldName) + } } } return nil diff --git a/exporter/grok_test.go b/exporter/grok_test.go index 6c10f26c..c6f8d346 100644 --- a/exporter/grok_test.go +++ b/exporter/grok_test.go @@ -19,6 +19,7 @@ import ( "gopkg.in/yaml.v2" "strings" "testing" + "text/template" ) func TestGrok(t *testing.T) { @@ -71,26 +72,22 @@ func testVerifyCaptureGroup(t *testing.T, patterns *Patterns, libonig *Oniguruma } expectOK(t, regex, ` name: test - value: val + value: '{{.val}}' labels: - - grok_field_name: user - prometheus_label: user - - grok_field_name: host - prometheus_label: something`) + user: '{{.user}}' + host: '{{.host}}'`) expectOK(t, regex, ` name: test`) expectError(t, regex, ` name: test - value: value + value: '{{.value}}' labels: - - grok_field_name: user - prometheus_label: user`) + user: '{{.user}}'`) expectError(t, regex, ` name: test - value: val + value: '{{.val}}' labels: - - grok_field_name: user2 - prometheus_label: user`) + user: '{{.user2}}'`) regex.Free() } @@ -106,13 +103,53 @@ func expect(t *testing.T, regex *OnigurumaRegexp, config string, isErrorExpected cfg := &v2.MetricConfig{} err := yaml.Unmarshal([]byte(config), cfg) if err != nil { - t.Error(err) + t.Fatal(err) + } + err = cfg.InitTemplates() + if err != nil { + t.Fatal(err) } err = VerifyFieldNames(cfg, regex) if isErrorExpected && err == nil { - t.Error("Expected error, but got no error.") + t.Fatal("Expected error, but got no error.") } if !isErrorExpected && err != nil { - t.Error("Expected ok, but got error.") + t.Fatal("Expected ok, but got error.") + } +} + +func TestReferencedGrokFields(t *testing.T) { + grokFieldTest(t, "test1", "{{.count_total}} items are made of {{.material}}", "count_total", "material") + grokFieldTest(t, "test2", "{{23 -}} < {{- 45}}") + grokFieldTest(t, "test3", "{{.conca -}} < {{- .tenated}}", "conca", "tenated") + grokFieldTest(t, "test4", "{{with $x := \"output\" | printf \"%q\"}}{{$x}}{{end}}{{.bla}}", "bla") + grokFieldTest(t, "test5", "") + // Templates not supported yet. + // grokFieldTest(t, "test6", ` + // {{define "T1"}}{{.value1_total}}{{end}} + // {{define "T2"}}{{.value2_total}}{{end}} + // {{define "T3"}}{{template "T1"}} / {{template "T2"}}{{end}} + // {{template "T3"}}`, "value1_total", "value2_total") +} + +func grokFieldTest(t *testing.T, name, tmplt string, expectedFields ...string) { + parsedTemplate, err := template.New(name).Parse(tmplt) + if err != nil { + t.Fatalf("%v: error parsing template: %v", name, err.Error()) + } + actualFields := referencedGrokFields(parsedTemplate) + if len(actualFields) != len(expectedFields) { + t.Fatalf("%v: expected: %v, actual: %v", name, expectedFields, actualFields) + } + for _, actualField := range actualFields { + found := false + for _, expectedField := range expectedFields { + if expectedField == actualField { + found = true + } + } + if !found { + t.Fatalf("%v: expected: %v, actual: %v", name, expectedFields, actualFields) + } } } diff --git a/exporter/metrics.go b/exporter/metrics.go index d957eaa6..1fb58eff 100644 --- a/exporter/metrics.go +++ b/exporter/metrics.go @@ -15,10 +15,13 @@ package exporter import ( + "bytes" "fmt" "github.com/fstab/grok_exporter/config/v2" "github.com/prometheus/client_golang/prometheus" "strconv" + "text/template" + "text/template/parse" ) type Metric interface { @@ -33,7 +36,7 @@ type Metric interface { type incMetric struct { name string regex *OnigurumaRegexp - labels []v2.Label + labels []*template.Template collector prometheus.Collector incFunc func(m *OnigurumaMatchResult) error } @@ -42,8 +45,8 @@ type incMetric struct { type observeMetric struct { name string regex *OnigurumaRegexp - value string - labels []v2.Label + value *template.Template + labels []*template.Template collector prometheus.Collector observeFunc func(m *OnigurumaMatchResult, val float64) error } @@ -65,20 +68,21 @@ func NewCounterMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { }, } } else { // counterVec - counterVec := prometheus.NewCounterVec(counterOpts, prometheusLabels(cfg.Labels)) - return &incMetric{ + counterVec := prometheus.NewCounterVec(counterOpts, prometheusLabels(cfg.LabelTemplates)) + result := &incMetric{ name: cfg.Name, regex: regex, - labels: cfg.Labels, + labels: cfg.LabelTemplates, collector: counterVec, incFunc: func(m *OnigurumaMatchResult) error { - vals, err := labelValues(m, cfg.Labels) + vals, err := labelValues(m, cfg.LabelTemplates) if err == nil { counterVec.WithLabelValues(vals...).Inc() } return err }, } + return result } } @@ -92,7 +96,7 @@ func NewGaugeMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { return &observeMetric{ name: cfg.Name, regex: regex, - value: cfg.Value, + value: cfg.ValueTemplate, collector: gauge, observeFunc: func(_ *OnigurumaMatchResult, val float64) error { if cfg.Cumulative { @@ -104,15 +108,15 @@ func NewGaugeMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { }, } } else { // gaugeVec - gaugeVec := prometheus.NewGaugeVec(gaugeOpts, prometheusLabels(cfg.Labels)) + gaugeVec := prometheus.NewGaugeVec(gaugeOpts, prometheusLabels(cfg.LabelTemplates)) return &observeMetric{ name: cfg.Name, regex: regex, - value: cfg.Value, + value: cfg.ValueTemplate, collector: gaugeVec, - labels: cfg.Labels, + labels: cfg.LabelTemplates, observeFunc: func(m *OnigurumaMatchResult, val float64) error { - vals, err := labelValues(m, cfg.Labels) + vals, err := labelValues(m, cfg.LabelTemplates) if err == nil { if cfg.Cumulative { gaugeVec.WithLabelValues(vals...).Add(val) @@ -139,7 +143,7 @@ func NewHistogramMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { return &observeMetric{ name: cfg.Name, regex: regex, - value: cfg.Value, + value: cfg.ValueTemplate, collector: histogram, observeFunc: func(_ *OnigurumaMatchResult, val float64) error { histogram.Observe(val) @@ -147,15 +151,15 @@ func NewHistogramMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { }, } } else { // histogramVec - histogramVec := prometheus.NewHistogramVec(histogramOpts, prometheusLabels(cfg.Labels)) + histogramVec := prometheus.NewHistogramVec(histogramOpts, prometheusLabels(cfg.LabelTemplates)) return &observeMetric{ name: cfg.Name, regex: regex, - value: cfg.Value, + value: cfg.ValueTemplate, collector: histogramVec, - labels: cfg.Labels, + labels: cfg.LabelTemplates, observeFunc: func(m *OnigurumaMatchResult, val float64) error { - vals, err := labelValues(m, cfg.Labels) + vals, err := labelValues(m, cfg.LabelTemplates) if err == nil { histogramVec.WithLabelValues(vals...).Observe(val) } @@ -178,7 +182,7 @@ func NewSummaryMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { return &observeMetric{ name: cfg.Name, regex: regex, - value: cfg.Value, + value: cfg.ValueTemplate, collector: summary, observeFunc: func(_ *OnigurumaMatchResult, val float64) error { summary.Observe(val) @@ -186,15 +190,15 @@ func NewSummaryMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { }, } } else { // summaryVec - summaryVec := prometheus.NewSummaryVec(summaryOpts, prometheusLabels(cfg.Labels)) + summaryVec := prometheus.NewSummaryVec(summaryOpts, prometheusLabels(cfg.LabelTemplates)) return &observeMetric{ name: cfg.Name, regex: regex, - value: cfg.Value, + value: cfg.ValueTemplate, collector: summaryVec, - labels: cfg.Labels, + labels: cfg.LabelTemplates, observeFunc: func(m *OnigurumaMatchResult, val float64) error { - vals, err := labelValues(m, cfg.Labels) + vals, err := labelValues(m, cfg.LabelTemplates) if err == nil { summaryVec.WithLabelValues(vals...).Observe(val) } @@ -227,7 +231,7 @@ func (m *observeMetric) Process(line string) (bool, error) { } defer matchResult.Free() if matchResult.IsMatch() { - stringVal, err := matchResult.Get(m.value) + stringVal, err := evalTemplate(matchResult, m.value) if err != nil { return true, fmt.Errorf("error while processing metric %v: %v", m.name, err.Error()) } @@ -258,22 +262,56 @@ func (m *observeMetric) Collector() prometheus.Collector { return m.collector } -func labelValues(matchResult *OnigurumaMatchResult, labels []v2.Label) ([]string, error) { - values := make([]string, 0, len(labels)) - for _, field := range labels { - value, err := matchResult.Get(field.GrokFieldName) +func labelValues(matchResult *OnigurumaMatchResult, templates []*template.Template) ([]string, error) { + result := make([]string, 0, len(templates)) + for _, t := range templates { + value, err := evalTemplate(matchResult, t) if err != nil { return nil, err } - values = append(values, value) + result = append(result, value) + } + return result, nil +} + +func evalTemplate(matchResult *OnigurumaMatchResult, t *template.Template) (string, error) { + grokFields := referencedGrokFields(t) + grokValues := make(map[string]string, len(grokFields)) + for _, field := range grokFields { + value, err := matchResult.Get(field) + if err != nil { + return "", err + } + grokValues[field] = value + } + var buf bytes.Buffer + err := t.Execute(&buf, grokValues) + if err != nil { + return "", fmt.Errorf("unexpected error while evaluating %v template: %v", t.Name(), err.Error()) } - return values, nil + return buf.String(), nil } -func prometheusLabels(labels []v2.Label) []string { - promLabels := make([]string, 0, len(labels)) - for _, label := range labels { - promLabels = append(promLabels, label.PrometheusLabel) +func prometheusLabels(templates []*template.Template) []string { + promLabels := make([]string, 0, len(templates)) + for _, t := range templates { + promLabels = append(promLabels, t.Name()) } return promLabels } + +func referencedGrokFields(t *template.Template) []string { + result := make([]string, 0) + for _, node := range t.Root.Nodes { + if actionNode, ok := node.(*parse.ActionNode); ok { + for _, cmd := range actionNode.Pipe.Cmds { + for _, arg := range cmd.Args { + if fieldNode, ok := arg.(*parse.FieldNode); ok { + result = append(result, fieldNode.Ident...) + } + } + } + } + } + return result +} diff --git a/exporter/metrics_test.go b/exporter/metrics_test.go index 36cb44f7..c790ebbf 100644 --- a/exporter/metrics_test.go +++ b/exporter/metrics_test.go @@ -24,15 +24,12 @@ import ( func TestCounterVec(t *testing.T) { regex := initCounterRegex(t) - counterCfg := &v2.MetricConfig{ + counterCfg := newMetricConfig(t, &v2.MetricConfig{ Name: "exim_rejected_rcpt_total", - Labels: []v2.Label{ - { - GrokFieldName: "message", - PrometheusLabel: "error_message", - }, + Labels: map[string]string{ + "error_message": "{{.message}}", }, - } + }) counter := NewCounterMetric(counterCfg, regex) counter.Process("some unrelated line") counter.Process("2016-04-26 10:19:57 H=(85.214.241.101) [36.224.138.227] F= rejected RCPT : relay not permitted") @@ -57,9 +54,9 @@ func TestCounterVec(t *testing.T) { func TestCounter(t *testing.T) { regex := initCounterRegex(t) - counterCfg := &v2.MetricConfig{ + counterCfg := newMetricConfig(t, &v2.MetricConfig{ Name: "exim_rejected_rcpt_total", - } + }) counter := NewCounterMetric(counterCfg, regex) counter.Process("some unrelated line") @@ -98,10 +95,10 @@ func initCounterRegex(t *testing.T) *OnigurumaRegexp { func TestGauge(t *testing.T) { regex := initGaugeRegex(t) - gaugeCfg := &v2.MetricConfig{ + gaugeCfg := newMetricConfig(t, &v2.MetricConfig{ Name: "temperature", - Value: "temperature", - } + Value: "{{.temperature}}", + }) gauge := NewGaugeMetric(gaugeCfg, regex) gauge.Process("Temperature in Berlin: 32") @@ -121,11 +118,11 @@ func TestGauge(t *testing.T) { func TestGaugeCumulative(t *testing.T) { regex := initGaugeRegex(t) - gaugeCfg := &v2.MetricConfig{ + gaugeCfg := newMetricConfig(t, &v2.MetricConfig{ Name: "temperature", - Value: "temperature", + Value: "{{.temperature}}", Cumulative: true, - } + }) gauge := NewGaugeMetric(gaugeCfg, regex) gauge.Process("Temperature in Berlin: 32") @@ -145,16 +142,13 @@ func TestGaugeCumulative(t *testing.T) { func TestGaugeVec(t *testing.T) { regex := initGaugeRegex(t) - gaugeCfg := &v2.MetricConfig{ + gaugeCfg := newMetricConfig(t, &v2.MetricConfig{ Name: "temperature", - Value: "temperature", - Labels: []v2.Label{ - { - GrokFieldName: "city", - PrometheusLabel: "city", - }, + Value: "{{.temperature}}", + Labels: map[string]string{ + "city": "{{.city}}", }, - } + }) gauge := NewGaugeMetric(gaugeCfg, regex) gauge.Process("Temperature in Berlin: 32") @@ -189,3 +183,11 @@ func initGaugeRegex(t *testing.T) *OnigurumaRegexp { } return regex } + +func newMetricConfig(t *testing.T, cfg *v2.MetricConfig) *v2.MetricConfig { + err := cfg.InitTemplates() + if err != nil { + t.Fatal(err) + } + return cfg +}