Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support extended durations in promtool unit tests (Fixes #6285) #6297

Open
wants to merge 2 commits into
base: master
from
Open
Changes from all commits
Commits
File filter...
Filter file types
Jump to…
Jump to file or symbol
Failed to load files and symbols.

Always

Just for now

@@ -0,0 +1,13 @@
# This is the rules file.

groups:
- name: example
rules:
- alert: InstanceDown
expr: up == 0
for: 5m
labels:
severity: page
annotations:
summary: "Instance {{ $labels.instance }} down"
description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."
@@ -0,0 +1,21 @@
rule_files:
- alerts.yml

evaluation_interval: 1m

tests:
- interval: 1m
input_series:
- series: 'up{job="prometheus", instance="localhost:9090"}'
values: "0+0x1440"
alert_rule_test:
- eval_time: 1d
alertname: InstanceDown
exp_alerts:
- exp_labels:
severity: page
instance: localhost:9090
job: prometheus
exp_annotations:
summary: "Instance localhost:9090 down"
description: "localhost:9090 of job prometheus has been down for more than 5 minutes."
@@ -29,6 +29,7 @@ import (
"github.com/pkg/errors"
yaml "gopkg.in/yaml.v2"

"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/pkg/labels"
"github.com/prometheus/prometheus/promql"
"github.com/prometheus/prometheus/rules"
@@ -75,15 +76,16 @@ func ruleUnitTest(filename string) []error {
}

if unitTestInp.EvaluationInterval == 0 {
unitTestInp.EvaluationInterval = 1 * time.Minute
unitTestInp.EvaluationInterval = model.Duration(1 * time.Minute)
}

// Bounds for evaluating the rules.
mint := time.Unix(0, 0)
maxd := unitTestInp.maxEvalTime()
maxt := mint.Add(maxd)
evalInterval := time.Duration(unitTestInp.EvaluationInterval)
// Rounding off to nearest Eval time (> maxt).
maxt = maxt.Add(unitTestInp.EvaluationInterval / 2).Round(unitTestInp.EvaluationInterval)
maxt = maxt.Add(evalInterval / 2).Round(evalInterval)

// Giving number for groups mentioned in the file for ordering.
// Lower number group should be evaluated before higher number group.
@@ -98,7 +100,7 @@ func ruleUnitTest(filename string) []error {
// Testing.
var errs []error
for _, t := range unitTestInp.Tests {
ers := t.test(mint, maxt, unitTestInp.EvaluationInterval, groupOrderMap,
ers := t.test(mint, maxt, evalInterval, groupOrderMap,
unitTestInp.RuleFiles...)
if ers != nil {
errs = append(errs, ers...)
@@ -113,10 +115,10 @@ func ruleUnitTest(filename string) []error {

// unitTestFile holds the contents of a single unit test file.
type unitTestFile struct {
RuleFiles []string `yaml:"rule_files"`
EvaluationInterval time.Duration `yaml:"evaluation_interval,omitempty"`
GroupEvalOrder []string `yaml:"group_eval_order"`
Tests []testGroup `yaml:"tests"`
RuleFiles []string `yaml:"rule_files"`
EvaluationInterval model.Duration `yaml:"evaluation_interval,omitempty"`
GroupEvalOrder []string `yaml:"group_eval_order"`
Tests []testGroup `yaml:"tests"`
}

func (utf *unitTestFile) maxEvalTime() time.Duration {
@@ -153,7 +155,7 @@ func resolveAndGlobFilepaths(baseDir string, utf *unitTestFile) error {

// testGroup is a group of input series and tests associated with it.
type testGroup struct {
Interval time.Duration `yaml:"interval"`
Interval model.Duration `yaml:"interval"`
InputSeries []series `yaml:"input_series"`
AlertRuleTests []alertTestCase `yaml:"alert_rule_test,omitempty"`
PromqlExprTests []promqlTestCase `yaml:"promql_expr_test,omitempty"`
@@ -178,7 +180,7 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou
Logger: log.NewNopLogger(),
}
m := rules.NewManager(opts)
groupsMap, ers := m.LoadGroups(tg.Interval, tg.ExternalLabels, ruleFiles...)
groupsMap, ers := m.LoadGroups(time.Duration(tg.Interval), tg.ExternalLabels, ruleFiles...)
if ers != nil {
return ers
}
@@ -189,11 +191,11 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou
// This avoids storing them in memory, as the number of evals might be high.

// All the `eval_time` for which we have unit tests for alerts.
alertEvalTimesMap := map[time.Duration]struct{}{}
alertEvalTimesMap := map[model.Duration]struct{}{}
// Map of all the eval_time+alertname combination present in the unit tests.
alertsInTest := make(map[time.Duration]map[string]struct{})
alertsInTest := make(map[model.Duration]map[string]struct{})
// Map of all the unit tests for given eval_time.
alertTests := make(map[time.Duration][]alertTestCase)
alertTests := make(map[model.Duration][]alertTestCase)
for _, alert := range tg.AlertRuleTests {
alertEvalTimesMap[alert.EvalTime] = struct{}{}

@@ -204,7 +206,7 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou

alertTests[alert.EvalTime] = append(alertTests[alert.EvalTime], alert)
}
alertEvalTimes := make([]time.Duration, 0, len(alertEvalTimesMap))
alertEvalTimes := make([]model.Duration, 0, len(alertEvalTimesMap))
for k := range alertEvalTimesMap {
alertEvalTimes = append(alertEvalTimes, k)
}
@@ -238,8 +240,8 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou
}

for {
if !(curr < len(alertEvalTimes) && ts.Sub(mint) <= alertEvalTimes[curr] &&
alertEvalTimes[curr] < ts.Add(evalInterval).Sub(mint)) {
if !(curr < len(alertEvalTimes) && ts.Sub(mint) <= time.Duration(alertEvalTimes[curr]) &&
time.Duration(alertEvalTimes[curr]) < ts.Add(time.Duration(evalInterval)).Sub(mint)) {
break
}

@@ -318,7 +320,7 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou
// Checking promql expressions.
Outer:
for _, testCase := range tg.PromqlExprTests {
got, err := query(suite.Context(), testCase.Expr, mint.Add(testCase.EvalTime),
got, err := query(suite.Context(), testCase.Expr, mint.Add(time.Duration(testCase.EvalTime)),
suite.QueryEngine(), suite.Queryable())
if err != nil {
errs = append(errs, errors.Errorf(" expr: %q, time: %s, err: %s", testCase.Expr,
@@ -369,15 +371,15 @@ Outer:

// seriesLoadingString returns the input series in PromQL notation.
func (tg *testGroup) seriesLoadingString() string {
result := ""
result += "load " + shortDuration(tg.Interval) + "\n"

result := fmt.Sprintf("load %v\n", shortDuration(tg.Interval))
for _, is := range tg.InputSeries {
result += " " + is.Series + " " + is.Values + "\n"
result += fmt.Sprintf(" %v %v\n", is.Series, is.Values)
}
return result
}

func shortDuration(d time.Duration) string {
func shortDuration(d model.Duration) string {
s := d.String()
if strings.HasSuffix(s, "m0s") {
s = s[:len(s)-2]
@@ -403,7 +405,7 @@ func orderedGroups(groupsMap map[string]*rules.Group, groupOrderMap map[string]i

// maxEvalTime returns the max eval time among all alert and promql unit tests.
func (tg *testGroup) maxEvalTime() time.Duration {
var maxd time.Duration
var maxd model.Duration
for _, alert := range tg.AlertRuleTests {
if alert.EvalTime > maxd {
maxd = alert.EvalTime
@@ -414,7 +416,7 @@ func (tg *testGroup) maxEvalTime() time.Duration {
maxd = pet.EvalTime
}
}
return maxd
return time.Duration(maxd)
}

func query(ctx context.Context, qs string, t time.Time, engine *promql.Engine, qu storage.Queryable) (promql.Vector, error) {
@@ -479,9 +481,9 @@ type series struct {
}

type alertTestCase struct {
EvalTime time.Duration `yaml:"eval_time"`
Alertname string `yaml:"alertname"`
ExpAlerts []alert `yaml:"exp_alerts"`
EvalTime model.Duration `yaml:"eval_time"`
Alertname string `yaml:"alertname"`
ExpAlerts []alert `yaml:"exp_alerts"`
}

type alert struct {
@@ -490,9 +492,9 @@ type alert struct {
}

type promqlTestCase struct {
Expr string `yaml:"expr"`
EvalTime time.Duration `yaml:"eval_time"`
ExpSamples []sample `yaml:"exp_samples"`
Expr string `yaml:"expr"`
EvalTime model.Duration `yaml:"eval_time"`
ExpSamples []sample `yaml:"exp_samples"`
}

type sample struct {
@@ -0,0 +1,42 @@
// Copyright 2018 The Prometheus Authors

This comment has been minimized.

Copy link
@codesome

codesome Nov 11, 2019

Member

There is already a PR for unit tests for unit test #6062, I would like to avoid duplication of effort. Can you add these test cases to that PR instead?

This comment has been minimized.

Copy link
@codesome

codesome Nov 11, 2019

Member

Or more like it would be better if we merge that PR first and merge this with the additional test case.

This comment has been minimized.

Copy link
@neufeldtech

neufeldtech Nov 11, 2019

Author

I'm fine with this PR going after #6062 is merged. I had created the one basic test to prove that deserialization of durations such as 1d was now working

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import "testing"

func TestRulesUnitTest(t *testing.T) {
type args struct {
files []string
}
tests := []struct {
name string
args args
want int
}{
{
name: "Passing Unit Tests",
args: args{
files: []string{"./testdata/unittest.yml"},
},
want: 0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := RulesUnitTest(tt.args.files...); got != tt.want {
t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want)
}
})
}
}
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.