Add ability to assert that a query fails with a particular error message

Signed-off-by: Charles Korn <charles.korn@grafana.com>

promql/test.go

@@ -292,6 +292,12 @@ func (t *test) parseEval(lines []string, i int) (int, *evalCmd, error) {
 			i--
 			break
 		}
+
+		if cmd.fail && strings.HasPrefix(defLine, "expected_message") {
+			cmd.expectedFailMessage = strings.TrimSpace(strings.TrimPrefix(defLine, "expected_message"))
+			break
+		}
+
 		if f, err := parseNumber(defLine); err == nil {
 			cmd.expect(0, parser.SequenceValue{Value: f})
 			break
@@ -441,8 +447,9 @@ type evalCmd struct {
 	step  time.Duration
 	line  int
 
-	isRange       bool // if false, instant query
-	fail, ordered bool
+	isRange             bool // if false, instant query
+	fail, ordered       bool
+	expectedFailMessage string
 
 	metrics  map[uint64]labels.Labels
 	expected map[uint64]entry
@@ -771,6 +778,10 @@ func (t *test) execRangeEval(cmd *evalCmd, engine QueryEngine) error {
 	res := q.Exec(t.context)
 	if res.Err != nil {
 		if cmd.fail {
+			if cmd.expectedFailMessage != "" && cmd.expectedFailMessage != res.Err.Error() {
+				return fmt.Errorf("expected error %q evaluating query %q (line %d), but got: %s", cmd.expectedFailMessage, cmd.expr, cmd.line, res.Err.Error())
+			}
+
 			return nil
 		}
 
@@ -803,6 +814,10 @@ func (t *test) execInstantEval(cmd *evalCmd, engine QueryEngine) error {
 		res := q.Exec(t.context)
 		if res.Err != nil {
 			if cmd.fail {
+				if cmd.expectedFailMessage != "" && cmd.expectedFailMessage != res.Err.Error() {
+					return fmt.Errorf("expected error %q evaluating query %q (line %d), but got: %s", cmd.expectedFailMessage, iq.expr, cmd.line, res.Err.Error())
+				}
+
 				continue
 			}
 			return fmt.Errorf("error evaluating query %q (line %d): %w", iq.expr, cmd.line, res.Err)

promql/test_test.go

@@ -262,6 +262,27 @@ eval_fail instant at 0m ceil({__name__=~'testmetric1|testmetric2'})
 			input:         `eval_fail instant at 0s vector(0)`,
 			expectedError: `expected error evaluating query "vector(0)" (line 1) but got none`,
 		},
+		"instant query expected to fail with specific error message, and query fails with that error": {
+			input: `
+load 5m
+	testmetric1{src="a",dst="b"} 0
+	testmetric2{src="a",dst="b"} 1
+
+eval_fail instant at 0m ceil({__name__=~'testmetric1|testmetric2'})
+	expected_message vector cannot contain metrics with the same labelset
+`,
+		},
+		"instant query expected to fail with specific error message, and query fails with a different error": {
+			input: `
+load 5m
+	testmetric1{src="a",dst="b"} 0
+	testmetric2{src="a",dst="b"} 1
+
+eval_fail instant at 0m ceil({__name__=~'testmetric1|testmetric2'})
+	expected_message something else went wrong
+`,
+			expectedError: `expected error "something else went wrong" evaluating query "ceil({__name__=~'testmetric1|testmetric2'})" (line 6), but got: vector cannot contain metrics with the same labelset`,
+		},
 		"instant query with results expected to match provided order, and result is in expected order": {
 			input: testData + `
 eval_ordered instant at 50m sort(http_requests)
@@ -383,6 +404,27 @@ eval_fail range from 0 to 10m step 5m ceil({__name__=~'testmetric1|testmetric2'}
 			input:         `eval_fail range from 0 to 10m step 5m vector(0)`,
 			expectedError: `expected error evaluating query "vector(0)" (line 1) but got none`,
 		},
+		"range query expected to fail with specific error message, and query fails with that error": {
+			input: `
+load 5m
+	testmetric1{src="a",dst="b"} 0
+	testmetric2{src="a",dst="b"} 1
+
+eval_fail range from 0 to 10m step 5m ceil({__name__=~'testmetric1|testmetric2'})
+	expected_message vector cannot contain metrics with the same labelset
+`,
+		},
+		"range query expected to fail with specific error message, and query fails with a different error": {
+			input: `
+load 5m
+	testmetric1{src="a",dst="b"} 0
+	testmetric2{src="a",dst="b"} 1
+
+eval_fail range from 0 to 10m step 5m ceil({__name__=~'testmetric1|testmetric2'})
+	expected_message something else went wrong
+`,
+			expectedError: `expected error "something else went wrong" evaluating query "ceil({__name__=~'testmetric1|testmetric2'})" (line 6), but got: vector cannot contain metrics with the same labelset`,
+		},
 		"range query with from and to timestamps in wrong order": {
 			input:         `eval range from 10m to 9m step 5m vector(0)`,
 			expectedError: `error in eval vector(0) (line 1): invalid test definition, end timestamp (9m) is before start timestamp (10m)`,