From e2087727e3047e29dc5ac7aacdd03d4468f7c75b Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Thu, 6 Feb 2020 23:45:00 -0500 Subject: [PATCH 01/10] binops in ast --- pkg/logql/ast.go | 31 +++++ pkg/logql/ast_test.go | 8 ++ pkg/logql/expr.y | 32 ++++- pkg/logql/expr.y.go | 252 +++++++++++++++++++++++---------------- pkg/logql/lex.go | 2 + pkg/logql/parser_test.go | 4 +- 6 files changed, 218 insertions(+), 111 deletions(-) diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index 84cb803410eb..014e5ed46683 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -227,6 +227,10 @@ const ( OpTypeTopK = "topk" OpTypeCountOverTime = "count_over_time" OpTypeRate = "rate" + + // binops + OpTypeAdd = "+" + OpTypeDiv = "/" ) // SampleExpr is a LogQL expression filtering logs and returning metric samples. @@ -370,6 +374,33 @@ func (e *vectorAggregationExpr) String() string { return formatOperation(e.operation, e.grouping, params...) } +type binOpExpr struct { + SampleExpr + RHS SampleExpr + op string +} + +func (e *binOpExpr) String() string { + return fmt.Sprintf("%s %s %s", e.SampleExpr.String(), e.op, e.RHS.String()) +} + +func mustNewBinOpExpr(op string, lhs, rhs Expr) SampleExpr { + left, ok := lhs.(SampleExpr) + if !ok { + panic(fmt.Errorf("unexpected type for binOpExpr (%T): %+v", lhs, lhs)) + } + + right, ok := rhs.(SampleExpr) + if !ok { + panic(fmt.Errorf("unexpected type for binOpExpr (%T): %+v", rhs, rhs)) + } + return &binOpExpr{ + SampleExpr: left, + RHS: right, + op: op, + } +} + // helper used to impl Stringer for vector and range aggregations // nolint:interfacer func formatOperation(op string, grouping *grouping, params ...string) string { diff --git a/pkg/logql/ast_test.go b/pkg/logql/ast_test.go index 303cf52986ea..063fecf1561b 100644 --- a/pkg/logql/ast_test.go +++ b/pkg/logql/ast_test.go @@ -43,6 +43,14 @@ func Test_SampleExpr_String(t *testing.T) { `sum(count_over_time({job="mysql"}[5m]))`, `topk(10,sum(rate({region="us-east1"}[5m])) by (name))`, `avg( rate( ( {job="nginx"} |= "GET" ) [10s] ) ) by (region)`, + `sum by (cluster) (count_over_time({job="mysql"}[5m]))`, + `sum by (cluster) (count_over_time({job="mysql"}[5m])) / sum by (cluster) (count_over_time({job="postgres"}[5m])) `, + ` + sum by (cluster) (count_over_time({job="postgres"}[5m])) / + sum by (cluster) (count_over_time({job="postgres"}[5m])) / + sum by (cluster) (count_over_time({job="postgres"}[5m])) + `, + `sum by (cluster) (count_over_time({job="mysql"}[5m])) / min(count_over_time({job="mysql"}[5m])) `, } { t.Run(tc, func(t *testing.T) { expr, err := ParseExpr(tc) diff --git a/pkg/logql/expr.y b/pkg/logql/expr.y index a053b3224d36..32c30af2ff57 100644 --- a/pkg/logql/expr.y +++ b/pkg/logql/expr.y @@ -21,6 +21,8 @@ import ( Selector []*labels.Matcher VectorAggregationExpr SampleExpr VectorOp string + BinOpExpr SampleExpr + BinOp string str string duration time.Duration int int64 @@ -41,26 +43,38 @@ import ( %type selector %type vectorAggregationExpr %type vectorOp +%type binOpExpr +%type binOp %token IDENTIFIER STRING %token DURATION %token MATCHERS LABELS EQ NEQ RE NRE OPEN_BRACE CLOSE_BRACE OPEN_BRACKET CLOSE_BRACKET COMMA DOT PIPE_MATCH PIPE_EXACT OPEN_PARENTHESIS CLOSE_PARENTHESIS BY WITHOUT COUNT_OVER_TIME RATE SUM AVG MAX MIN COUNT STDDEV STDVAR BOTTOMK TOPK +// Operators are listed with increasing precedence. +%token +ADD +DIV + +%left ADD +%left DIV + %% root: expr { exprlex.(*lexer).expr = $1 }; expr: - logExpr { $$ = $1 } - | rangeAggregationExpr { $$ = $1 } - | vectorAggregationExpr { $$ = $1 } + logExpr { $$ = $1 } + | rangeAggregationExpr { $$ = $1 } + | vectorAggregationExpr { $$ = $1 } + | binOpExpr { $$ = $1 } + | OPEN_PARENTHESIS expr CLOSE_PARENTHESIS { $$ = $2 } ; logExpr: selector { $$ = newMatcherExpr($1)} | logExpr filter STRING { $$ = NewFilterExpr( $1, $2, $3 ) } - | OPEN_PARENTHESIS logExpr CLOSE_PARENTHESIS { $$ = $2} + | OPEN_PARENTHESIS logExpr CLOSE_PARENTHESIS { $$ = $2 } | logExpr filter error | logExpr error ; @@ -115,6 +129,16 @@ matcher: | IDENTIFIER NRE STRING { $$ = mustNewMatcher(labels.MatchNotRegexp, $1, $3) } ; +// TODO(owen-d): add (on,ignoring) clauses to binOpExpr +// https://prometheus.io/docs/prometheus/latest/querying/operators/ +binOpExpr: + expr binOp expr { $$ = mustNewBinOpExpr($2, $1, $3) } + +binOp: + ADD { $$ = OpTypeAdd } + | DIV { $$ = OpTypeDiv } + + vectorOp: SUM { $$ = OpTypeSum } | AVG { $$ = OpTypeAvg } diff --git a/pkg/logql/expr.y.go b/pkg/logql/expr.y.go index 4730258790db..c6b56d3c8826 100644 --- a/pkg/logql/expr.y.go +++ b/pkg/logql/expr.y.go @@ -25,6 +25,8 @@ type exprSymType struct { Selector []*labels.Matcher VectorAggregationExpr SampleExpr VectorOp string + BinOpExpr SampleExpr + BinOp string str string duration time.Duration int int64 @@ -62,6 +64,8 @@ const STDDEV = 57374 const STDVAR = 57375 const BOTTOMK = 57376 const TOPK = 57377 +const ADD = 57378 +const DIV = 57379 var exprToknames = [...]string{ "$end", @@ -99,6 +103,8 @@ var exprToknames = [...]string{ "STDVAR", "BOTTOMK", "TOPK", + "ADD", + "DIV", } var exprStatenames = [...]string{} @@ -113,92 +119,103 @@ var exprExca = [...]int{ -2, 0, -1, 3, 1, 2, + 22, 2, + 36, 2, + 37, 2, + -2, 0, + -1, 33, + 36, 2, + 37, 2, -2, 0, } const exprPrivate = 57344 -const exprLast = 149 +const exprLast = 156 var exprAct = [...]int{ - 31, 5, 4, 22, 36, 69, 10, 41, 30, 49, - 32, 33, 86, 46, 7, 32, 33, 88, 11, 12, - 13, 14, 16, 17, 15, 18, 19, 20, 21, 90, - 89, 48, 45, 44, 11, 12, 13, 14, 16, 17, - 15, 18, 19, 20, 21, 58, 85, 86, 3, 68, - 67, 63, 87, 84, 23, 71, 28, 72, 61, 65, - 64, 47, 27, 29, 26, 80, 81, 58, 82, 83, - 66, 24, 25, 53, 40, 76, 75, 74, 42, 11, - 12, 13, 14, 16, 17, 15, 18, 19, 20, 21, - 92, 93, 62, 59, 10, 78, 10, 59, 77, 73, - 91, 27, 43, 26, 7, 27, 37, 26, 23, 51, - 24, 25, 70, 79, 24, 25, 27, 60, 26, 23, - 9, 50, 23, 61, 52, 24, 25, 27, 40, 26, - 27, 39, 26, 35, 38, 37, 24, 25, 6, 24, - 25, 54, 55, 56, 57, 8, 34, 2, 1, + 36, 5, 4, 26, 41, 76, 11, 3, 48, 24, + 25, 37, 38, 97, 7, 33, 46, 96, 12, 13, + 14, 15, 17, 18, 16, 19, 20, 21, 22, 92, + 24, 25, 35, 66, 37, 38, 91, 52, 51, 2, + 93, 31, 49, 30, 93, 95, 72, 32, 71, 94, + 28, 29, 65, 86, 56, 55, 75, 74, 69, 70, + 53, 27, 78, 43, 79, 68, 54, 34, 73, 31, + 83, 30, 87, 88, 65, 89, 90, 60, 28, 29, + 58, 12, 13, 14, 15, 17, 18, 16, 19, 20, + 21, 22, 57, 85, 82, 59, 84, 99, 100, 12, + 13, 14, 15, 17, 18, 16, 19, 20, 21, 22, + 27, 11, 81, 45, 68, 66, 44, 80, 31, 50, + 30, 27, 40, 31, 42, 30, 27, 28, 29, 31, + 47, 30, 28, 29, 31, 67, 30, 98, 28, 29, + 23, 47, 42, 28, 29, 61, 62, 63, 64, 77, + 6, 10, 8, 9, 39, 1, } var exprPact = [...]int{ - -7, -1000, -1000, 120, -1000, -1000, -1000, 83, 42, -13, - 131, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 129, -1000, -1000, -1000, -1000, -1000, 106, 81, - 9, 40, 10, -12, 107, 59, -1000, 132, -1000, -1000, - -1000, 95, 117, 81, 38, 37, 53, 54, 108, 108, - -1000, -1000, 102, -1000, 94, 72, 71, 70, 93, -1000, - -1000, -1000, 52, 91, -8, -8, 54, 31, 24, 30, - -1000, -5, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 8, 7, -1000, -1000, 96, -1000, -1000, -8, - -8, -1000, -1000, -1000, + -7, -1000, -27, 124, -1000, -1000, -1000, -7, -1000, 46, + 11, 120, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -7, -1000, -1000, 111, -1000, -1000, -1000, + -1000, -1000, -6, 119, 98, 56, 45, 34, 33, 78, + 63, -1000, 136, -27, -1000, -1000, -1000, -1000, 113, 59, + 98, 26, 24, 51, 74, 145, 145, -1000, -1000, 138, + -1000, 112, 107, 89, 65, 91, -1000, -1000, -1000, 108, + 31, -12, -12, 74, 14, 7, 27, -1000, 23, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -5, + -9, -1000, -1000, 133, -1000, -1000, -12, -12, -1000, -1000, + -1000, } var exprPgo = [...]int{ - 0, 148, 147, 3, 0, 5, 48, 7, 4, 146, - 2, 145, 138, 1, 120, + 0, 155, 39, 3, 0, 5, 7, 8, 4, 154, + 2, 153, 152, 1, 151, 150, 140, } var exprR1 = [...]int{ - 0, 1, 2, 2, 2, 6, 6, 6, 6, 6, - 7, 7, 7, 7, 7, 10, 13, 13, 13, 13, - 13, 13, 13, 13, 13, 13, 3, 3, 3, 3, - 12, 12, 12, 9, 9, 8, 8, 8, 8, 14, - 14, 14, 14, 14, 14, 14, 14, 14, 11, 11, - 5, 5, 4, 4, + 0, 1, 2, 2, 2, 2, 2, 6, 6, 6, + 6, 6, 7, 7, 7, 7, 7, 10, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 3, 3, + 3, 3, 12, 12, 12, 9, 9, 8, 8, 8, + 8, 15, 16, 16, 14, 14, 14, 14, 14, 14, + 14, 14, 14, 11, 11, 5, 5, 4, 4, } var exprR2 = [...]int{ - 0, 1, 1, 1, 1, 1, 3, 3, 3, 2, - 2, 3, 3, 3, 2, 4, 4, 4, 5, 5, - 5, 5, 6, 7, 6, 7, 1, 1, 1, 1, - 3, 3, 3, 1, 3, 3, 3, 3, 3, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 3, 4, 4, + 0, 1, 1, 1, 1, 1, 3, 1, 3, 3, + 3, 2, 2, 3, 3, 3, 2, 4, 4, 4, + 5, 5, 5, 5, 6, 7, 6, 7, 1, 1, + 1, 1, 3, 3, 3, 1, 3, 3, 3, 3, + 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 3, 4, 4, } var exprChk = [...]int{ - -1000, -1, -2, -6, -10, -13, -12, 21, -11, -14, - 13, 25, 26, 27, 28, 31, 29, 30, 32, 33, - 34, 35, -3, 2, 19, 20, 12, 10, -6, 21, - 21, -4, 23, 24, -9, 2, -8, 4, 5, 2, - 22, -7, -6, 21, -10, -13, 4, 21, 21, 21, - 14, 2, 17, 14, 9, 10, 11, 12, -3, 2, - 22, 6, -6, -7, 22, 22, 17, -10, -13, -5, - 4, -5, -8, 5, 5, 5, 5, 5, 2, 22, - -4, -4, -13, -10, 22, 22, 17, 22, 22, 22, - 22, 4, -4, -4, + -1000, -1, -2, -6, -10, -13, -15, 21, -12, -11, + -14, 13, 25, 26, 27, 28, 31, 29, 30, 32, + 33, 34, 35, -16, 36, 37, -3, 2, 19, 20, + 12, 10, -2, -6, 21, 21, -4, 23, 24, -9, + 2, -8, 4, -2, 5, 2, 22, 22, -7, -6, + 21, -10, -13, 4, 21, 21, 21, 14, 2, 17, + 14, 9, 10, 11, 12, -3, 2, 22, 6, -6, + -7, 22, 22, 17, -10, -13, -5, 4, -5, -8, + 5, 5, 5, 5, 5, 2, 22, -4, -4, -13, + -10, 22, 22, 17, 22, 22, 22, 22, 4, -4, + -4, } var exprDef = [...]int{ - 0, -2, 1, -2, 3, 4, 5, 0, 0, 0, - 0, 48, 49, 39, 40, 41, 42, 43, 44, 45, - 46, 47, 0, 9, 26, 27, 28, 29, 0, 0, - 0, 0, 0, 0, 0, 0, 33, 0, 6, 8, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 30, 31, 0, 32, 0, 0, 0, 0, 0, 14, - 15, 10, 0, 0, 16, 17, 0, 0, 0, 0, - 50, 0, 34, 35, 36, 37, 38, 11, 13, 12, - 20, 21, 0, 0, 18, 19, 0, 52, 53, 22, - 24, 51, 23, 25, + 0, -2, 1, -2, 3, 4, 5, 0, 7, 0, + 0, 0, 53, 54, 44, 45, 46, 47, 48, 49, + 50, 51, 52, 0, 42, 43, 0, 11, 28, 29, + 30, 31, 0, -2, 0, 0, 0, 0, 0, 0, + 0, 35, 0, 41, 8, 10, 6, 9, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 32, 33, 0, + 34, 0, 0, 0, 0, 0, 16, 17, 12, 0, + 0, 18, 19, 0, 0, 0, 0, 55, 0, 36, + 37, 38, 39, 40, 13, 15, 14, 22, 23, 0, + 0, 20, 21, 0, 57, 58, 24, 26, 56, 25, + 27, } var exprTok1 = [...]int{ @@ -209,7 +226,7 @@ var exprTok2 = [...]int{ 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, 34, 35, + 32, 33, 34, 35, 36, 37, } var exprTok3 = [...]int{ 0, @@ -574,223 +591,248 @@ exprdefault: case 5: exprDollar = exprS[exprpt-1 : exprpt+1] { - exprVAL.LogExpr = newMatcherExpr(exprDollar[1].Selector) + exprVAL.Expr = exprDollar[1].BinOpExpr } case 6: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.LogExpr = NewFilterExpr(exprDollar[1].LogExpr, exprDollar[2].Filter, exprDollar[3].str) + exprVAL.Expr = exprDollar[2].Expr } case 7: + exprDollar = exprS[exprpt-1 : exprpt+1] + { + exprVAL.LogExpr = newMatcherExpr(exprDollar[1].Selector) + } + case 8: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.LogExpr = NewFilterExpr(exprDollar[1].LogExpr, exprDollar[2].Filter, exprDollar[3].str) + } + case 9: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LogExpr = exprDollar[2].LogExpr } - case 10: + case 12: exprDollar = exprS[exprpt-2 : exprpt+1] { exprVAL.LogRangeExpr = newLogRange(exprDollar[1].LogExpr, exprDollar[2].duration) } - case 11: + case 13: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LogRangeExpr = addFilterToLogRangeExpr(exprDollar[1].LogRangeExpr, exprDollar[2].Filter, exprDollar[3].str) } - case 12: + case 14: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LogRangeExpr = exprDollar[2].LogRangeExpr } - case 15: + case 17: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.RangeAggregationExpr = newRangeAggregationExpr(exprDollar[3].LogRangeExpr, exprDollar[1].RangeOp) } - case 16: + case 18: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[3].RangeAggregationExpr, exprDollar[1].VectorOp, nil, nil) } - case 17: + case 19: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[3].VectorAggregationExpr, exprDollar[1].VectorOp, nil, nil) } - case 18: + case 20: exprDollar = exprS[exprpt-5 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[4].RangeAggregationExpr, exprDollar[1].VectorOp, exprDollar[2].Grouping, nil) } - case 19: + case 21: exprDollar = exprS[exprpt-5 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[4].VectorAggregationExpr, exprDollar[1].VectorOp, exprDollar[2].Grouping, nil) } - case 20: + case 22: exprDollar = exprS[exprpt-5 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[3].RangeAggregationExpr, exprDollar[1].VectorOp, exprDollar[5].Grouping, nil) } - case 21: + case 23: exprDollar = exprS[exprpt-5 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[3].VectorAggregationExpr, exprDollar[1].VectorOp, exprDollar[5].Grouping, nil) } - case 22: + case 24: exprDollar = exprS[exprpt-6 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[5].VectorAggregationExpr, exprDollar[1].VectorOp, nil, &exprDollar[3].str) } - case 23: + case 25: exprDollar = exprS[exprpt-7 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[5].VectorAggregationExpr, exprDollar[1].VectorOp, exprDollar[7].Grouping, &exprDollar[3].str) } - case 24: + case 26: exprDollar = exprS[exprpt-6 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[5].RangeAggregationExpr, exprDollar[1].VectorOp, nil, &exprDollar[3].str) } - case 25: + case 27: exprDollar = exprS[exprpt-7 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[5].RangeAggregationExpr, exprDollar[1].VectorOp, exprDollar[7].Grouping, &exprDollar[3].str) } - case 26: + case 28: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Filter = labels.MatchRegexp } - case 27: + case 29: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Filter = labels.MatchEqual } - case 28: + case 30: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Filter = labels.MatchNotRegexp } - case 29: + case 31: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Filter = labels.MatchNotEqual } - case 30: + case 32: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Selector = exprDollar[2].Matchers } - case 31: + case 33: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Selector = exprDollar[2].Matchers } - case 32: + case 34: exprDollar = exprS[exprpt-3 : exprpt+1] { } - case 33: + case 35: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Matchers = []*labels.Matcher{exprDollar[1].Matcher} } - case 34: + case 36: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Matchers = append(exprDollar[1].Matchers, exprDollar[3].Matcher) } - case 35: + case 37: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Matcher = mustNewMatcher(labels.MatchEqual, exprDollar[1].str, exprDollar[3].str) } - case 36: + case 38: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Matcher = mustNewMatcher(labels.MatchNotEqual, exprDollar[1].str, exprDollar[3].str) } - case 37: + case 39: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Matcher = mustNewMatcher(labels.MatchRegexp, exprDollar[1].str, exprDollar[3].str) } - case 38: + case 40: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Matcher = mustNewMatcher(labels.MatchNotRegexp, exprDollar[1].str, exprDollar[3].str) } - case 39: + case 41: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.BinOpExpr = mustNewBinOpExpr(exprDollar[2].BinOp, exprDollar[1].Expr, exprDollar[3].Expr) + } + case 42: + exprDollar = exprS[exprpt-1 : exprpt+1] + { + exprVAL.BinOp = OpTypeAdd + } + case 43: + exprDollar = exprS[exprpt-1 : exprpt+1] + { + exprVAL.BinOp = OpTypeDiv + } + case 44: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeSum } - case 40: + case 45: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeAvg } - case 41: + case 46: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeCount } - case 42: + case 47: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMax } - case 43: + case 48: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMin } - case 44: + case 49: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStddev } - case 45: + case 50: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStdvar } - case 46: + case 51: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeBottomK } - case 47: + case 52: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeTopK } - case 48: + case 53: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpTypeCountOverTime } - case 49: + case 54: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpTypeRate } - case 50: + case 55: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Labels = []string{exprDollar[1].str} } - case 51: + case 56: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Labels = append(exprDollar[1].Labels, exprDollar[3].str) } - case 52: + case 57: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: false, groups: exprDollar[3].Labels} } - case 53: + case 58: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: true, groups: exprDollar[3].Labels} diff --git a/pkg/logql/lex.go b/pkg/logql/lex.go index d368e7b69e9a..c3812876872a 100644 --- a/pkg/logql/lex.go +++ b/pkg/logql/lex.go @@ -36,6 +36,8 @@ var tokens = map[string]int{ OpTypeStdvar: STDVAR, OpTypeBottomK: BOTTOMK, OpTypeTopK: TOPK, + "/": DIV, + "+": ADD, } type lexer struct { diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go index 97b5b3742783..6eab9b23568d 100644 --- a/pkg/logql/parser_test.go +++ b/pkg/logql/parser_test.go @@ -534,7 +534,7 @@ func TestParse(t *testing.T) { { in: `{foo="bar"} "foo"`, err: ParseError{ - msg: "syntax error: unexpected STRING, expecting != or !~ or |~ or |=", + msg: "syntax error: unexpected STRING", line: 1, col: 13, }, @@ -542,7 +542,7 @@ func TestParse(t *testing.T) { { in: `{foo="bar"} foo`, err: ParseError{ - msg: "syntax error: unexpected IDENTIFIER, expecting != or !~ or |~ or |=", + msg: "syntax error: unexpected IDENTIFIER", line: 1, col: 13, }, From d55c587cd3a9734c67f1e82a6935e260d98de2cd Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Fri, 7 Feb 2020 09:24:16 -0500 Subject: [PATCH 02/10] bin op associativity & precedence --- pkg/logql/expr.y | 19 ++---- pkg/logql/expr.y.go | 123 +++++++++++++++++++------------------- pkg/logql/parser_test.go | 124 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 188 insertions(+), 78 deletions(-) diff --git a/pkg/logql/expr.y b/pkg/logql/expr.y index 32c30af2ff57..76f22dffc174 100644 --- a/pkg/logql/expr.y +++ b/pkg/logql/expr.y @@ -22,7 +22,7 @@ import ( VectorAggregationExpr SampleExpr VectorOp string BinOpExpr SampleExpr - BinOp string + binOp string str string duration time.Duration int int64 @@ -44,7 +44,6 @@ import ( %type vectorAggregationExpr %type vectorOp %type binOpExpr -%type binOp %token IDENTIFIER STRING %token DURATION @@ -52,12 +51,8 @@ import ( OPEN_PARENTHESIS CLOSE_PARENTHESIS BY WITHOUT COUNT_OVER_TIME RATE SUM AVG MAX MIN COUNT STDDEV STDVAR BOTTOMK TOPK // Operators are listed with increasing precedence. -%token -ADD -DIV - -%left ADD -%left DIV +%left ADD +%left DIV %% @@ -132,12 +127,8 @@ matcher: // TODO(owen-d): add (on,ignoring) clauses to binOpExpr // https://prometheus.io/docs/prometheus/latest/querying/operators/ binOpExpr: - expr binOp expr { $$ = mustNewBinOpExpr($2, $1, $3) } - -binOp: - ADD { $$ = OpTypeAdd } - | DIV { $$ = OpTypeDiv } - + expr ADD expr { $$ = mustNewBinOpExpr("+", $1, $3) } + | expr DIV expr { $$ = mustNewBinOpExpr("/", $1, $3) } vectorOp: SUM { $$ = OpTypeSum } diff --git a/pkg/logql/expr.y.go b/pkg/logql/expr.y.go index c6b56d3c8826..ace0d61aab98 100644 --- a/pkg/logql/expr.y.go +++ b/pkg/logql/expr.y.go @@ -26,7 +26,7 @@ type exprSymType struct { VectorAggregationExpr SampleExpr VectorOp string BinOpExpr SampleExpr - BinOp string + binOp string str string duration time.Duration int int64 @@ -123,7 +123,7 @@ var exprExca = [...]int{ 36, 2, 37, 2, -2, 0, - -1, 33, + -1, 32, 36, 2, 37, 2, -2, 0, @@ -131,45 +131,45 @@ var exprExca = [...]int{ const exprPrivate = 57344 -const exprLast = 156 +const exprLast = 157 var exprAct = [...]int{ - 36, 5, 4, 26, 41, 76, 11, 3, 48, 24, - 25, 37, 38, 97, 7, 33, 46, 96, 12, 13, - 14, 15, 17, 18, 16, 19, 20, 21, 22, 92, - 24, 25, 35, 66, 37, 38, 91, 52, 51, 2, - 93, 31, 49, 30, 93, 95, 72, 32, 71, 94, - 28, 29, 65, 86, 56, 55, 75, 74, 69, 70, - 53, 27, 78, 43, 79, 68, 54, 34, 73, 31, - 83, 30, 87, 88, 65, 89, 90, 60, 28, 29, + 35, 5, 4, 25, 40, 76, 11, 3, 48, 23, + 24, 24, 36, 37, 7, 32, 46, 97, 12, 13, + 14, 15, 17, 18, 16, 19, 20, 21, 22, 96, + 23, 24, 34, 93, 36, 37, 52, 51, 95, 93, + 92, 49, 2, 91, 94, 72, 71, 11, 56, 55, + 31, 54, 65, 33, 73, 50, 75, 74, 69, 70, + 53, 26, 78, 60, 79, 68, 42, 43, 83, 30, + 82, 29, 87, 88, 65, 89, 90, 81, 27, 28, 58, 12, 13, 14, 15, 17, 18, 16, 19, 20, - 21, 22, 57, 85, 82, 59, 84, 99, 100, 12, + 21, 22, 57, 85, 80, 59, 84, 99, 100, 12, 13, 14, 15, 17, 18, 16, 19, 20, 21, 22, - 27, 11, 81, 45, 68, 66, 44, 80, 31, 50, - 30, 27, 40, 31, 42, 30, 27, 28, 29, 31, - 47, 30, 28, 29, 31, 67, 30, 98, 28, 29, - 23, 47, 42, 28, 29, 61, 62, 63, 64, 77, - 6, 10, 8, 9, 39, 1, + 26, 98, 6, 45, 68, 66, 44, 39, 30, 41, + 29, 66, 41, 30, 77, 29, 26, 27, 28, 30, + 47, 29, 27, 28, 30, 86, 29, 26, 27, 28, + 10, 67, 8, 27, 28, 30, 47, 29, 61, 62, + 63, 64, 9, 38, 27, 28, 1, } var exprPact = [...]int{ - -7, -1000, -27, 124, -1000, -1000, -1000, -7, -1000, 46, - 11, 120, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -7, -1000, -1000, 111, -1000, -1000, -1000, - -1000, -1000, -6, 119, 98, 56, 45, 34, 33, 78, - 63, -1000, 136, -27, -1000, -1000, -1000, -1000, 113, 59, - 98, 26, 24, 51, 74, 145, 145, -1000, -1000, 138, - -1000, 112, 107, 89, 65, 91, -1000, -1000, -1000, 108, - 31, -12, -12, 74, 14, 7, 27, -1000, 23, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -5, - -9, -1000, -1000, 133, -1000, -1000, -12, -12, -1000, -1000, + -7, -1000, -27, 135, -1000, -1000, -1000, -7, -1000, 32, + 11, 115, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -7, -7, 111, -1000, -1000, -1000, -1000, + -1000, -6, 124, 34, 56, 30, 28, 27, 78, 49, + -1000, 139, -26, -1000, -1000, -1000, -1000, -1000, 119, 59, + 34, 24, 23, 37, 74, 120, 120, -1000, -1000, 118, + -1000, 89, 72, 65, 63, 91, -1000, -1000, -1000, 108, + 113, -11, -11, 74, 21, 18, 22, -1000, 16, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 7, + -5, -1000, -1000, 107, -1000, -1000, -11, -11, -1000, -1000, -1000, } var exprPgo = [...]int{ - 0, 155, 39, 3, 0, 5, 7, 8, 4, 154, - 2, 153, 152, 1, 151, 150, 140, + 0, 156, 42, 3, 0, 5, 7, 8, 4, 153, + 2, 152, 142, 1, 140, 112, } var exprR1 = [...]int{ @@ -177,8 +177,8 @@ var exprR1 = [...]int{ 6, 6, 7, 7, 7, 7, 7, 10, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 3, 3, 3, 3, 12, 12, 12, 9, 9, 8, 8, 8, - 8, 15, 16, 16, 14, 14, 14, 14, 14, 14, - 14, 14, 14, 11, 11, 5, 5, 4, 4, + 8, 15, 15, 14, 14, 14, 14, 14, 14, 14, + 14, 14, 11, 11, 5, 5, 4, 4, } var exprR2 = [...]int{ @@ -186,16 +186,16 @@ var exprR2 = [...]int{ 3, 2, 2, 3, 3, 3, 2, 4, 4, 4, 5, 5, 5, 5, 6, 7, 6, 7, 1, 1, 1, 1, 3, 3, 3, 1, 3, 3, 3, 3, - 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 3, 4, 4, + 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 3, 4, 4, } var exprChk = [...]int{ -1000, -1, -2, -6, -10, -13, -15, 21, -12, -11, -14, 13, 25, 26, 27, 28, 31, 29, 30, 32, - 33, 34, 35, -16, 36, 37, -3, 2, 19, 20, - 12, 10, -2, -6, 21, 21, -4, 23, 24, -9, - 2, -8, 4, -2, 5, 2, 22, 22, -7, -6, + 33, 34, 35, 36, 37, -3, 2, 19, 20, 12, + 10, -2, -6, 21, 21, -4, 23, 24, -9, 2, + -8, 4, -2, -2, 5, 2, 22, 22, -7, -6, 21, -10, -13, 4, 21, 21, 21, 14, 2, 17, 14, 9, 10, 11, 12, -3, 2, 22, 6, -6, -7, 22, 22, 17, -10, -13, -5, 4, -5, -8, @@ -206,15 +206,15 @@ var exprChk = [...]int{ var exprDef = [...]int{ 0, -2, 1, -2, 3, 4, 5, 0, 7, 0, - 0, 0, 53, 54, 44, 45, 46, 47, 48, 49, - 50, 51, 52, 0, 42, 43, 0, 11, 28, 29, - 30, 31, 0, -2, 0, 0, 0, 0, 0, 0, - 0, 35, 0, 41, 8, 10, 6, 9, 0, 0, + 0, 0, 52, 53, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 0, 0, 0, 11, 28, 29, 30, + 31, 0, -2, 0, 0, 0, 0, 0, 0, 0, + 35, 0, 41, 42, 8, 10, 6, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 33, 0, 34, 0, 0, 0, 0, 0, 16, 17, 12, 0, - 0, 18, 19, 0, 0, 0, 0, 55, 0, 36, + 0, 18, 19, 0, 0, 0, 0, 54, 0, 36, 37, 38, 39, 40, 13, 15, 14, 22, 23, 0, - 0, 20, 21, 0, 57, 58, 24, 26, 56, 25, + 0, 20, 21, 0, 56, 57, 24, 26, 55, 25, 27, } var exprTok1 = [...]int{ @@ -750,89 +750,84 @@ exprdefault: case 41: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.BinOpExpr = mustNewBinOpExpr(exprDollar[2].BinOp, exprDollar[1].Expr, exprDollar[3].Expr) + exprVAL.BinOpExpr = mustNewBinOpExpr("+", exprDollar[1].Expr, exprDollar[3].Expr) } case 42: - exprDollar = exprS[exprpt-1 : exprpt+1] + exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.BinOp = OpTypeAdd + exprVAL.BinOpExpr = mustNewBinOpExpr("/", exprDollar[1].Expr, exprDollar[3].Expr) } case 43: - exprDollar = exprS[exprpt-1 : exprpt+1] - { - exprVAL.BinOp = OpTypeDiv - } - case 44: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeSum } - case 45: + case 44: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeAvg } - case 46: + case 45: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeCount } - case 47: + case 46: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMax } - case 48: + case 47: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMin } - case 49: + case 48: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStddev } - case 50: + case 49: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStdvar } - case 51: + case 50: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeBottomK } - case 52: + case 51: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeTopK } - case 53: + case 52: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpTypeCountOverTime } - case 54: + case 53: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpTypeRate } - case 55: + case 54: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Labels = []string{exprDollar[1].str} } - case 56: + case 55: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Labels = append(exprDollar[1].Labels, exprDollar[3].str) } - case 57: + case 56: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: false, groups: exprDollar[3].Labels} } - case 58: + case 57: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: true, groups: exprDollar[3].Labels} diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go index 6eab9b23568d..11c92081c55e 100644 --- a/pkg/logql/parser_test.go +++ b/pkg/logql/parser_test.go @@ -547,6 +547,130 @@ func TestParse(t *testing.T) { col: 13, }, }, + { + // require left associativity + in: ` +sum(count_over_time({foo="bar"}[5m])) by (foo) / +sum(count_over_time({foo="bar"}[5m])) by (foo) / +sum(count_over_time({foo="bar"}[5m])) by (foo) +`, + exp: mustNewBinOpExpr( + OpTypeDiv, + mustNewBinOpExpr( + OpTypeDiv, + mustNewVectorAggregationExpr(newRangeAggregationExpr( + &logRange{ + left: &matchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, OpTypeCountOverTime), + "sum", + &grouping{ + without: false, + groups: []string{"foo"}, + }, + nil, + ), + mustNewVectorAggregationExpr(newRangeAggregationExpr( + &logRange{ + left: &matchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, OpTypeCountOverTime), + "sum", + &grouping{ + without: false, + groups: []string{"foo"}, + }, + nil, + ), + ), + mustNewVectorAggregationExpr(newRangeAggregationExpr( + &logRange{ + left: &matchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, OpTypeCountOverTime), + "sum", + &grouping{ + without: false, + groups: []string{"foo"}, + }, + nil, + ), + ), + }, + { + // operator precedence before left associativity + in: ` +sum(count_over_time({foo="bar"}[5m])) by (foo) + +sum(count_over_time({foo="bar"}[5m])) by (foo) / +sum(count_over_time({foo="bar"}[5m])) by (foo) +`, + exp: mustNewBinOpExpr( + OpTypeAdd, + mustNewVectorAggregationExpr(newRangeAggregationExpr( + &logRange{ + left: &matchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, OpTypeCountOverTime), + "sum", + &grouping{ + without: false, + groups: []string{"foo"}, + }, + nil, + ), + mustNewBinOpExpr( + OpTypeDiv, + mustNewVectorAggregationExpr(newRangeAggregationExpr( + &logRange{ + left: &matchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, OpTypeCountOverTime), + "sum", + &grouping{ + without: false, + groups: []string{"foo"}, + }, + nil, + ), + mustNewVectorAggregationExpr(newRangeAggregationExpr( + &logRange{ + left: &matchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, OpTypeCountOverTime), + "sum", + &grouping{ + without: false, + groups: []string{"foo"}, + }, + nil, + ), + ), + ), + }, } { t.Run(tc.in, func(t *testing.T) { ast, err := ParseExpr(tc.in) From 519d226d3cfcf0c76eb3927befd0e1310dd12f82 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Fri, 7 Feb 2020 11:50:34 -0500 Subject: [PATCH 03/10] binOpEvaluator work --- pkg/logql/evaluator.go | 100 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/pkg/logql/evaluator.go b/pkg/logql/evaluator.go index 589fdf802ebd..444cc2a65970 100644 --- a/pkg/logql/evaluator.go +++ b/pkg/logql/evaluator.go @@ -94,6 +94,8 @@ func (ev *defaultEvaluator) Evaluator(ctx context.Context, expr SampleExpr, q Pa return ev.vectorAggEvaluator(ctx, e, q) case *rangeAggregationExpr: return ev.rangeAggEvaluator(ctx, e, q) + case *binOpExpr: + return ev.binOpEvaluator(ctx, e, q) default: return nil, errors.Errorf("unexpected type (%T): %v", e, e) @@ -325,3 +327,101 @@ func (ev *defaultEvaluator) rangeAggEvaluator(ctx context.Context, expr *rangeAg }, vecIter.Close) } + +func (ev *defaultEvaluator) binOpEvaluator( + ctx context.Context, + expr *binOpExpr, + q Params, +) (StepEvaluator, error) { + lhs, err := ev.Evaluator(ctx, expr.SampleExpr, q) + if err != nil { + return nil, err + } + rhs, err := ev.Evaluator(ctx, expr.RHS, q) + if err != nil { + return nil, err + } + + return newStepEvaluator(func() (bool, int64, promql.Vector) { + pairs := map[uint64]promql.Vector{} + var ts int64 + + // populate pairs + for _, eval := range []StepEvaluator{lhs, rhs} { + next, timestamp, vec := eval.Next() + ts = timestamp + + // These should _always_ happen at the same step on each evaluator. + if !next { + return next, ts, nil + } + + for _, sample := range vec { + // TODO(owen-d): this seems wildly inefficient: we're calculating + // the hash on each sample & step per evaluator. + // We seem limited to this approach due to using the StepEvaluator ifc. + hash := sample.Metric.Hash() + pair, ok := pairs[hash] + if !ok { + pair = make(promql.Vector, 0, 2) + } + pairs[hash] = append(pair, sample) + + } + } + + results := make(promql.Vector, 0, len(pairs)) + for _, pair := range pairs { + // merge + results = append(results, ev.mergeBinOp(expr.op, pair[0], pair[1:]...)) + } + + return true, ts, results + }, func() (lastError error) { + for _, ev := range []StepEvaluator{lhs, rhs} { + if err := ev.Close(); err != nil { + lastError = err + } + } + return lastError + }) +} + +func (ev *defaultEvaluator) mergeBinOp(op string, first promql.Sample, rest ...promql.Sample) promql.Sample { + // mergers should be able to handle nil rests (think of merging vectors where a series is only present in one), + // but will only ever be passed two samples for merging. We use the (...promql.Sample) type + // in the function signature as it handles both empty and 1-length vectors. + var merger func(first promql.Sample, rest ...promql.Sample) promql.Sample + + switch op { + case OpTypeAdd: + merger = func(first promql.Sample, rest ...promql.Sample) promql.Sample { + res := promql.Sample{ + Metric: first.Metric.Copy(), + Point: first.Point, + } + for _, sample := range rest { + res.Point.V += sample.Point.V + } + return res + } + case OpTypeDiv: + merger = func(first promql.Sample, rest ...promql.Sample) promql.Sample { + res := promql.Sample{ + Metric: first.Metric.Copy(), + Point: first.Point, + } + + for _, sample := range rest { + res.Point.V /= sample.Point.V + } + return res + } + + default: + panic(errors.Errorf("should never happen: unexpected operation: (%s)", op)) + } + + return merger(first, rest...) + +} From 05809bd5eda2de6b14f0338df8e346e8d6f9ab30 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Fri, 7 Feb 2020 14:05:59 -0500 Subject: [PATCH 04/10] defers close only if constructed without error --- pkg/logql/engine.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/logql/engine.go b/pkg/logql/engine.go index fab60ff2fcd2..a479df76e8c5 100644 --- a/pkg/logql/engine.go +++ b/pkg/logql/engine.go @@ -214,10 +214,11 @@ func (ng *engine) exec(ctx context.Context, q *query) (promql.Value, error) { func (ng *engine) evalSample(ctx context.Context, expr SampleExpr, q *query) (promql.Value, error) { stepEvaluator, err := ng.evaluator.Evaluator(ctx, expr, q) - defer helpers.LogError("closing SampleExpr", stepEvaluator.Close) if err != nil { return nil, err } + defer helpers.LogError("closing SampleExpr", stepEvaluator.Close) + seriesIndex := map[uint64]*promql.Series{} next, ts, vec := stepEvaluator.Next() From d6c2217c9e8d3756fcd649040d0d555b39852e07 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Fri, 7 Feb 2020 14:14:42 -0500 Subject: [PATCH 05/10] tests binary ops --- pkg/logql/engine_test.go | 87 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 83 insertions(+), 4 deletions(-) diff --git a/pkg/logql/engine_test.go b/pkg/logql/engine_test.go index d8717ad485e1..321597707d23 100644 --- a/pkg/logql/engine_test.go +++ b/pkg/logql/engine_test.go @@ -653,13 +653,17 @@ func TestEngine_NewRangeQuery(t *testing.T) { }, }, { - `bottomk(3,rate(({app=~"foo|bar"} |~".+bar")[1m])) without (app)`, time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + `bottomk(3,rate(({app=~"foo|bar|fuzz|buzz"} |~".+bar")[1m])) without (app)`, time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, [][]*logproto.Stream{ - {newStream(testSize, factor(10, identity), `{app="foo"}`), newStream(testSize, factor(20, identity), `{app="bar"}`), - newStream(testSize, factor(5, identity), `{app="fuzz"}`), newStream(testSize, identity, `{app="buzz"}`)}, + { + newStream(testSize, factor(10, identity), `{app="foo"}`), + newStream(testSize, factor(20, identity), `{app="bar"}`), + newStream(testSize, factor(5, identity), `{app="fuzz"}`), + newStream(testSize, identity, `{app="buzz"}`), + }, }, []SelectParams{ - {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}|~".+bar"`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar|fuzz|buzz"}|~".+bar"`}}, }, promql.Matrix{ promql.Series{ @@ -676,6 +680,81 @@ func TestEngine_NewRangeQuery(t *testing.T) { }, }, }, + // test binary ops + { + `sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) / sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m]))`, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}|~".+bar"`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 1}, {T: 90 * 1000, V: 1}, {T: 120 * 1000, V: 1}, {T: 150 * 1000, V: 1}, {T: 180 * 1000, V: 1}}, + }, + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "foo"}}, + Points: []promql.Point{{T: 60 * 1000, V: 1}, {T: 90 * 1000, V: 1}, {T: 120 * 1000, V: 1}, {T: 150 * 1000, V: 1}, {T: 180 * 1000, V: 1}}, + }, + }, + }, + { + `sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) + sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m]))`, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}|~".+bar"`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 0.4}, {T: 90 * 1000, V: 0.4}, {T: 120 * 1000, V: 0.4}, {T: 150 * 1000, V: 0.4}, {T: 180 * 1000, V: 0.4}}, + }, + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "foo"}}, + Points: []promql.Point{{T: 60 * 1000, V: 0.4}, {T: 90 * 1000, V: 0.4}, {T: 120 * 1000, V: 0.4}, {T: 150 * 1000, V: 0.4}, {T: 180 * 1000, V: 0.4}}, + }, + }, + }, + // tests precedence: should be x + (x/x) + { + ` +sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) + +sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) / +sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) +`, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}|~".+bar"`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 1.2}, {T: 90 * 1000, V: 1.2}, {T: 120 * 1000, V: 1.2}, {T: 150 * 1000, V: 1.2}, {T: 180 * 1000, V: 1.2}}, + }, + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "foo"}}, + Points: []promql.Point{{T: 60 * 1000, V: 1.2}, {T: 90 * 1000, V: 1.2}, {T: 120 * 1000, V: 1.2}, {T: 150 * 1000, V: 1.2}, {T: 180 * 1000, V: 1.2}}, + }, + }, + }, } { test := test t.Run(fmt.Sprintf("%s %s", test.qs, test.direction), func(t *testing.T) { From 7ea39b9dfe00c7ff72f9f7124bf804e8e76a7a22 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Fri, 7 Feb 2020 17:32:11 -0500 Subject: [PATCH 06/10] more binops --- pkg/logql/ast.go | 10 +- pkg/logql/engine_test.go | 215 ++++++++++++++++++++++++++++++++++-- pkg/logql/evaluator.go | 131 +++++++++++++++++----- pkg/logql/evaluator_test.go | 31 ++++++ pkg/logql/expr.y | 21 +++- pkg/logql/expr.y.go | 210 +++++++++++++++++++++++------------ pkg/logql/lex.go | 12 +- 7 files changed, 510 insertions(+), 120 deletions(-) create mode 100644 pkg/logql/evaluator_test.go diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index 014e5ed46683..c1716922ab91 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -229,8 +229,14 @@ const ( OpTypeRate = "rate" // binops - OpTypeAdd = "+" - OpTypeDiv = "/" + OpTypeOr = "or" + OpTypeAnd = "and" + OpTypeUnless = "unless" + OpTypeAdd = "+" + OpTypeSub = "-" + OpTypeMul = "*" + OpTypeDiv = "/" + OpTypeMod = "%" ) // SampleExpr is a LogQL expression filtering logs and returning metric samples. diff --git a/pkg/logql/engine_test.go b/pkg/logql/engine_test.go index 321597707d23..44c6b7805d2a 100644 --- a/pkg/logql/engine_test.go +++ b/pkg/logql/engine_test.go @@ -680,60 +680,251 @@ func TestEngine_NewRangeQuery(t *testing.T) { }, }, }, - // test binary ops + // binops { - `sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) / sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m]))`, + ` +rate({app="foo"}[1m]) or +rate({app="bar"}[1m]) +`, time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, [][]*logproto.Stream{ { newStream(testSize, factor(5, identity), `{app="foo"}`), + }, + { newStream(testSize, factor(5, identity), `{app="bar"}`), }, }, []SelectParams{ - {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}|~".+bar"`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="foo"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, }, promql.Matrix{ promql.Series{ Metric: labels.Labels{{Name: "app", Value: "bar"}}, - Points: []promql.Point{{T: 60 * 1000, V: 1}, {T: 90 * 1000, V: 1}, {T: 120 * 1000, V: 1}, {T: 150 * 1000, V: 1}, {T: 180 * 1000, V: 1}}, + Points: []promql.Point{{T: 60 * 1000, V: 0.2}, {T: 90 * 1000, V: 0.2}, {T: 120 * 1000, V: 0.2}, {T: 150 * 1000, V: 0.2}, {T: 180 * 1000, V: 0.2}}, }, promql.Series{ Metric: labels.Labels{{Name: "app", Value: "foo"}}, - Points: []promql.Point{{T: 60 * 1000, V: 1}, {T: 90 * 1000, V: 1}, {T: 120 * 1000, V: 1}, {T: 150 * 1000, V: 1}, {T: 180 * 1000, V: 1}}, + Points: []promql.Point{{T: 60 * 1000, V: 0.2}, {T: 90 * 1000, V: 0.2}, {T: 120 * 1000, V: 0.2}, {T: 150 * 1000, V: 0.2}, {T: 180 * 1000, V: 0.2}}, }, }, }, { - `sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) + sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m]))`, + ` +rate({app=~"foo|bar"}[1m]) and +rate({app="bar"}[1m]) +`, time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, [][]*logproto.Stream{ { newStream(testSize, factor(5, identity), `{app="foo"}`), newStream(testSize, factor(5, identity), `{app="bar"}`), }, + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, }, []SelectParams{ - {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}|~".+bar"`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, }, promql.Matrix{ promql.Series{ Metric: labels.Labels{{Name: "app", Value: "bar"}}, - Points: []promql.Point{{T: 60 * 1000, V: 0.4}, {T: 90 * 1000, V: 0.4}, {T: 120 * 1000, V: 0.4}, {T: 150 * 1000, V: 0.4}, {T: 180 * 1000, V: 0.4}}, + Points: []promql.Point{{T: 60 * 1000, V: 0.2}, {T: 90 * 1000, V: 0.2}, {T: 120 * 1000, V: 0.2}, {T: 150 * 1000, V: 0.2}, {T: 180 * 1000, V: 0.2}}, + }, + }, + }, + { + ` + rate({app=~"foo|bar"}[1m]) unless + rate({app="bar"}[1m]) + `, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + { + newStream(testSize, factor(5, identity), `{app="bar"}`), }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ promql.Series{ Metric: labels.Labels{{Name: "app", Value: "foo"}}, + Points: []promql.Point{{T: 60 * 1000, V: 0.2}, {T: 90 * 1000, V: 0.2}, {T: 120 * 1000, V: 0.2}, {T: 150 * 1000, V: 0.2}, {T: 180 * 1000, V: 0.2}}, + }, + }, + }, + { + ` + rate({app=~"foo|bar"}[1m]) + + rate({app="bar"}[1m]) + `, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, Points: []promql.Point{{T: 60 * 1000, V: 0.4}, {T: 90 * 1000, V: 0.4}, {T: 120 * 1000, V: 0.4}, {T: 150 * 1000, V: 0.4}, {T: 180 * 1000, V: 0.4}}, }, }, }, + { + ` + rate({app=~"foo|bar"}[1m]) - + rate({app="bar"}[1m]) + `, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 0}, {T: 90 * 1000, V: 0}, {T: 120 * 1000, V: 0}, {T: 150 * 1000, V: 0}, {T: 180 * 1000, V: 0}}, + }, + }, + }, + { + ` + count_over_time({app=~"foo|bar"}[1m]) * + count_over_time({app="bar"}[1m]) + `, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 144}, {T: 90 * 1000, V: 144}, {T: 120 * 1000, V: 144}, {T: 150 * 1000, V: 144}, {T: 180 * 1000, V: 144}}, + }, + }, + }, + { + ` + count_over_time({app=~"foo|bar"}[1m]) * + count_over_time({app="bar"}[1m]) + `, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 144}, {T: 90 * 1000, V: 144}, {T: 120 * 1000, V: 144}, {T: 150 * 1000, V: 144}, {T: 180 * 1000, V: 144}}, + }, + }, + }, + { + ` + count_over_time({app=~"foo|bar"}[1m]) / + count_over_time({app="bar"}[1m]) + `, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 1}, {T: 90 * 1000, V: 1}, {T: 120 * 1000, V: 1}, {T: 150 * 1000, V: 1}, {T: 180 * 1000, V: 1}}, + }, + }, + }, + { + ` + count_over_time({app=~"foo|bar"}[1m]) % + count_over_time({app="bar"}[1m]) + `, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 0}, {T: 90 * 1000, V: 0}, {T: 120 * 1000, V: 0}, {T: 150 * 1000, V: 0}, {T: 180 * 1000, V: 0}}, + }, + }, + }, // tests precedence: should be x + (x/x) { ` -sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) + -sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) / -sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) -`, + sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) + + sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) / + sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) + `, time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, [][]*logproto.Stream{ { diff --git a/pkg/logql/evaluator.go b/pkg/logql/evaluator.go index 444cc2a65970..e740f6990176 100644 --- a/pkg/logql/evaluator.go +++ b/pkg/logql/evaluator.go @@ -343,11 +343,11 @@ func (ev *defaultEvaluator) binOpEvaluator( } return newStepEvaluator(func() (bool, int64, promql.Vector) { - pairs := map[uint64]promql.Vector{} + pairs := map[uint64][2]*promql.Sample{} var ts int64 // populate pairs - for _, eval := range []StepEvaluator{lhs, rhs} { + for i, eval := range []StepEvaluator{lhs, rhs} { next, timestamp, vec := eval.Next() ts = timestamp @@ -361,19 +361,22 @@ func (ev *defaultEvaluator) binOpEvaluator( // the hash on each sample & step per evaluator. // We seem limited to this approach due to using the StepEvaluator ifc. hash := sample.Metric.Hash() - pair, ok := pairs[hash] - if !ok { - pair = make(promql.Vector, 0, 2) + pair := pairs[hash] + pair[i] = &promql.Sample{ + Metric: sample.Metric, + Point: sample.Point, } - pairs[hash] = append(pair, sample) - + pairs[hash] = pair } } results := make(promql.Vector, 0, len(pairs)) for _, pair := range pairs { + // merge - results = append(results, ev.mergeBinOp(expr.op, pair[0], pair[1:]...)) + if merged := ev.mergeBinOp(expr.op, pair[0], pair[1]); merged != nil { + results = append(results, *merged) + } } return true, ts, results @@ -387,41 +390,117 @@ func (ev *defaultEvaluator) binOpEvaluator( }) } -func (ev *defaultEvaluator) mergeBinOp(op string, first promql.Sample, rest ...promql.Sample) promql.Sample { - // mergers should be able to handle nil rests (think of merging vectors where a series is only present in one), - // but will only ever be passed two samples for merging. We use the (...promql.Sample) type - // in the function signature as it handles both empty and 1-length vectors. - var merger func(first promql.Sample, rest ...promql.Sample) promql.Sample +func (ev *defaultEvaluator) mergeBinOp(op string, left, right *promql.Sample) *promql.Sample { + var merger func(left, right *promql.Sample) *promql.Sample switch op { + case OpTypeOr: + merger = func(left, right *promql.Sample) *promql.Sample { + // return the left entry found (prefers left hand side) + if left != nil { + return left + } + return right + } + + case OpTypeAnd: + merger = func(left, right *promql.Sample) *promql.Sample { + // return left sample if there's a second sample for that label set + if left != nil && right != nil { + return left + } + return nil + } + + case OpTypeUnless: + merger = func(left, right *promql.Sample) *promql.Sample { + // return left sample if there's not a second sample for that label set + if right == nil { + return left + } + return nil + } + case OpTypeAdd: - merger = func(first promql.Sample, rest ...promql.Sample) promql.Sample { + merger = func(left, right *promql.Sample) *promql.Sample { + if left == nil || right == nil { + return nil + } + res := promql.Sample{ + Metric: left.Metric, + Point: left.Point, + } + res.Point.V += right.Point.V + return &res + } + + case OpTypeSub: + merger = func(left, right *promql.Sample) *promql.Sample { + if left == nil || right == nil { + return nil + } res := promql.Sample{ - Metric: first.Metric.Copy(), - Point: first.Point, + Metric: left.Metric, + Point: left.Point, + } + res.Point.V -= right.Point.V + return &res + } + + case OpTypeMul: + merger = func(left, right *promql.Sample) *promql.Sample { + if left == nil || right == nil { + return nil } - for _, sample := range rest { - res.Point.V += sample.Point.V + res := promql.Sample{ + Metric: left.Metric, + Point: left.Point, } - return res + res.Point.V *= right.Point.V + return &res } + case OpTypeDiv: - merger = func(first promql.Sample, rest ...promql.Sample) promql.Sample { + merger = func(left, right *promql.Sample) *promql.Sample { + if left == nil || right == nil { + return nil + } res := promql.Sample{ - Metric: first.Metric.Copy(), - Point: first.Point, + Metric: left.Metric.Copy(), + Point: left.Point, } - for _, sample := range rest { - res.Point.V /= sample.Point.V + // guard against divide by zero + if right.Point.V == 0 { + res.Point.V = math.NaN() + } else { + res.Point.V /= right.Point.V + } + return &res + } + + case OpTypeMod: + merger = func(left, right *promql.Sample) *promql.Sample { + if left == nil || right == nil { + return nil + } + res := promql.Sample{ + Metric: left.Metric, + Point: left.Point, + } + // guard against divide by zero + if right.Point.V == 0 { + res.Point.V = math.NaN() + } else { + res.Point.V = math.Mod(res.Point.V, right.Point.V) } - return res + return &res } default: panic(errors.Errorf("should never happen: unexpected operation: (%s)", op)) } - return merger(first, rest...) + return merger(left, right) } diff --git a/pkg/logql/evaluator_test.go b/pkg/logql/evaluator_test.go new file mode 100644 index 000000000000..23ef3d2f71c9 --- /dev/null +++ b/pkg/logql/evaluator_test.go @@ -0,0 +1,31 @@ +package logql + +import ( + "math" + "testing" + + "github.com/prometheus/prometheus/promql" + "github.com/stretchr/testify/require" +) + +func TestDefaultEvaluator_DivideByZero(t *testing.T) { + ev := &defaultEvaluator{} + + require.Equal(t, true, math.IsNaN(ev.mergeBinOp(OpTypeDiv, + &promql.Sample{ + Point: promql.Point{T: 1, V: 1}, + }, + &promql.Sample{ + Point: promql.Point{T: 1, V: 0}, + }, + ).Point.V)) + + require.Equal(t, true, math.IsNaN(ev.mergeBinOp(OpTypeMod, + &promql.Sample{ + Point: promql.Point{T: 1, V: 1}, + }, + &promql.Sample{ + Point: promql.Point{T: 1, V: 0}, + }, + ).Point.V)) +} diff --git a/pkg/logql/expr.y b/pkg/logql/expr.y index 76f22dffc174..2e8e68d725c8 100644 --- a/pkg/logql/expr.y +++ b/pkg/logql/expr.y @@ -51,8 +51,10 @@ import ( OPEN_PARENTHESIS CLOSE_PARENTHESIS BY WITHOUT COUNT_OVER_TIME RATE SUM AVG MAX MIN COUNT STDDEV STDVAR BOTTOMK TOPK // Operators are listed with increasing precedence. -%left ADD -%left DIV +%left OR +%left AND UNLESS +%left ADD SUB +%left MUL DIV MOD %% @@ -125,10 +127,19 @@ matcher: ; // TODO(owen-d): add (on,ignoring) clauses to binOpExpr -// https://prometheus.io/docs/prometheus/latest/querying/operators/ +// Comparison operators are currently avoided due to symbol collisions in our grammar: "!=" means not equal in prometheus, +// but is part of our filter grammar. +// reference: https://prometheus.io/docs/prometheus/latest/querying/operators/ +// Operator precedence only works if each of these is listed separately. binOpExpr: - expr ADD expr { $$ = mustNewBinOpExpr("+", $1, $3) } - | expr DIV expr { $$ = mustNewBinOpExpr("/", $1, $3) } + expr OR expr { $$ = mustNewBinOpExpr("or", $1, $3) } + | expr AND expr { $$ = mustNewBinOpExpr("and", $1, $3) } + | expr UNLESS expr { $$ = mustNewBinOpExpr("unless", $1, $3) } + | expr ADD expr { $$ = mustNewBinOpExpr("+", $1, $3) } + | expr SUB expr { $$ = mustNewBinOpExpr("-", $1, $3) } + | expr MUL expr { $$ = mustNewBinOpExpr("*", $1, $3) } + | expr DIV expr { $$ = mustNewBinOpExpr("/", $1, $3) } + | expr MOD expr { $$ = mustNewBinOpExpr("%", $1, $3) } vectorOp: SUM { $$ = OpTypeSum } diff --git a/pkg/logql/expr.y.go b/pkg/logql/expr.y.go index ace0d61aab98..72c1a935ecaf 100644 --- a/pkg/logql/expr.y.go +++ b/pkg/logql/expr.y.go @@ -64,8 +64,14 @@ const STDDEV = 57374 const STDVAR = 57375 const BOTTOMK = 57376 const TOPK = 57377 -const ADD = 57378 -const DIV = 57379 +const OR = 57378 +const AND = 57379 +const UNLESS = 57380 +const ADD = 57381 +const SUB = 57382 +const MUL = 57383 +const DIV = 57384 +const MOD = 57385 var exprToknames = [...]string{ "$end", @@ -103,8 +109,14 @@ var exprToknames = [...]string{ "STDVAR", "BOTTOMK", "TOPK", + "OR", + "AND", + "UNLESS", "ADD", + "SUB", + "MUL", "DIV", + "MOD", } var exprStatenames = [...]string{} @@ -122,54 +134,71 @@ var exprExca = [...]int{ 22, 2, 36, 2, 37, 2, + 38, 2, + 39, 2, + 40, 2, + 41, 2, + 42, 2, + 43, 2, -2, 0, - -1, 32, + -1, 38, 36, 2, 37, 2, + 38, 2, + 39, 2, + 40, 2, + 41, 2, + 42, 2, + 43, 2, -2, 0, } const exprPrivate = 57344 -const exprLast = 157 +const exprLast = 199 var exprAct = [...]int{ - 35, 5, 4, 25, 40, 76, 11, 3, 48, 23, - 24, 24, 36, 37, 7, 32, 46, 97, 12, 13, - 14, 15, 17, 18, 16, 19, 20, 21, 22, 96, - 23, 24, 34, 93, 36, 37, 52, 51, 95, 93, - 92, 49, 2, 91, 94, 72, 71, 11, 56, 55, - 31, 54, 65, 33, 73, 50, 75, 74, 69, 70, - 53, 26, 78, 60, 79, 68, 42, 43, 83, 30, - 82, 29, 87, 88, 65, 89, 90, 81, 27, 28, - 58, 12, 13, 14, 15, 17, 18, 16, 19, 20, - 21, 22, 57, 85, 80, 59, 84, 99, 100, 12, - 13, 14, 15, 17, 18, 16, 19, 20, 21, 22, - 26, 98, 6, 45, 68, 66, 44, 39, 30, 41, - 29, 66, 41, 30, 77, 29, 26, 27, 28, 30, - 47, 29, 27, 28, 30, 86, 29, 26, 27, 28, - 10, 67, 8, 27, 28, 30, 47, 29, 61, 62, - 63, 64, 9, 38, 27, 28, 1, + 41, 5, 4, 31, 46, 88, 58, 3, 60, 26, + 27, 28, 29, 30, 105, 38, 28, 29, 30, 107, + 23, 24, 25, 26, 27, 28, 29, 30, 23, 24, + 25, 26, 27, 28, 29, 30, 40, 68, 42, 43, + 42, 43, 64, 63, 109, 108, 104, 61, 24, 25, + 26, 27, 28, 29, 30, 32, 105, 103, 84, 80, + 83, 106, 67, 36, 77, 35, 66, 39, 87, 86, + 81, 82, 33, 34, 90, 59, 91, 85, 72, 97, + 95, 11, 96, 11, 99, 100, 77, 101, 102, 7, + 65, 62, 70, 12, 13, 14, 15, 17, 18, 16, + 19, 20, 21, 22, 69, 57, 94, 71, 56, 111, + 112, 12, 13, 14, 15, 17, 18, 16, 19, 20, + 21, 22, 12, 13, 14, 15, 17, 18, 16, 19, + 20, 21, 22, 2, 78, 93, 92, 45, 32, 47, + 6, 37, 36, 110, 35, 47, 36, 89, 35, 10, + 8, 33, 34, 9, 98, 33, 34, 48, 49, 50, + 51, 52, 53, 54, 55, 78, 44, 1, 0, 32, + 0, 0, 0, 36, 0, 35, 0, 36, 0, 35, + 32, 0, 33, 34, 80, 79, 33, 34, 36, 59, + 35, 73, 74, 75, 76, 0, 0, 33, 34, } var exprPact = [...]int{ - -7, -1000, -27, 135, -1000, -1000, -1000, -7, -1000, 32, - 11, 115, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -7, -7, 111, -1000, -1000, -1000, -1000, - -1000, -6, 124, 34, 56, 30, 28, 27, 78, 49, - -1000, 139, -26, -1000, -1000, -1000, -1000, -1000, 119, 59, - 34, 24, 23, 37, 74, 120, 120, -1000, -1000, 118, - -1000, 89, 72, 65, 63, 91, -1000, -1000, -1000, 108, - 113, -11, -11, 74, 21, 18, 22, -1000, 16, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 7, - -5, -1000, -1000, 107, -1000, -1000, -11, -11, -1000, -1000, - -1000, + 68, -1000, -8, 136, -1000, -1000, -1000, 68, -1000, 46, + 15, 135, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, 68, 68, 68, 68, 68, 68, 68, + 68, 103, -1000, -1000, -1000, -1000, -1000, -16, 167, 70, + 86, 45, 41, 16, 90, 64, -1000, 182, 11, -30, + -30, -25, -25, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + 163, 178, 70, 38, 36, 60, 97, 143, 143, -1000, + -1000, 141, -1000, 131, 130, 101, 75, 77, -1000, -1000, + -1000, 53, 132, 17, 17, 97, 35, 24, 39, -1000, + -3, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, 23, 22, -1000, -1000, 139, -1000, -1000, 17, 17, + -1000, -1000, -1000, } var exprPgo = [...]int{ - 0, 156, 42, 3, 0, 5, 7, 8, 4, 153, - 2, 152, 142, 1, 140, 112, + 0, 167, 133, 3, 0, 5, 7, 8, 4, 166, + 2, 153, 150, 1, 149, 140, } var exprR1 = [...]int{ @@ -177,8 +206,9 @@ var exprR1 = [...]int{ 6, 6, 7, 7, 7, 7, 7, 10, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 3, 3, 3, 3, 12, 12, 12, 9, 9, 8, 8, 8, - 8, 15, 15, 14, 14, 14, 14, 14, 14, 14, - 14, 14, 11, 11, 5, 5, 4, 4, + 8, 15, 15, 15, 15, 15, 15, 15, 15, 14, + 14, 14, 14, 14, 14, 14, 14, 14, 11, 11, + 5, 5, 4, 4, } var exprR2 = [...]int{ @@ -186,36 +216,39 @@ var exprR2 = [...]int{ 3, 2, 2, 3, 3, 3, 2, 4, 4, 4, 5, 5, 5, 5, 6, 7, 6, 7, 1, 1, 1, 1, 3, 3, 3, 1, 3, 3, 3, 3, - 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 3, 4, 4, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 3, 4, 4, } var exprChk = [...]int{ -1000, -1, -2, -6, -10, -13, -15, 21, -12, -11, -14, 13, 25, 26, 27, 28, 31, 29, 30, 32, - 33, 34, 35, 36, 37, -3, 2, 19, 20, 12, - 10, -2, -6, 21, 21, -4, 23, 24, -9, 2, - -8, 4, -2, -2, 5, 2, 22, 22, -7, -6, - 21, -10, -13, 4, 21, 21, 21, 14, 2, 17, - 14, 9, 10, 11, 12, -3, 2, 22, 6, -6, - -7, 22, 22, 17, -10, -13, -5, 4, -5, -8, - 5, 5, 5, 5, 5, 2, 22, -4, -4, -13, - -10, 22, 22, 17, 22, 22, 22, 22, 4, -4, - -4, + 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, + 43, -3, 2, 19, 20, 12, 10, -2, -6, 21, + 21, -4, 23, 24, -9, 2, -8, 4, -2, -2, + -2, -2, -2, -2, -2, -2, 5, 2, 22, 22, + -7, -6, 21, -10, -13, 4, 21, 21, 21, 14, + 2, 17, 14, 9, 10, 11, 12, -3, 2, 22, + 6, -6, -7, 22, 22, 17, -10, -13, -5, 4, + -5, -8, 5, 5, 5, 5, 5, 2, 22, -4, + -4, -13, -10, 22, 22, 17, 22, 22, 22, 22, + 4, -4, -4, } var exprDef = [...]int{ 0, -2, 1, -2, 3, 4, 5, 0, 7, 0, - 0, 0, 52, 53, 43, 44, 45, 46, 47, 48, - 49, 50, 51, 0, 0, 0, 11, 28, 29, 30, - 31, 0, -2, 0, 0, 0, 0, 0, 0, 0, - 35, 0, 41, 42, 8, 10, 6, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 32, 33, 0, - 34, 0, 0, 0, 0, 0, 16, 17, 12, 0, - 0, 18, 19, 0, 0, 0, 0, 54, 0, 36, - 37, 38, 39, 40, 13, 15, 14, 22, 23, 0, - 0, 20, 21, 0, 56, 57, 24, 26, 55, 25, - 27, + 0, 0, 58, 59, 49, 50, 51, 52, 53, 54, + 55, 56, 57, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 11, 28, 29, 30, 31, 0, -2, 0, + 0, 0, 0, 0, 0, 0, 35, 0, 41, 42, + 43, 44, 45, 46, 47, 48, 8, 10, 6, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, + 33, 0, 34, 0, 0, 0, 0, 0, 16, 17, + 12, 0, 0, 18, 19, 0, 0, 0, 0, 60, + 0, 36, 37, 38, 39, 40, 13, 15, 14, 22, + 23, 0, 0, 20, 21, 0, 62, 63, 24, 26, + 61, 25, 27, } var exprTok1 = [...]int{ @@ -226,7 +259,8 @@ var exprTok2 = [...]int{ 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, 34, 35, 36, 37, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, + 42, 43, } var exprTok3 = [...]int{ 0, @@ -750,84 +784,114 @@ exprdefault: case 41: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.BinOpExpr = mustNewBinOpExpr("+", exprDollar[1].Expr, exprDollar[3].Expr) + exprVAL.BinOpExpr = mustNewBinOpExpr("or", exprDollar[1].Expr, exprDollar[3].Expr) } case 42: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.BinOpExpr = mustNewBinOpExpr("/", exprDollar[1].Expr, exprDollar[3].Expr) + exprVAL.BinOpExpr = mustNewBinOpExpr("and", exprDollar[1].Expr, exprDollar[3].Expr) } case 43: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.BinOpExpr = mustNewBinOpExpr("unless", exprDollar[1].Expr, exprDollar[3].Expr) + } + case 44: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.BinOpExpr = mustNewBinOpExpr("+", exprDollar[1].Expr, exprDollar[3].Expr) + } + case 45: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.BinOpExpr = mustNewBinOpExpr("-", exprDollar[1].Expr, exprDollar[3].Expr) + } + case 46: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.BinOpExpr = mustNewBinOpExpr("*", exprDollar[1].Expr, exprDollar[3].Expr) + } + case 47: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.BinOpExpr = mustNewBinOpExpr("/", exprDollar[1].Expr, exprDollar[3].Expr) + } + case 48: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.BinOpExpr = mustNewBinOpExpr("%", exprDollar[1].Expr, exprDollar[3].Expr) + } + case 49: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeSum } - case 44: + case 50: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeAvg } - case 45: + case 51: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeCount } - case 46: + case 52: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMax } - case 47: + case 53: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMin } - case 48: + case 54: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStddev } - case 49: + case 55: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStdvar } - case 50: + case 56: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeBottomK } - case 51: + case 57: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeTopK } - case 52: + case 58: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpTypeCountOverTime } - case 53: + case 59: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpTypeRate } - case 54: + case 60: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Labels = []string{exprDollar[1].str} } - case 55: + case 61: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Labels = append(exprDollar[1].Labels, exprDollar[3].str) } - case 56: + case 62: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: false, groups: exprDollar[3].Labels} } - case 57: + case 63: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: true, groups: exprDollar[3].Labels} diff --git a/pkg/logql/lex.go b/pkg/logql/lex.go index c3812876872a..0d70fd551529 100644 --- a/pkg/logql/lex.go +++ b/pkg/logql/lex.go @@ -36,8 +36,16 @@ var tokens = map[string]int{ OpTypeStdvar: STDVAR, OpTypeBottomK: BOTTOMK, OpTypeTopK: TOPK, - "/": DIV, - "+": ADD, + + // binops + OpTypeOr: OR, + OpTypeAnd: AND, + OpTypeUnless: UNLESS, + OpTypeAdd: ADD, + OpTypeSub: SUB, + OpTypeMul: MUL, + OpTypeDiv: DIV, + OpTypeMod: MOD, } type lexer struct { From 3f193851aaf1b034bad4d67be862aa5c2783c986 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Fri, 7 Feb 2020 17:58:31 -0500 Subject: [PATCH 07/10] updates docs --- docs/logql.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/docs/logql.md b/docs/logql.md index ba5d2b2980e7..a100ed5b3530 100644 --- a/docs/logql.md +++ b/docs/logql.md @@ -151,3 +151,49 @@ by level: Get the rate of HTTP GET requests from NGINX logs: > `avg(rate(({job="nginx"} |= "GET")[10s])) by (region)` + +### Binary Operators + +#### Arithmetic Binary Operators + +Arithmetic binary operators +The following binary arithmetic operators exist in Loki: + +- + (addition) +- - (subtraction) +- * (multiplication) +- / (division) +- % (modulo) + +Binary arithmetic operators are defined only between two vectors. + +Between two instant vectors, a binary arithmetic operator is applied to each entry in the left-hand side vector and its matching element in the right-hand vector. The result is propagated into the result vector with the grouping labels becoming the output label set. Entries for which no matching entry in the right-hand vector can be found are not part of the result. + +##### Examples + +Get proportion of warning logs to error logs for the `foo` app + +> `sum(rate({app="foo", level="warn"}[1m])) / sum(rate({app="foo", level="error"}[1m]))` + +Operators on the same precedence level are left-associative (queries substituted with numbers here for simplicity). For example, 2 * 3 % 2 is equivalent to (2 * 3) % 2. However, some operators have different priorities: 1 + 2 / 3 will still be 1 + ( 2 / 3 ). These function identically to mathematical conventions. + + +#### Logical/set binary operators + +These logical/set binary operators are only defined between two vectors: + +- and (intersection) +- or (union) +- unless (complement) + +`vector1 and vector2` results in a vector consisting of the elements of vector1 for which there are elements in vector2 with exactly matching label sets. Other elements are dropped. + +`vector1 or vector2` results in a vector that contains all original elements (label sets + values) of vector1 and additionally all elements of vector2 which do not have matching label sets in vector1. + +`vector1 unless vector2` results in a vector consisting of the elements of vector1 for which there are no elements in vector2 with exactly matching label sets. All matching elements in both vectors are dropped. + +##### Examples + +This contrived query will return the intersection of these queries, effectively `rate({app="bar"})` + +> `rate({app=~"foo|bar"}[1m]) and rate({app="bar"}[1m])` From 0ec42b09ec16568777914d3bc10ebf3a89145d22 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Fri, 7 Feb 2020 18:40:59 -0500 Subject: [PATCH 08/10] changelog --- CHANGELOG.md | 1 + docs/logql.md | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c910b53664e..52833472abde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Features +* [1662](https://github.com/grafana/loki/pull/1662) **owen-d**: Introduces binary operators in LogQL * [1572](https://github.com/grafana/loki/pull/1572) **owen-d**: Introduces the `querier.query-ingesters-within` flag and associated yaml config. When enabled, queries for a time range that do not overlap this lookback interval will not be sent to the ingesters. * [1558](https://github.com/grafana/loki/pull/1558) **owen-d**: Introduces `ingester.max-chunk-age` which specifies the maximum chunk age before it's cut. * [1565](https://github.com/grafana/loki/pull/1565) **owen-d**: The query frontend's `split_queries_by_interval` can now be specified as an override diff --git a/docs/logql.md b/docs/logql.md index a100ed5b3530..5628cee63842 100644 --- a/docs/logql.md +++ b/docs/logql.md @@ -159,11 +159,11 @@ Get the rate of HTTP GET requests from NGINX logs: Arithmetic binary operators The following binary arithmetic operators exist in Loki: -- + (addition) -- - (subtraction) -- * (multiplication) -- / (division) -- % (modulo) +- `+` (addition) +- `-` (subtraction) +- `*` (multiplication) +- `/` (division) +- `%` (modulo) Binary arithmetic operators are defined only between two vectors. @@ -182,9 +182,9 @@ Operators on the same precedence level are left-associative (queries substituted These logical/set binary operators are only defined between two vectors: -- and (intersection) -- or (union) -- unless (complement) +- `and` (intersection) +- `or` (union) +- `unless` (complement) `vector1 and vector2` results in a vector consisting of the elements of vector1 for which there are elements in vector2 with exactly matching label sets. Other elements are dropped. From d5095dde68134c6f772e73b1d3be4c53302b32ef Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Tue, 11 Feb 2020 10:52:10 -0500 Subject: [PATCH 09/10] better logql parsing errors for binops Signed-off-by: Owen Diehl --- pkg/logql/ast.go | 12 ++++++++++-- pkg/logql/engine_test.go | 11 ++++------- pkg/logql/parser_test.go | 40 ++++++++++++++++++++++++++++++++-------- 3 files changed, 46 insertions(+), 17 deletions(-) diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index c1716922ab91..cc06fee302e3 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -393,12 +393,20 @@ func (e *binOpExpr) String() string { func mustNewBinOpExpr(op string, lhs, rhs Expr) SampleExpr { left, ok := lhs.(SampleExpr) if !ok { - panic(fmt.Errorf("unexpected type for binOpExpr (%T): %+v", lhs, lhs)) + panic(newParseError(fmt.Sprintf( + "unexpected type for left leg of binary operation (%s): %T", + op, + lhs, + ), 0, 0)) } right, ok := rhs.(SampleExpr) if !ok { - panic(fmt.Errorf("unexpected type for binOpExpr (%T): %+v", rhs, rhs)) + panic(newParseError(fmt.Sprintf( + "unexpected type for right leg of binary operation (%s): %T", + op, + rhs, + ), 0, 0)) } return &binOpExpr{ SampleExpr: left, diff --git a/pkg/logql/engine_test.go b/pkg/logql/engine_test.go index 44c6b7805d2a..2ef46deab957 100644 --- a/pkg/logql/engine_test.go +++ b/pkg/logql/engine_test.go @@ -682,10 +682,7 @@ func TestEngine_NewRangeQuery(t *testing.T) { }, // binops { - ` -rate({app="foo"}[1m]) or -rate({app="bar"}[1m]) -`, + `rate({app="foo"}[1m]) or rate({app="bar"}[1m])`, time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, [][]*logproto.Stream{ { @@ -712,9 +709,9 @@ rate({app="bar"}[1m]) }, { ` -rate({app=~"foo|bar"}[1m]) and -rate({app="bar"}[1m]) -`, + rate({app=~"foo|bar"}[1m]) and + rate({app="bar"}[1m]) + `, time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, [][]*logproto.Stream{ { diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go index 11c92081c55e..56441838dc7a 100644 --- a/pkg/logql/parser_test.go +++ b/pkg/logql/parser_test.go @@ -550,10 +550,10 @@ func TestParse(t *testing.T) { { // require left associativity in: ` -sum(count_over_time({foo="bar"}[5m])) by (foo) / -sum(count_over_time({foo="bar"}[5m])) by (foo) / -sum(count_over_time({foo="bar"}[5m])) by (foo) -`, + sum(count_over_time({foo="bar"}[5m])) by (foo) / + sum(count_over_time({foo="bar"}[5m])) by (foo) / + sum(count_over_time({foo="bar"}[5m])) by (foo) + `, exp: mustNewBinOpExpr( OpTypeDiv, mustNewBinOpExpr( @@ -612,10 +612,10 @@ sum(count_over_time({foo="bar"}[5m])) by (foo) { // operator precedence before left associativity in: ` -sum(count_over_time({foo="bar"}[5m])) by (foo) + -sum(count_over_time({foo="bar"}[5m])) by (foo) / -sum(count_over_time({foo="bar"}[5m])) by (foo) -`, + sum(count_over_time({foo="bar"}[5m])) by (foo) + + sum(count_over_time({foo="bar"}[5m])) by (foo) / + sum(count_over_time({foo="bar"}[5m])) by (foo) + `, exp: mustNewBinOpExpr( OpTypeAdd, mustNewVectorAggregationExpr(newRangeAggregationExpr( @@ -671,6 +671,30 @@ sum(count_over_time({foo="bar"}[5m])) by (foo) ), ), }, + { + in: `{foo="bar"} + {foo="bar"}`, + err: ParseError{ + msg: `unexpected type for left leg of binary operation (+): *logql.matchersExpr`, + line: 0, + col: 0, + }, + }, + { + in: `sum(count_over_time({foo="bar"}[5m])) by (foo) - {foo="bar"}`, + err: ParseError{ + msg: `unexpected type for right leg of binary operation (-): *logql.matchersExpr`, + line: 0, + col: 0, + }, + }, + { + in: `{foo="bar"} / sum(count_over_time({foo="bar"}[5m])) by (foo)`, + err: ParseError{ + msg: `unexpected type for left leg of binary operation (/): *logql.matchersExpr`, + line: 0, + col: 0, + }, + }, } { t.Run(tc.in, func(t *testing.T) { ast, err := ParseExpr(tc.in) From e4ed85da7718479dc2abcbea82d3cd1c6683d7b1 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Tue, 11 Feb 2020 11:26:04 -0500 Subject: [PATCH 10/10] adds ^ operator --- docs/logql.md | 1 + pkg/logql/ast.go | 1 + pkg/logql/engine_test.go | 21 +++++ pkg/logql/evaluator.go | 14 ++++ pkg/logql/expr.y | 2 + pkg/logql/expr.y.go | 162 +++++++++++++++++++++------------------ pkg/logql/lex.go | 1 + pkg/logql/parser_test.go | 5 +- 8 files changed, 128 insertions(+), 79 deletions(-) diff --git a/docs/logql.md b/docs/logql.md index 5628cee63842..b7ff4168111c 100644 --- a/docs/logql.md +++ b/docs/logql.md @@ -164,6 +164,7 @@ The following binary arithmetic operators exist in Loki: - `*` (multiplication) - `/` (division) - `%` (modulo) +- `^` (power/exponentiation) Binary arithmetic operators are defined only between two vectors. diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index cc06fee302e3..e00659f9f590 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -237,6 +237,7 @@ const ( OpTypeMul = "*" OpTypeDiv = "/" OpTypeMod = "%" + OpTypePow = "^" ) // SampleExpr is a LogQL expression filtering logs and returning metric samples. diff --git a/pkg/logql/engine_test.go b/pkg/logql/engine_test.go index 2ef46deab957..347ce06687de 100644 --- a/pkg/logql/engine_test.go +++ b/pkg/logql/engine_test.go @@ -3,6 +3,7 @@ package logql import ( "context" "fmt" + "math" "testing" "time" @@ -943,6 +944,26 @@ func TestEngine_NewRangeQuery(t *testing.T) { }, }, }, + { + ` + count_over_time({app="bar"}[1m]) ^ count_over_time({app="bar"}[1m]) + `, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: math.Pow(12, 12)}, {T: 90 * 1000, V: math.Pow(12, 12)}, {T: 120 * 1000, V: math.Pow(12, 12)}, {T: 150 * 1000, V: math.Pow(12, 12)}, {T: 180 * 1000, V: math.Pow(12, 12)}}, + }, + }, + }, } { test := test t.Run(fmt.Sprintf("%s %s", test.qs, test.direction), func(t *testing.T) { diff --git a/pkg/logql/evaluator.go b/pkg/logql/evaluator.go index e740f6990176..2b951124d3eb 100644 --- a/pkg/logql/evaluator.go +++ b/pkg/logql/evaluator.go @@ -497,6 +497,20 @@ func (ev *defaultEvaluator) mergeBinOp(op string, left, right *promql.Sample) *p return &res } + case OpTypePow: + merger = func(left, right *promql.Sample) *promql.Sample { + if left == nil || right == nil { + return nil + } + + res := promql.Sample{ + Metric: left.Metric, + Point: left.Point, + } + res.Point.V = math.Pow(left.Point.V, right.Point.V) + return &res + } + default: panic(errors.Errorf("should never happen: unexpected operation: (%s)", op)) } diff --git a/pkg/logql/expr.y b/pkg/logql/expr.y index 2e8e68d725c8..9a811985191f 100644 --- a/pkg/logql/expr.y +++ b/pkg/logql/expr.y @@ -55,6 +55,7 @@ import ( %left AND UNLESS %left ADD SUB %left MUL DIV MOD +%right POW %% @@ -140,6 +141,7 @@ binOpExpr: | expr MUL expr { $$ = mustNewBinOpExpr("*", $1, $3) } | expr DIV expr { $$ = mustNewBinOpExpr("/", $1, $3) } | expr MOD expr { $$ = mustNewBinOpExpr("%", $1, $3) } + | expr POW expr { $$ = mustNewBinOpExpr("^", $1, $3) } vectorOp: SUM { $$ = OpTypeSum } diff --git a/pkg/logql/expr.y.go b/pkg/logql/expr.y.go index 72c1a935ecaf..6e2a851a6a0a 100644 --- a/pkg/logql/expr.y.go +++ b/pkg/logql/expr.y.go @@ -72,6 +72,7 @@ const SUB = 57382 const MUL = 57383 const DIV = 57384 const MOD = 57385 +const POW = 57386 var exprToknames = [...]string{ "$end", @@ -117,6 +118,7 @@ var exprToknames = [...]string{ "MUL", "DIV", "MOD", + "POW", } var exprStatenames = [...]string{} @@ -140,8 +142,9 @@ var exprExca = [...]int{ 41, 2, 42, 2, 43, 2, + 44, 2, -2, 0, - -1, 38, + -1, 39, 36, 2, 37, 2, 38, 2, @@ -150,55 +153,57 @@ var exprExca = [...]int{ 41, 2, 42, 2, 43, 2, + 44, 2, -2, 0, } const exprPrivate = 57344 -const exprLast = 199 +const exprLast = 202 var exprAct = [...]int{ - 41, 5, 4, 31, 46, 88, 58, 3, 60, 26, - 27, 28, 29, 30, 105, 38, 28, 29, 30, 107, - 23, 24, 25, 26, 27, 28, 29, 30, 23, 24, - 25, 26, 27, 28, 29, 30, 40, 68, 42, 43, - 42, 43, 64, 63, 109, 108, 104, 61, 24, 25, - 26, 27, 28, 29, 30, 32, 105, 103, 84, 80, - 83, 106, 67, 36, 77, 35, 66, 39, 87, 86, - 81, 82, 33, 34, 90, 59, 91, 85, 72, 97, - 95, 11, 96, 11, 99, 100, 77, 101, 102, 7, - 65, 62, 70, 12, 13, 14, 15, 17, 18, 16, - 19, 20, 21, 22, 69, 57, 94, 71, 56, 111, - 112, 12, 13, 14, 15, 17, 18, 16, 19, 20, - 21, 22, 12, 13, 14, 15, 17, 18, 16, 19, - 20, 21, 22, 2, 78, 93, 92, 45, 32, 47, - 6, 37, 36, 110, 35, 47, 36, 89, 35, 10, - 8, 33, 34, 9, 98, 33, 34, 48, 49, 50, - 51, 52, 53, 54, 55, 78, 44, 1, 0, 32, - 0, 0, 0, 36, 0, 35, 0, 36, 0, 35, - 32, 0, 33, 34, 80, 79, 33, 34, 36, 59, - 35, 73, 74, 75, 76, 0, 0, 33, 34, + 42, 5, 4, 32, 47, 90, 60, 3, 62, 26, + 27, 28, 29, 30, 31, 39, 28, 29, 30, 31, + 23, 24, 25, 26, 27, 28, 29, 30, 31, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 31, 41, + 70, 43, 44, 66, 65, 43, 44, 111, 63, 24, + 25, 26, 27, 28, 29, 30, 31, 33, 107, 107, + 110, 82, 106, 109, 108, 37, 79, 36, 105, 86, + 89, 88, 83, 84, 34, 35, 92, 61, 93, 85, + 69, 68, 40, 11, 87, 11, 101, 102, 79, 103, + 104, 7, 67, 64, 72, 12, 13, 14, 15, 17, + 18, 16, 19, 20, 21, 22, 71, 74, 97, 73, + 96, 113, 114, 12, 13, 14, 15, 17, 18, 16, + 19, 20, 21, 22, 12, 13, 14, 15, 17, 18, + 16, 19, 20, 21, 22, 2, 80, 95, 99, 59, + 33, 98, 58, 38, 37, 94, 36, 46, 37, 48, + 36, 112, 48, 34, 35, 91, 100, 34, 35, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 80, 6, + 10, 8, 33, 9, 45, 1, 37, 0, 36, 0, + 37, 0, 36, 33, 0, 34, 35, 82, 81, 34, + 35, 37, 61, 36, 75, 76, 77, 78, 0, 0, + 34, 35, } var exprPact = [...]int{ - 68, -1000, -8, 136, -1000, -1000, -1000, 68, -1000, 46, - 15, 135, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, 68, 68, 68, 68, 68, 68, 68, - 68, 103, -1000, -1000, -1000, -1000, -1000, -16, 167, 70, - 86, 45, 41, 16, 90, 64, -1000, 182, 11, -30, - -30, -25, -25, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - 163, 178, 70, 38, 36, 60, 97, 143, 143, -1000, - -1000, 141, -1000, 131, 130, 101, 75, 77, -1000, -1000, - -1000, 53, 132, 17, 17, 97, 35, 24, 39, -1000, - -3, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, 23, 22, -1000, -1000, 139, -1000, -1000, 17, 17, - -1000, -1000, -1000, + 70, -1000, -7, 138, -1000, -1000, -1000, 70, -1000, 61, + 18, 145, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 137, -1000, -1000, -1000, -1000, -1000, -16, 170, + 72, 88, 60, 59, 19, 92, 93, -1000, 185, 12, + -30, -30, -25, -25, -6, -6, -6, -6, -1000, -1000, + -1000, -1000, 166, 181, 72, 57, 47, 67, 99, 151, + 151, -1000, -1000, 148, -1000, 140, 132, 105, 103, 136, + -1000, -1000, -1000, 55, 134, 22, 22, 99, 46, 40, + 42, -1000, 41, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, 38, 25, -1000, -1000, 147, -1000, -1000, + 22, 22, -1000, -1000, -1000, } var exprPgo = [...]int{ - 0, 167, 133, 3, 0, 5, 7, 8, 4, 166, - 2, 153, 150, 1, 149, 140, + 0, 175, 135, 3, 0, 5, 7, 8, 4, 174, + 2, 173, 171, 1, 170, 169, } var exprR1 = [...]int{ @@ -206,9 +211,9 @@ var exprR1 = [...]int{ 6, 6, 7, 7, 7, 7, 7, 10, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 3, 3, 3, 3, 12, 12, 12, 9, 9, 8, 8, 8, - 8, 15, 15, 15, 15, 15, 15, 15, 15, 14, - 14, 14, 14, 14, 14, 14, 14, 14, 11, 11, - 5, 5, 4, 4, + 8, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 14, 14, 14, 14, 14, 14, 14, 14, 14, 11, + 11, 5, 5, 4, 4, } var exprR2 = [...]int{ @@ -216,39 +221,39 @@ var exprR2 = [...]int{ 3, 2, 2, 3, 3, 3, 2, 4, 4, 4, 5, 5, 5, 5, 6, 7, 6, 7, 1, 1, 1, 1, 3, 3, 3, 1, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 3, 4, 4, + 1, 1, 3, 4, 4, } var exprChk = [...]int{ -1000, -1, -2, -6, -10, -13, -15, 21, -12, -11, -14, 13, 25, 26, 27, 28, 31, 29, 30, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, - 43, -3, 2, 19, 20, 12, 10, -2, -6, 21, - 21, -4, 23, 24, -9, 2, -8, 4, -2, -2, - -2, -2, -2, -2, -2, -2, 5, 2, 22, 22, - -7, -6, 21, -10, -13, 4, 21, 21, 21, 14, - 2, 17, 14, 9, 10, 11, 12, -3, 2, 22, - 6, -6, -7, 22, 22, 17, -10, -13, -5, 4, - -5, -8, 5, 5, 5, 5, 5, 2, 22, -4, - -4, -13, -10, 22, 22, 17, 22, 22, 22, 22, - 4, -4, -4, + 43, 44, -3, 2, 19, 20, 12, 10, -2, -6, + 21, 21, -4, 23, 24, -9, 2, -8, 4, -2, + -2, -2, -2, -2, -2, -2, -2, -2, 5, 2, + 22, 22, -7, -6, 21, -10, -13, 4, 21, 21, + 21, 14, 2, 17, 14, 9, 10, 11, 12, -3, + 2, 22, 6, -6, -7, 22, 22, 17, -10, -13, + -5, 4, -5, -8, 5, 5, 5, 5, 5, 2, + 22, -4, -4, -13, -10, 22, 22, 17, 22, 22, + 22, 22, 4, -4, -4, } var exprDef = [...]int{ 0, -2, 1, -2, 3, 4, 5, 0, 7, 0, - 0, 0, 58, 59, 49, 50, 51, 52, 53, 54, - 55, 56, 57, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 11, 28, 29, 30, 31, 0, -2, 0, - 0, 0, 0, 0, 0, 0, 35, 0, 41, 42, - 43, 44, 45, 46, 47, 48, 8, 10, 6, 9, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, - 33, 0, 34, 0, 0, 0, 0, 0, 16, 17, - 12, 0, 0, 18, 19, 0, 0, 0, 0, 60, - 0, 36, 37, 38, 39, 40, 13, 15, 14, 22, - 23, 0, 0, 20, 21, 0, 62, 63, 24, 26, - 61, 25, 27, + 0, 0, 59, 60, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 11, 28, 29, 30, 31, 0, -2, + 0, 0, 0, 0, 0, 0, 0, 35, 0, 41, + 42, 43, 44, 45, 46, 47, 48, 49, 8, 10, + 6, 9, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 32, 33, 0, 34, 0, 0, 0, 0, 0, + 16, 17, 12, 0, 0, 18, 19, 0, 0, 0, + 0, 61, 0, 36, 37, 38, 39, 40, 13, 15, + 14, 22, 23, 0, 0, 20, 21, 0, 63, 64, + 24, 26, 62, 25, 27, } var exprTok1 = [...]int{ @@ -260,7 +265,7 @@ var exprTok2 = [...]int{ 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, - 42, 43, + 42, 43, 44, } var exprTok3 = [...]int{ 0, @@ -822,76 +827,81 @@ exprdefault: exprVAL.BinOpExpr = mustNewBinOpExpr("%", exprDollar[1].Expr, exprDollar[3].Expr) } case 49: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.BinOpExpr = mustNewBinOpExpr("^", exprDollar[1].Expr, exprDollar[3].Expr) + } + case 50: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeSum } - case 50: + case 51: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeAvg } - case 51: + case 52: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeCount } - case 52: + case 53: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMax } - case 53: + case 54: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMin } - case 54: + case 55: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStddev } - case 55: + case 56: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStdvar } - case 56: + case 57: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeBottomK } - case 57: + case 58: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeTopK } - case 58: + case 59: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpTypeCountOverTime } - case 59: + case 60: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpTypeRate } - case 60: + case 61: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Labels = []string{exprDollar[1].str} } - case 61: + case 62: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Labels = append(exprDollar[1].Labels, exprDollar[3].str) } - case 62: + case 63: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: false, groups: exprDollar[3].Labels} } - case 63: + case 64: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: true, groups: exprDollar[3].Labels} diff --git a/pkg/logql/lex.go b/pkg/logql/lex.go index 0d70fd551529..eedc0dbc9b57 100644 --- a/pkg/logql/lex.go +++ b/pkg/logql/lex.go @@ -46,6 +46,7 @@ var tokens = map[string]int{ OpTypeMul: MUL, OpTypeDiv: DIV, OpTypeMod: MOD, + OpTypePow: POW, } type lexer struct { diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go index 56441838dc7a..a2c9d0103427 100644 --- a/pkg/logql/parser_test.go +++ b/pkg/logql/parser_test.go @@ -548,16 +548,15 @@ func TestParse(t *testing.T) { }, }, { - // require left associativity in: ` - sum(count_over_time({foo="bar"}[5m])) by (foo) / + sum(count_over_time({foo="bar"}[5m])) by (foo) ^ sum(count_over_time({foo="bar"}[5m])) by (foo) / sum(count_over_time({foo="bar"}[5m])) by (foo) `, exp: mustNewBinOpExpr( OpTypeDiv, mustNewBinOpExpr( - OpTypeDiv, + OpTypePow, mustNewVectorAggregationExpr(newRangeAggregationExpr( &logRange{ left: &matchersExpr{