diff --git a/kubernetes/controllers/servicelevelobjective_test.go b/kubernetes/controllers/servicelevelobjective_test.go index b614beeae..07d308e58 100644 --- a/kubernetes/controllers/servicelevelobjective_test.go +++ b/kubernetes/controllers/servicelevelobjective_test.go @@ -89,7 +89,7 @@ func Test_makePrometheusRule(t *testing.T) { Rules: []monitoringv1.Rule{ { Record: "http_requests:increase4w", - Expr: intstr.FromString(`sum by(status) (increase(http_requests_total{job="app"}[4w]))`), + Expr: intstr.FromString(`sum by (status) (increase(http_requests_total{job="app"}[4w]))`), Labels: map[string]string{ "job": "app", "slo": "http", @@ -200,7 +200,7 @@ func Test_makeConfigMap(t *testing.T) { - interval: 2m30s name: http-increase rules: - - expr: sum by(status) (increase(http_requests_total{job="app"}[4w])) + - expr: sum by (status) (increase(http_requests_total{job="app"}[4w])) labels: job: app slo: http diff --git a/slo/promql.go b/slo/promql.go index a629b59b0..9fd62f357 100644 --- a/slo/promql.go +++ b/slo/promql.go @@ -85,7 +85,7 @@ func (o Objective) QueryErrors(window model.Duration) string { } if o.Indicator.Latency != nil && o.Indicator.Latency.Total.Name != "" { - expr, err := parser.ParseExpr(`sum by(grouping) (metric{matchers="total"}) - sum by(grouping) (errorMetric{matchers="errors"})`) + expr, err := parser.ParseExpr(`sum by (grouping) (metric{matchers="total"}) - sum by (grouping) (errorMetric{matchers="errors"})`) if err != nil { return "" } @@ -394,7 +394,7 @@ func (r objectiveReplacer) replace(node parser.Node) { func (o Objective) RequestRange(timerange time.Duration) string { if o.Indicator.Ratio != nil && o.Indicator.Ratio.Total.Name != "" { - expr, err := parser.ParseExpr(`sum by(group) (rate(metric{}[1s])) > 0`) + expr, err := parser.ParseExpr(`sum by (group) (rate(metric{}[1s])) > 0`) if err != nil { return err.Error() } @@ -433,7 +433,7 @@ func (o Objective) RequestRange(timerange time.Duration) string { func (o Objective) ErrorsRange(timerange time.Duration) string { if o.Indicator.Ratio != nil && o.Indicator.Ratio.Total.Name != "" { - expr, err := parser.ParseExpr(`sum by(group) (rate(errorMetric{matchers="errors"}[1s])) / scalar(sum(rate(metric{matchers="total"}[1s]))) > 0`) + expr, err := parser.ParseExpr(`sum by (group) (rate(errorMetric{matchers="errors"}[1s])) / scalar(sum(rate(metric{matchers="total"}[1s]))) > 0`) if err != nil { return err.Error() } @@ -477,7 +477,7 @@ func (o Objective) DurationRange(timerange time.Duration, percentile float64) st } if o.Indicator.Latency != nil && o.Indicator.Latency.Total.Name != "" { - expr, err := parser.ParseExpr(`histogram_quantile(0.420, sum by(le) (rate(errorMetric{matchers="errors"}[1s])))`) + expr, err := parser.ParseExpr(`histogram_quantile(0.420, sum by (le) (rate(errorMetric{matchers="errors"}[1s])))`) if err != nil { return err.Error() } diff --git a/slo/promql_test.go b/slo/promql_test.go index 952366597..0947e681f 100644 --- a/slo/promql_test.go +++ b/slo/promql_test.go @@ -283,11 +283,11 @@ func TestObjective_QueryTotal(t *testing.T) { }, { name: "http-ratio-grouping", objective: objectiveHTTPRatioGrouping(), - expected: `sum by(job, handler) (http_requests:increase4w{job="thanos-receive-default",slo="monitoring-http-errors"})`, + expected: `sum by (job, handler) (http_requests:increase4w{job="thanos-receive-default",slo="monitoring-http-errors"})`, }, { name: "http-ratio-grouping-regex", objective: objectiveHTTPRatioGroupingRegex(), - expected: `sum by(job, handler) (http_requests:increase4w{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"})`, + expected: `sum by (job, handler) (http_requests:increase4w{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"})`, }, { name: "grpc-ratio", objective: objectiveGRPCRatio(), @@ -295,7 +295,7 @@ func TestObjective_QueryTotal(t *testing.T) { }, { name: "grpc-ratio-grouping", objective: objectiveGRPCRatioGrouping(), - expected: `sum by(job, handler) (grpc_server_handled:increase4w{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"})`, + expected: `sum by (job, handler) (grpc_server_handled:increase4w{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"})`, }, { name: "http-latency", objective: objectiveHTTPLatency(), @@ -303,11 +303,11 @@ func TestObjective_QueryTotal(t *testing.T) { }, { name: "http-latency-grouping", objective: objectiveHTTPLatencyGrouping(), - expected: `sum by(job, handler) (http_request_duration_seconds:increase4w{code=~"2..",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"})`, + expected: `sum by (job, handler) (http_request_duration_seconds:increase4w{code=~"2..",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"})`, }, { name: "http-latency-grouping-regex", objective: objectiveHTTPLatencyGroupingRegex(), - expected: `sum by(job, handler) (http_request_duration_seconds:increase4w{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"})`, + expected: `sum by (job, handler) (http_request_duration_seconds:increase4w{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"})`, }, { name: "grpc-latency", objective: objectiveGRPCLatency(), @@ -315,7 +315,7 @@ func TestObjective_QueryTotal(t *testing.T) { }, { name: "grpc-latency-grouping", objective: objectiveGRPCLatencyGrouping(), - expected: `sum by(job, handler) (grpc_server_handling_seconds:increase1w{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"})`, + expected: `sum by (job, handler) (grpc_server_handling_seconds:increase1w{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"})`, }, { name: "operator-ratio", objective: objectiveOperator(), @@ -323,7 +323,7 @@ func TestObjective_QueryTotal(t *testing.T) { }, { name: "operator-ratio-grouping", objective: objectiveOperatorGrouping(), - expected: `sum by(namespace) (prometheus_operator_reconcile_operations:increase2w{slo="monitoring-prometheus-operator-errors"})`, + expected: `sum by (namespace) (prometheus_operator_reconcile_operations:increase2w{slo="monitoring-prometheus-operator-errors"})`, }, { name: "apiserver-write-response-errors", objective: objectiveAPIServerRatio(), @@ -331,7 +331,7 @@ func TestObjective_QueryTotal(t *testing.T) { }, { name: "apiserver-read-resource-latency", objective: objectiveAPIServerLatency(), - expected: `sum by(resource, verb) (apiserver_request_duration_seconds:increase2w{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"})`, + expected: `sum by (resource, verb) (apiserver_request_duration_seconds:increase2w{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"})`, }} for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { @@ -352,11 +352,11 @@ func TestObjective_QueryErrors(t *testing.T) { }, { name: "http-ratio-grouping", objective: objectiveHTTPRatioGrouping(), - expected: `sum by(job, handler) (http_requests:increase4w{code=~"5..",job="thanos-receive-default",slo="monitoring-http-errors"})`, + expected: `sum by (job, handler) (http_requests:increase4w{code=~"5..",job="thanos-receive-default",slo="monitoring-http-errors"})`, }, { name: "http-ratio-grouping-regex", objective: objectiveHTTPRatioGroupingRegex(), - expected: `sum by(job, handler) (http_requests:increase4w{code=~"5..",handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"})`, + expected: `sum by (job, handler) (http_requests:increase4w{code=~"5..",handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"})`, }, { name: "grpc-ratio", objective: objectiveGRPCRatio(), @@ -364,7 +364,7 @@ func TestObjective_QueryErrors(t *testing.T) { }, { name: "grpc-ratio-grouping", objective: objectiveGRPCRatioGrouping(), - expected: `sum by(job, handler) (grpc_server_handled:increase4w{grpc_code=~"Aborted|Unavailable|Internal|Unknown|Unimplemented|DataLoss",grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"})`, + expected: `sum by (job, handler) (grpc_server_handled:increase4w{grpc_code=~"Aborted|Unavailable|Internal|Unknown|Unimplemented|DataLoss",grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"})`, }, { name: "http-latency", objective: objectiveHTTPLatency(), @@ -372,11 +372,11 @@ func TestObjective_QueryErrors(t *testing.T) { }, { name: "http-latency-grouping", objective: objectiveHTTPLatencyGrouping(), - expected: `sum by(job, handler) (http_request_duration_seconds:increase4w{code=~"2..",job="metrics-service-thanos-receive-default",le="",slo="monitoring-http-latency"}) - sum by(job, handler) (http_request_duration_seconds:increase4w{code=~"2..",job="metrics-service-thanos-receive-default",le="1",slo="monitoring-http-latency"})`, + expected: `sum by (job, handler) (http_request_duration_seconds:increase4w{code=~"2..",job="metrics-service-thanos-receive-default",le="",slo="monitoring-http-latency"}) - sum by (job, handler) (http_request_duration_seconds:increase4w{code=~"2..",job="metrics-service-thanos-receive-default",le="1",slo="monitoring-http-latency"})`, }, { name: "http-latency-grouping-regex", objective: objectiveHTTPLatencyGroupingRegex(), - expected: `sum by(job, handler) (http_request_duration_seconds:increase4w{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default",le="",slo="monitoring-http-latency"}) - sum by(job, handler) (http_request_duration_seconds:increase4w{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default",le="1",slo="monitoring-http-latency"})`, + expected: `sum by (job, handler) (http_request_duration_seconds:increase4w{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default",le="",slo="monitoring-http-latency"}) - sum by (job, handler) (http_request_duration_seconds:increase4w{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default",le="1",slo="monitoring-http-latency"})`, }, { name: "grpc-latency", objective: objectiveGRPCLatency(), @@ -384,7 +384,7 @@ func TestObjective_QueryErrors(t *testing.T) { }, { name: "grpc-latency-grouping", objective: objectiveGRPCLatencyGrouping(), - expected: `sum by(job, handler) (grpc_server_handling_seconds:increase1w{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",le="",slo="monitoring-grpc-latency"}) - sum by(job, handler) (grpc_server_handling_seconds:increase1w{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",le="0.6",slo="monitoring-grpc-latency"})`, + expected: `sum by (job, handler) (grpc_server_handling_seconds:increase1w{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",le="",slo="monitoring-grpc-latency"}) - sum by (job, handler) (grpc_server_handling_seconds:increase1w{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",le="0.6",slo="monitoring-grpc-latency"})`, }, { name: "operator-ratio", objective: objectiveOperator(), @@ -392,7 +392,7 @@ func TestObjective_QueryErrors(t *testing.T) { }, { name: "operator-ratio-grouping", objective: objectiveOperatorGrouping(), - expected: `sum by(namespace) (prometheus_operator_reconcile_errors:increase2w{slo="monitoring-prometheus-operator-errors"})`, + expected: `sum by (namespace) (prometheus_operator_reconcile_errors:increase2w{slo="monitoring-prometheus-operator-errors"})`, }, { name: "apiserver-write-response-errors", objective: objectiveAPIServerRatio(), @@ -400,7 +400,7 @@ func TestObjective_QueryErrors(t *testing.T) { }, { name: "apiserver-read-resource-latency", objective: objectiveAPIServerLatency(), - expected: `sum by(resource, verb) (apiserver_request_duration_seconds:increase2w{job="apiserver",le="",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"}) - sum by(resource, verb) (apiserver_request_duration_seconds:increase2w{job="apiserver",le="0.1",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"})`, + expected: `sum by (resource, verb) (apiserver_request_duration_seconds:increase2w{job="apiserver",le="",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"}) - sum by (resource, verb) (apiserver_request_duration_seconds:increase2w{job="apiserver",le="0.1",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"})`, }} for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { @@ -581,27 +581,27 @@ func TestObjective_RequestRange(t *testing.T) { name: "http-ratio", objective: objectiveHTTPRatio(), timerange: 6 * time.Hour, - expected: `sum by(code) (rate(http_requests_total{job="thanos-receive-default"}[6h])) > 0`, + expected: `sum by (code) (rate(http_requests_total{job="thanos-receive-default"}[6h])) > 0`, }, { name: "http-ratio-grouping", objective: objectiveHTTPRatioGrouping(), timerange: 6 * time.Hour, - expected: `sum by(code) (rate(http_requests_total{job="thanos-receive-default"}[6h])) > 0`, + expected: `sum by (code) (rate(http_requests_total{job="thanos-receive-default"}[6h])) > 0`, }, { name: "http-ratio-grouping-regex", objective: objectiveHTTPRatioGroupingRegex(), timerange: 6 * time.Hour, - expected: `sum by(code) (rate(http_requests_total{handler=~"/api.*",job="thanos-receive-default"}[6h])) > 0`, + expected: `sum by (code) (rate(http_requests_total{handler=~"/api.*",job="thanos-receive-default"}[6h])) > 0`, }, { name: "grpc-ratio", objective: objectiveGRPCRatio(), timerange: 6 * time.Hour, - expected: `sum by(grpc_code) (rate(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[6h])) > 0`, + expected: `sum by (grpc_code) (rate(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[6h])) > 0`, }, { name: "grpc-ratio-grouping", objective: objectiveGRPCRatioGrouping(), timerange: 6 * time.Hour, - expected: `sum by(grpc_code) (rate(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[6h])) > 0`, + expected: `sum by (grpc_code) (rate(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[6h])) > 0`, }, { name: "http-latency", objective: objectiveHTTPLatency(), @@ -641,7 +641,7 @@ func TestObjective_RequestRange(t *testing.T) { name: "apiserver-write-response-errors", objective: objectiveAPIServerRatio(), timerange: 2 * time.Hour, - expected: `sum by(code) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h])) > 0`, + expected: `sum by (code) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h])) > 0`, }, { name: "apiserver-read-resource-latency", objective: objectiveAPIServerLatency(), @@ -665,27 +665,27 @@ func TestObjective_ErrorsRange(t *testing.T) { name: "http-ratio", objective: objectiveHTTPRatio(), timerange: 6 * time.Hour, - expected: `sum by(code) (rate(http_requests_total{code=~"5..",job="thanos-receive-default"}[6h])) / scalar(sum(rate(http_requests_total{job="thanos-receive-default"}[6h]))) > 0`, + expected: `sum by (code) (rate(http_requests_total{code=~"5..",job="thanos-receive-default"}[6h])) / scalar(sum(rate(http_requests_total{job="thanos-receive-default"}[6h]))) > 0`, }, { name: "http-ratio-grouping", objective: objectiveHTTPRatioGrouping(), timerange: 6 * time.Hour, - expected: `sum by(code) (rate(http_requests_total{code=~"5..",job="thanos-receive-default"}[6h])) / scalar(sum(rate(http_requests_total{job="thanos-receive-default"}[6h]))) > 0`, + expected: `sum by (code) (rate(http_requests_total{code=~"5..",job="thanos-receive-default"}[6h])) / scalar(sum(rate(http_requests_total{job="thanos-receive-default"}[6h]))) > 0`, }, { name: "http-ratio-grouping-regex", objective: objectiveHTTPRatioGroupingRegex(), timerange: 6 * time.Hour, - expected: `sum by(code) (rate(http_requests_total{code=~"5..",handler=~"/api.*",job="thanos-receive-default"}[6h])) / scalar(sum(rate(http_requests_total{handler=~"/api.*",job="thanos-receive-default"}[6h]))) > 0`, + expected: `sum by (code) (rate(http_requests_total{code=~"5..",handler=~"/api.*",job="thanos-receive-default"}[6h])) / scalar(sum(rate(http_requests_total{handler=~"/api.*",job="thanos-receive-default"}[6h]))) > 0`, }, { name: "grpc-ratio", objective: objectiveGRPCRatio(), timerange: 6 * time.Hour, - expected: `sum by(grpc_code) (rate(grpc_server_handled_total{grpc_code=~"Aborted|Unavailable|Internal|Unknown|Unimplemented|DataLoss",grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[6h])) / scalar(sum(rate(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[6h]))) > 0`, + expected: `sum by (grpc_code) (rate(grpc_server_handled_total{grpc_code=~"Aborted|Unavailable|Internal|Unknown|Unimplemented|DataLoss",grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[6h])) / scalar(sum(rate(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[6h]))) > 0`, }, { name: "grpc-ratio-grouping", objective: objectiveGRPCRatioGrouping(), timerange: 6 * time.Hour, - expected: `sum by(grpc_code) (rate(grpc_server_handled_total{grpc_code=~"Aborted|Unavailable|Internal|Unknown|Unimplemented|DataLoss",grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[6h])) / scalar(sum(rate(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[6h]))) > 0`, + expected: `sum by (grpc_code) (rate(grpc_server_handled_total{grpc_code=~"Aborted|Unavailable|Internal|Unknown|Unimplemented|DataLoss",grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[6h])) / scalar(sum(rate(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[6h]))) > 0`, }, { name: "http-latency", objective: objectiveHTTPLatency(), @@ -725,7 +725,7 @@ func TestObjective_ErrorsRange(t *testing.T) { name: "apiserver-write-response-errors", objective: objectiveAPIServerRatio(), timerange: 2 * time.Hour, - expected: `sum by(code) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h])) / scalar(sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h]))) > 0`, + expected: `sum by (code) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h])) / scalar(sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h]))) > 0`, }, { name: "apiserver-read-resource-latency", objective: objectiveAPIServerLatency(), @@ -775,27 +775,27 @@ func TestObjective_DurationRange(t *testing.T) { name: "http-latency", objective: objectiveHTTPLatency(), timerange: time.Hour, - expected: `histogram_quantile(0.95, sum by(le) (rate(http_request_duration_seconds_bucket{code=~"2..",job="metrics-service-thanos-receive-default"}[1h])))`, + expected: `histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket{code=~"2..",job="metrics-service-thanos-receive-default"}[1h])))`, }, { name: "http-latency-grouping", objective: objectiveHTTPLatencyGrouping(), timerange: time.Hour, - expected: `histogram_quantile(0.95, sum by(le) (rate(http_request_duration_seconds_bucket{code=~"2..",job="metrics-service-thanos-receive-default"}[1h])))`, + expected: `histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket{code=~"2..",job="metrics-service-thanos-receive-default"}[1h])))`, }, { name: "http-latency-grouping-regex", objective: objectiveHTTPLatencyGroupingRegex(), timerange: time.Hour, - expected: `histogram_quantile(0.95, sum by(le) (rate(http_request_duration_seconds_bucket{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[1h])))`, + expected: `histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[1h])))`, }, { name: "grpc-latency", objective: objectiveGRPCLatency(), timerange: time.Hour, - expected: `histogram_quantile(0.95, sum by(le) (rate(grpc_server_handling_seconds_bucket{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1h])))`, + expected: `histogram_quantile(0.95, sum by (le) (rate(grpc_server_handling_seconds_bucket{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1h])))`, }, { name: "grpc-latency-grouping", objective: objectiveGRPCLatencyGrouping(), timerange: time.Hour, - expected: `histogram_quantile(0.95, sum by(le) (rate(grpc_server_handling_seconds_bucket{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1h])))`, + expected: `histogram_quantile(0.95, sum by (le) (rate(grpc_server_handling_seconds_bucket{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1h])))`, }, { name: "operator-ratio", objective: objectiveOperator(), @@ -815,7 +815,7 @@ func TestObjective_DurationRange(t *testing.T) { name: "apiserver-read-resource-latency", objective: objectiveAPIServerLatency(), timerange: 2 * time.Hour, - expected: `histogram_quantile(0.95, sum by(le) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[2h])))`, + expected: `histogram_quantile(0.95, sum by (le) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[2h])))`, }} for _, tc := range testcases { diff --git a/slo/rules.go b/slo/rules.go index 6c973208c..43c8fa255 100644 --- a/slo/rules.go +++ b/slo/rules.go @@ -253,7 +253,7 @@ func (o Objective) BurnrateName(rate time.Duration) string { func (o Objective) Burnrate(timerange time.Duration) string { if o.Indicator.Ratio != nil && o.Indicator.Ratio.Total.Name != "" { - expr, err := parser.ParseExpr(`sum by(grouping) (rate(errorMetric{matchers="errors"}[1s])) / sum by(grouping) (rate(metric{matchers="total"}[1s]))`) + expr, err := parser.ParseExpr(`sum by (grouping) (rate(errorMetric{matchers="errors"}[1s])) / sum by (grouping) (rate(metric{matchers="total"}[1s]))`) if err != nil { return err.Error() } @@ -288,12 +288,12 @@ func (o Objective) Burnrate(timerange time.Duration) string { if o.Indicator.Latency != nil && o.Indicator.Latency.Total.Name != "" { query := ` ( - sum by(grouping) (rate(metric{matchers="total"}[1s])) + sum by (grouping) (rate(metric{matchers="total"}[1s])) - - sum by(grouping) (rate(errorMetric{matchers="errors"}[1s])) + sum by (grouping) (rate(errorMetric{matchers="errors"}[1s])) ) / - sum by(grouping) (rate(metric{matchers="total"}[1s])) + sum by (grouping) (rate(metric{matchers="total"}[1s])) ` expr, err := parser.ParseExpr(query) if err != nil { diff --git a/slo/rules_test.go b/slo/rules_test.go index 36ee57aaa..f7c89efad 100644 --- a/slo/rules_test.go +++ b/slo/rules_test.go @@ -78,31 +78,31 @@ func TestObjective_Burnrates(t *testing.T) { Interval: "30s", Rules: []monitoringv1.Rule{{ Record: "http_requests:burnrate5m", - Expr: intstr.FromString(`sum by(handler, job) (rate(http_requests_total{code=~"5..",job="thanos-receive-default"}[5m])) / sum by(handler, job) (rate(http_requests_total{job="thanos-receive-default"}[5m]))`), + Expr: intstr.FromString(`sum by (handler, job) (rate(http_requests_total{code=~"5..",job="thanos-receive-default"}[5m])) / sum by (handler, job) (rate(http_requests_total{job="thanos-receive-default"}[5m]))`), Labels: map[string]string{"slo": "monitoring-http-errors"}, }, { Record: "http_requests:burnrate30m", - Expr: intstr.FromString(`sum by(handler, job) (rate(http_requests_total{code=~"5..",job="thanos-receive-default"}[30m])) / sum by(handler, job) (rate(http_requests_total{job="thanos-receive-default"}[30m]))`), + Expr: intstr.FromString(`sum by (handler, job) (rate(http_requests_total{code=~"5..",job="thanos-receive-default"}[30m])) / sum by (handler, job) (rate(http_requests_total{job="thanos-receive-default"}[30m]))`), Labels: map[string]string{"slo": "monitoring-http-errors"}, }, { Record: "http_requests:burnrate1h", - Expr: intstr.FromString(`sum by(handler, job) (rate(http_requests_total{code=~"5..",job="thanos-receive-default"}[1h])) / sum by(handler, job) (rate(http_requests_total{job="thanos-receive-default"}[1h]))`), + Expr: intstr.FromString(`sum by (handler, job) (rate(http_requests_total{code=~"5..",job="thanos-receive-default"}[1h])) / sum by (handler, job) (rate(http_requests_total{job="thanos-receive-default"}[1h]))`), Labels: map[string]string{"slo": "monitoring-http-errors"}, }, { Record: "http_requests:burnrate2h", - Expr: intstr.FromString(`sum by(handler, job) (rate(http_requests_total{code=~"5..",job="thanos-receive-default"}[2h])) / sum by(handler, job) (rate(http_requests_total{job="thanos-receive-default"}[2h]))`), + Expr: intstr.FromString(`sum by (handler, job) (rate(http_requests_total{code=~"5..",job="thanos-receive-default"}[2h])) / sum by (handler, job) (rate(http_requests_total{job="thanos-receive-default"}[2h]))`), Labels: map[string]string{"slo": "monitoring-http-errors"}, }, { Record: "http_requests:burnrate6h", - Expr: intstr.FromString(`sum by(handler, job) (rate(http_requests_total{code=~"5..",job="thanos-receive-default"}[6h])) / sum by(handler, job) (rate(http_requests_total{job="thanos-receive-default"}[6h]))`), + Expr: intstr.FromString(`sum by (handler, job) (rate(http_requests_total{code=~"5..",job="thanos-receive-default"}[6h])) / sum by (handler, job) (rate(http_requests_total{job="thanos-receive-default"}[6h]))`), Labels: map[string]string{"slo": "monitoring-http-errors"}, }, { Record: "http_requests:burnrate1d", - Expr: intstr.FromString(`sum by(handler, job) (rate(http_requests_total{code=~"5..",job="thanos-receive-default"}[1d])) / sum by(handler, job) (rate(http_requests_total{job="thanos-receive-default"}[1d]))`), + Expr: intstr.FromString(`sum by (handler, job) (rate(http_requests_total{code=~"5..",job="thanos-receive-default"}[1d])) / sum by (handler, job) (rate(http_requests_total{job="thanos-receive-default"}[1d]))`), Labels: map[string]string{"slo": "monitoring-http-errors"}, }, { Record: "http_requests:burnrate4d", - Expr: intstr.FromString(`sum by(handler, job) (rate(http_requests_total{code=~"5..",job="thanos-receive-default"}[4d])) / sum by(handler, job) (rate(http_requests_total{job="thanos-receive-default"}[4d]))`), + Expr: intstr.FromString(`sum by (handler, job) (rate(http_requests_total{code=~"5..",job="thanos-receive-default"}[4d])) / sum by (handler, job) (rate(http_requests_total{job="thanos-receive-default"}[4d]))`), Labels: map[string]string{"slo": "monitoring-http-errors"}, }, { Alert: "ErrorBudgetBurn", @@ -134,31 +134,31 @@ func TestObjective_Burnrates(t *testing.T) { Interval: "30s", Rules: []monitoringv1.Rule{{ Record: "http_requests:burnrate5m", - Expr: intstr.FromString(`sum by(handler, job) (rate(http_requests_total{code=~"5..",handler=~"/api.*",job="thanos-receive-default"}[5m])) / sum by(handler, job) (rate(http_requests_total{handler=~"/api.*",job="thanos-receive-default"}[5m]))`), + Expr: intstr.FromString(`sum by (handler, job) (rate(http_requests_total{code=~"5..",handler=~"/api.*",job="thanos-receive-default"}[5m])) / sum by (handler, job) (rate(http_requests_total{handler=~"/api.*",job="thanos-receive-default"}[5m]))`), Labels: map[string]string{"slo": "monitoring-http-errors"}, }, { Record: "http_requests:burnrate30m", - Expr: intstr.FromString(`sum by(handler, job) (rate(http_requests_total{code=~"5..",handler=~"/api.*",job="thanos-receive-default"}[30m])) / sum by(handler, job) (rate(http_requests_total{handler=~"/api.*",job="thanos-receive-default"}[30m]))`), + Expr: intstr.FromString(`sum by (handler, job) (rate(http_requests_total{code=~"5..",handler=~"/api.*",job="thanos-receive-default"}[30m])) / sum by (handler, job) (rate(http_requests_total{handler=~"/api.*",job="thanos-receive-default"}[30m]))`), Labels: map[string]string{"slo": "monitoring-http-errors"}, }, { Record: "http_requests:burnrate1h", - Expr: intstr.FromString(`sum by(handler, job) (rate(http_requests_total{code=~"5..",handler=~"/api.*",job="thanos-receive-default"}[1h])) / sum by(handler, job) (rate(http_requests_total{handler=~"/api.*",job="thanos-receive-default"}[1h]))`), + Expr: intstr.FromString(`sum by (handler, job) (rate(http_requests_total{code=~"5..",handler=~"/api.*",job="thanos-receive-default"}[1h])) / sum by (handler, job) (rate(http_requests_total{handler=~"/api.*",job="thanos-receive-default"}[1h]))`), Labels: map[string]string{"slo": "monitoring-http-errors"}, }, { Record: "http_requests:burnrate2h", - Expr: intstr.FromString(`sum by(handler, job) (rate(http_requests_total{code=~"5..",handler=~"/api.*",job="thanos-receive-default"}[2h])) / sum by(handler, job) (rate(http_requests_total{handler=~"/api.*",job="thanos-receive-default"}[2h]))`), + Expr: intstr.FromString(`sum by (handler, job) (rate(http_requests_total{code=~"5..",handler=~"/api.*",job="thanos-receive-default"}[2h])) / sum by (handler, job) (rate(http_requests_total{handler=~"/api.*",job="thanos-receive-default"}[2h]))`), Labels: map[string]string{"slo": "monitoring-http-errors"}, }, { Record: "http_requests:burnrate6h", - Expr: intstr.FromString(`sum by(handler, job) (rate(http_requests_total{code=~"5..",handler=~"/api.*",job="thanos-receive-default"}[6h])) / sum by(handler, job) (rate(http_requests_total{handler=~"/api.*",job="thanos-receive-default"}[6h]))`), + Expr: intstr.FromString(`sum by (handler, job) (rate(http_requests_total{code=~"5..",handler=~"/api.*",job="thanos-receive-default"}[6h])) / sum by (handler, job) (rate(http_requests_total{handler=~"/api.*",job="thanos-receive-default"}[6h]))`), Labels: map[string]string{"slo": "monitoring-http-errors"}, }, { Record: "http_requests:burnrate1d", - Expr: intstr.FromString(`sum by(handler, job) (rate(http_requests_total{code=~"5..",handler=~"/api.*",job="thanos-receive-default"}[1d])) / sum by(handler, job) (rate(http_requests_total{handler=~"/api.*",job="thanos-receive-default"}[1d]))`), + Expr: intstr.FromString(`sum by (handler, job) (rate(http_requests_total{code=~"5..",handler=~"/api.*",job="thanos-receive-default"}[1d])) / sum by (handler, job) (rate(http_requests_total{handler=~"/api.*",job="thanos-receive-default"}[1d]))`), Labels: map[string]string{"slo": "monitoring-http-errors"}, }, { Record: "http_requests:burnrate4d", - Expr: intstr.FromString(`sum by(handler, job) (rate(http_requests_total{code=~"5..",handler=~"/api.*",job="thanos-receive-default"}[4d])) / sum by(handler, job) (rate(http_requests_total{handler=~"/api.*",job="thanos-receive-default"}[4d]))`), + Expr: intstr.FromString(`sum by (handler, job) (rate(http_requests_total{code=~"5..",handler=~"/api.*",job="thanos-receive-default"}[4d])) / sum by (handler, job) (rate(http_requests_total{handler=~"/api.*",job="thanos-receive-default"}[4d]))`), Labels: map[string]string{"slo": "monitoring-http-errors"}, }, { Alert: "ErrorBudgetBurn", @@ -246,31 +246,31 @@ func TestObjective_Burnrates(t *testing.T) { Interval: "30s", Rules: []monitoringv1.Rule{{ Record: "grpc_server_handled:burnrate5m", - Expr: intstr.FromString(`sum by(handler, job) (rate(grpc_server_handled_total{grpc_code=~"Aborted|Unavailable|Internal|Unknown|Unimplemented|DataLoss",grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[5m])) / sum by(handler, job) (rate(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[5m]))`), + Expr: intstr.FromString(`sum by (handler, job) (rate(grpc_server_handled_total{grpc_code=~"Aborted|Unavailable|Internal|Unknown|Unimplemented|DataLoss",grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[5m])) / sum by (handler, job) (rate(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[5m]))`), Labels: map[string]string{"grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "slo": "monitoring-grpc-errors"}, }, { Record: "grpc_server_handled:burnrate30m", - Expr: intstr.FromString(`sum by(handler, job) (rate(grpc_server_handled_total{grpc_code=~"Aborted|Unavailable|Internal|Unknown|Unimplemented|DataLoss",grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[30m])) / sum by(handler, job) (rate(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[30m]))`), + Expr: intstr.FromString(`sum by (handler, job) (rate(grpc_server_handled_total{grpc_code=~"Aborted|Unavailable|Internal|Unknown|Unimplemented|DataLoss",grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[30m])) / sum by (handler, job) (rate(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[30m]))`), Labels: map[string]string{"grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "slo": "monitoring-grpc-errors"}, }, { Record: "grpc_server_handled:burnrate1h", - Expr: intstr.FromString(`sum by(handler, job) (rate(grpc_server_handled_total{grpc_code=~"Aborted|Unavailable|Internal|Unknown|Unimplemented|DataLoss",grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1h])) / sum by(handler, job) (rate(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1h]))`), + Expr: intstr.FromString(`sum by (handler, job) (rate(grpc_server_handled_total{grpc_code=~"Aborted|Unavailable|Internal|Unknown|Unimplemented|DataLoss",grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1h])) / sum by (handler, job) (rate(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1h]))`), Labels: map[string]string{"grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "slo": "monitoring-grpc-errors"}, }, { Record: "grpc_server_handled:burnrate2h", - Expr: intstr.FromString(`sum by(handler, job) (rate(grpc_server_handled_total{grpc_code=~"Aborted|Unavailable|Internal|Unknown|Unimplemented|DataLoss",grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[2h])) / sum by(handler, job) (rate(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[2h]))`), + Expr: intstr.FromString(`sum by (handler, job) (rate(grpc_server_handled_total{grpc_code=~"Aborted|Unavailable|Internal|Unknown|Unimplemented|DataLoss",grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[2h])) / sum by (handler, job) (rate(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[2h]))`), Labels: map[string]string{"grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "slo": "monitoring-grpc-errors"}, }, { Record: "grpc_server_handled:burnrate6h", - Expr: intstr.FromString(`sum by(handler, job) (rate(grpc_server_handled_total{grpc_code=~"Aborted|Unavailable|Internal|Unknown|Unimplemented|DataLoss",grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[6h])) / sum by(handler, job) (rate(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[6h]))`), + Expr: intstr.FromString(`sum by (handler, job) (rate(grpc_server_handled_total{grpc_code=~"Aborted|Unavailable|Internal|Unknown|Unimplemented|DataLoss",grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[6h])) / sum by (handler, job) (rate(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[6h]))`), Labels: map[string]string{"grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "slo": "monitoring-grpc-errors"}, }, { Record: "grpc_server_handled:burnrate1d", - Expr: intstr.FromString(`sum by(handler, job) (rate(grpc_server_handled_total{grpc_code=~"Aborted|Unavailable|Internal|Unknown|Unimplemented|DataLoss",grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1d])) / sum by(handler, job) (rate(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1d]))`), + Expr: intstr.FromString(`sum by (handler, job) (rate(grpc_server_handled_total{grpc_code=~"Aborted|Unavailable|Internal|Unknown|Unimplemented|DataLoss",grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1d])) / sum by (handler, job) (rate(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1d]))`), Labels: map[string]string{"grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "slo": "monitoring-grpc-errors"}, }, { Record: "grpc_server_handled:burnrate4d", - Expr: intstr.FromString(`sum by(handler, job) (rate(grpc_server_handled_total{grpc_code=~"Aborted|Unavailable|Internal|Unknown|Unimplemented|DataLoss",grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[4d])) / sum by(handler, job) (rate(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[4d]))`), + Expr: intstr.FromString(`sum by (handler, job) (rate(grpc_server_handled_total{grpc_code=~"Aborted|Unavailable|Internal|Unknown|Unimplemented|DataLoss",grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[4d])) / sum by (handler, job) (rate(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[4d]))`), Labels: map[string]string{"grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "slo": "monitoring-grpc-errors"}, }, { Alert: "ErrorBudgetBurn", @@ -358,31 +358,31 @@ func TestObjective_Burnrates(t *testing.T) { Interval: "30s", Rules: []monitoringv1.Rule{{ Record: "http_request_duration_seconds:burnrate5m", - Expr: intstr.FromString(`(sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[5m])) - sum by(handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",job="metrics-service-thanos-receive-default",le="1"}[5m]))) / sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[5m]))`), + Expr: intstr.FromString(`(sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[5m])) - sum by (handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",job="metrics-service-thanos-receive-default",le="1"}[5m]))) / sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[5m]))`), Labels: map[string]string{"slo": "monitoring-http-latency"}, }, { Record: "http_request_duration_seconds:burnrate30m", - Expr: intstr.FromString(`(sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[30m])) - sum by(handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",job="metrics-service-thanos-receive-default",le="1"}[30m]))) / sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[30m]))`), + Expr: intstr.FromString(`(sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[30m])) - sum by (handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",job="metrics-service-thanos-receive-default",le="1"}[30m]))) / sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[30m]))`), Labels: map[string]string{"slo": "monitoring-http-latency"}, }, { Record: "http_request_duration_seconds:burnrate1h", - Expr: intstr.FromString(`(sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[1h])) - sum by(handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",job="metrics-service-thanos-receive-default",le="1"}[1h]))) / sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[1h]))`), + Expr: intstr.FromString(`(sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[1h])) - sum by (handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",job="metrics-service-thanos-receive-default",le="1"}[1h]))) / sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[1h]))`), Labels: map[string]string{"slo": "monitoring-http-latency"}, }, { Record: "http_request_duration_seconds:burnrate2h", - Expr: intstr.FromString(`(sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[2h])) - sum by(handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",job="metrics-service-thanos-receive-default",le="1"}[2h]))) / sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[2h]))`), + Expr: intstr.FromString(`(sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[2h])) - sum by (handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",job="metrics-service-thanos-receive-default",le="1"}[2h]))) / sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[2h]))`), Labels: map[string]string{"slo": "monitoring-http-latency"}, }, { Record: "http_request_duration_seconds:burnrate6h", - Expr: intstr.FromString(`(sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[6h])) - sum by(handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",job="metrics-service-thanos-receive-default",le="1"}[6h]))) / sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[6h]))`), + Expr: intstr.FromString(`(sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[6h])) - sum by (handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",job="metrics-service-thanos-receive-default",le="1"}[6h]))) / sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[6h]))`), Labels: map[string]string{"slo": "monitoring-http-latency"}, }, { Record: "http_request_duration_seconds:burnrate1d", - Expr: intstr.FromString(`(sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[1d])) - sum by(handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",job="metrics-service-thanos-receive-default",le="1"}[1d]))) / sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[1d]))`), + Expr: intstr.FromString(`(sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[1d])) - sum by (handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",job="metrics-service-thanos-receive-default",le="1"}[1d]))) / sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[1d]))`), Labels: map[string]string{"slo": "monitoring-http-latency"}, }, { Record: "http_request_duration_seconds:burnrate4d", - Expr: intstr.FromString(`(sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[4d])) - sum by(handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",job="metrics-service-thanos-receive-default",le="1"}[4d]))) / sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[4d]))`), + Expr: intstr.FromString(`(sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[4d])) - sum by (handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",job="metrics-service-thanos-receive-default",le="1"}[4d]))) / sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[4d]))`), Labels: map[string]string{"slo": "monitoring-http-latency"}, }, { Alert: "ErrorBudgetBurn", @@ -414,31 +414,31 @@ func TestObjective_Burnrates(t *testing.T) { Interval: "30s", Rules: []monitoringv1.Rule{{ Record: "http_request_duration_seconds:burnrate5m", - Expr: intstr.FromString(`(sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[5m])) - sum by(handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default",le="1"}[5m]))) / sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[5m]))`), + Expr: intstr.FromString(`(sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[5m])) - sum by (handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default",le="1"}[5m]))) / sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[5m]))`), Labels: map[string]string{"slo": "monitoring-http-latency"}, }, { Record: "http_request_duration_seconds:burnrate30m", - Expr: intstr.FromString(`(sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[30m])) - sum by(handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default",le="1"}[30m]))) / sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[30m]))`), + Expr: intstr.FromString(`(sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[30m])) - sum by (handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default",le="1"}[30m]))) / sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[30m]))`), Labels: map[string]string{"slo": "monitoring-http-latency"}, }, { Record: "http_request_duration_seconds:burnrate1h", - Expr: intstr.FromString(`(sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[1h])) - sum by(handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default",le="1"}[1h]))) / sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[1h]))`), + Expr: intstr.FromString(`(sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[1h])) - sum by (handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default",le="1"}[1h]))) / sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[1h]))`), Labels: map[string]string{"slo": "monitoring-http-latency"}, }, { Record: "http_request_duration_seconds:burnrate2h", - Expr: intstr.FromString(`(sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[2h])) - sum by(handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default",le="1"}[2h]))) / sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[2h]))`), + Expr: intstr.FromString(`(sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[2h])) - sum by (handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default",le="1"}[2h]))) / sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[2h]))`), Labels: map[string]string{"slo": "monitoring-http-latency"}, }, { Record: "http_request_duration_seconds:burnrate6h", - Expr: intstr.FromString(`(sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[6h])) - sum by(handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default",le="1"}[6h]))) / sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[6h]))`), + Expr: intstr.FromString(`(sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[6h])) - sum by (handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default",le="1"}[6h]))) / sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[6h]))`), Labels: map[string]string{"slo": "monitoring-http-latency"}, }, { Record: "http_request_duration_seconds:burnrate1d", - Expr: intstr.FromString(`(sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[1d])) - sum by(handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default",le="1"}[1d]))) / sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[1d]))`), + Expr: intstr.FromString(`(sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[1d])) - sum by (handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default",le="1"}[1d]))) / sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[1d]))`), Labels: map[string]string{"slo": "monitoring-http-latency"}, }, { Record: "http_request_duration_seconds:burnrate4d", - Expr: intstr.FromString(`(sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[4d])) - sum by(handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default",le="1"}[4d]))) / sum by(handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[4d]))`), + Expr: intstr.FromString(`(sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[4d])) - sum by (handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default",le="1"}[4d]))) / sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[4d]))`), Labels: map[string]string{"slo": "monitoring-http-latency"}, }, { Alert: "ErrorBudgetBurn", @@ -526,31 +526,31 @@ func TestObjective_Burnrates(t *testing.T) { Interval: "30s", Rules: []monitoringv1.Rule{{ Record: "grpc_server_handling_seconds:burnrate1m", - Expr: intstr.FromString(`(sum by(handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1m])) - sum by(handler, job) (rate(grpc_server_handling_seconds_bucket{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",le="0.6"}[1m]))) / sum by(handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1m]))`), + Expr: intstr.FromString(`(sum by (handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1m])) - sum by (handler, job) (rate(grpc_server_handling_seconds_bucket{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",le="0.6"}[1m]))) / sum by (handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1m]))`), Labels: map[string]string{"slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore"}, }, { Record: "grpc_server_handling_seconds:burnrate8m", - Expr: intstr.FromString(`(sum by(handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[8m])) - sum by(handler, job) (rate(grpc_server_handling_seconds_bucket{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",le="0.6"}[8m]))) / sum by(handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[8m]))`), + Expr: intstr.FromString(`(sum by (handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[8m])) - sum by (handler, job) (rate(grpc_server_handling_seconds_bucket{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",le="0.6"}[8m]))) / sum by (handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[8m]))`), Labels: map[string]string{"slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore"}, }, { Record: "grpc_server_handling_seconds:burnrate15m", - Expr: intstr.FromString(`(sum by(handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[15m])) - sum by(handler, job) (rate(grpc_server_handling_seconds_bucket{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",le="0.6"}[15m]))) / sum by(handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[15m]))`), + Expr: intstr.FromString(`(sum by (handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[15m])) - sum by (handler, job) (rate(grpc_server_handling_seconds_bucket{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",le="0.6"}[15m]))) / sum by (handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[15m]))`), Labels: map[string]string{"slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore"}, }, { Record: "grpc_server_handling_seconds:burnrate30m", - Expr: intstr.FromString(`(sum by(handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[30m])) - sum by(handler, job) (rate(grpc_server_handling_seconds_bucket{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",le="0.6"}[30m]))) / sum by(handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[30m]))`), + Expr: intstr.FromString(`(sum by (handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[30m])) - sum by (handler, job) (rate(grpc_server_handling_seconds_bucket{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",le="0.6"}[30m]))) / sum by (handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[30m]))`), Labels: map[string]string{"slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore"}, }, { Record: "grpc_server_handling_seconds:burnrate1h30m", - Expr: intstr.FromString(`(sum by(handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1h30m])) - sum by(handler, job) (rate(grpc_server_handling_seconds_bucket{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",le="0.6"}[1h30m]))) / sum by(handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1h30m]))`), + Expr: intstr.FromString(`(sum by (handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1h30m])) - sum by (handler, job) (rate(grpc_server_handling_seconds_bucket{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",le="0.6"}[1h30m]))) / sum by (handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1h30m]))`), Labels: map[string]string{"slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore"}, }, { Record: "grpc_server_handling_seconds:burnrate6h", - Expr: intstr.FromString(`(sum by(handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[6h])) - sum by(handler, job) (rate(grpc_server_handling_seconds_bucket{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",le="0.6"}[6h]))) / sum by(handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[6h]))`), + Expr: intstr.FromString(`(sum by (handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[6h])) - sum by (handler, job) (rate(grpc_server_handling_seconds_bucket{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",le="0.6"}[6h]))) / sum by (handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[6h]))`), Labels: map[string]string{"slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore"}, }, { Record: "grpc_server_handling_seconds:burnrate1d", - Expr: intstr.FromString(`(sum by(handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1d])) - sum by(handler, job) (rate(grpc_server_handling_seconds_bucket{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",le="0.6"}[1d]))) / sum by(handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1d]))`), + Expr: intstr.FromString(`(sum by (handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1d])) - sum by (handler, job) (rate(grpc_server_handling_seconds_bucket{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",le="0.6"}[1d]))) / sum by (handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1d]))`), Labels: map[string]string{"slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore"}, }, { Alert: "ErrorBudgetBurn", @@ -638,31 +638,31 @@ func TestObjective_Burnrates(t *testing.T) { Interval: "30s", Rules: []monitoringv1.Rule{{ Record: "prometheus_operator_reconcile_operations:burnrate3m", - Expr: intstr.FromString(`sum by(namespace) (rate(prometheus_operator_reconcile_errors_total[3m])) / sum by(namespace) (rate(prometheus_operator_reconcile_operations_total[3m]))`), + Expr: intstr.FromString(`sum by (namespace) (rate(prometheus_operator_reconcile_errors_total[3m])) / sum by (namespace) (rate(prometheus_operator_reconcile_operations_total[3m]))`), Labels: map[string]string{"slo": "monitoring-prometheus-operator-errors"}, }, { Record: "prometheus_operator_reconcile_operations:burnrate15m", - Expr: intstr.FromString(`sum by(namespace) (rate(prometheus_operator_reconcile_errors_total[15m])) / sum by(namespace) (rate(prometheus_operator_reconcile_operations_total[15m]))`), + Expr: intstr.FromString(`sum by (namespace) (rate(prometheus_operator_reconcile_errors_total[15m])) / sum by (namespace) (rate(prometheus_operator_reconcile_operations_total[15m]))`), Labels: map[string]string{"slo": "monitoring-prometheus-operator-errors"}, }, { Record: "prometheus_operator_reconcile_operations:burnrate30m", - Expr: intstr.FromString(`sum by(namespace) (rate(prometheus_operator_reconcile_errors_total[30m])) / sum by(namespace) (rate(prometheus_operator_reconcile_operations_total[30m]))`), + Expr: intstr.FromString(`sum by (namespace) (rate(prometheus_operator_reconcile_errors_total[30m])) / sum by (namespace) (rate(prometheus_operator_reconcile_operations_total[30m]))`), Labels: map[string]string{"slo": "monitoring-prometheus-operator-errors"}, }, { Record: "prometheus_operator_reconcile_operations:burnrate1h", - Expr: intstr.FromString(`sum by(namespace) (rate(prometheus_operator_reconcile_errors_total[1h])) / sum by(namespace) (rate(prometheus_operator_reconcile_operations_total[1h]))`), + Expr: intstr.FromString(`sum by (namespace) (rate(prometheus_operator_reconcile_errors_total[1h])) / sum by (namespace) (rate(prometheus_operator_reconcile_operations_total[1h]))`), Labels: map[string]string{"slo": "monitoring-prometheus-operator-errors"}, }, { Record: "prometheus_operator_reconcile_operations:burnrate3h", - Expr: intstr.FromString(`sum by(namespace) (rate(prometheus_operator_reconcile_errors_total[3h])) / sum by(namespace) (rate(prometheus_operator_reconcile_operations_total[3h]))`), + Expr: intstr.FromString(`sum by (namespace) (rate(prometheus_operator_reconcile_errors_total[3h])) / sum by (namespace) (rate(prometheus_operator_reconcile_operations_total[3h]))`), Labels: map[string]string{"slo": "monitoring-prometheus-operator-errors"}, }, { Record: "prometheus_operator_reconcile_operations:burnrate12h", - Expr: intstr.FromString(`sum by(namespace) (rate(prometheus_operator_reconcile_errors_total[12h])) / sum by(namespace) (rate(prometheus_operator_reconcile_operations_total[12h]))`), + Expr: intstr.FromString(`sum by (namespace) (rate(prometheus_operator_reconcile_errors_total[12h])) / sum by (namespace) (rate(prometheus_operator_reconcile_operations_total[12h]))`), Labels: map[string]string{"slo": "monitoring-prometheus-operator-errors"}, }, { Record: "prometheus_operator_reconcile_operations:burnrate2d", - Expr: intstr.FromString(`sum by(namespace) (rate(prometheus_operator_reconcile_errors_total[2d])) / sum by(namespace) (rate(prometheus_operator_reconcile_operations_total[2d]))`), + Expr: intstr.FromString(`sum by (namespace) (rate(prometheus_operator_reconcile_errors_total[2d])) / sum by (namespace) (rate(prometheus_operator_reconcile_operations_total[2d]))`), Labels: map[string]string{"slo": "monitoring-prometheus-operator-errors"}, }, { Alert: "ErrorBudgetBurn", @@ -694,31 +694,31 @@ func TestObjective_Burnrates(t *testing.T) { Interval: "30s", Rules: []monitoringv1.Rule{{ Record: "apiserver_request:burnrate3m", - Expr: intstr.FromString(`sum by(verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3m])) / sum by(verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3m]))`), + Expr: intstr.FromString(`sum by (verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3m])) / sum by (verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3m]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-write-response-errors"}, }, { Record: "apiserver_request:burnrate15m", - Expr: intstr.FromString(`sum by(verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[15m])) / sum by(verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[15m]))`), + Expr: intstr.FromString(`sum by (verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[15m])) / sum by (verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[15m]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-write-response-errors"}, }, { Record: "apiserver_request:burnrate30m", - Expr: intstr.FromString(`sum by(verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m])) / sum by(verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m]))`), + Expr: intstr.FromString(`sum by (verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m])) / sum by (verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-write-response-errors"}, }, { Record: "apiserver_request:burnrate1h", - Expr: intstr.FromString(`sum by(verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h])) / sum by(verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h]))`), + Expr: intstr.FromString(`sum by (verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h])) / sum by (verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-write-response-errors"}, }, { Record: "apiserver_request:burnrate3h", - Expr: intstr.FromString(`sum by(verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3h])) / sum by(verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3h]))`), + Expr: intstr.FromString(`sum by (verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3h])) / sum by (verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3h]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-write-response-errors"}, }, { Record: "apiserver_request:burnrate12h", - Expr: intstr.FromString(`sum by(verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[12h])) / sum by(verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[12h]))`), + Expr: intstr.FromString(`sum by (verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[12h])) / sum by (verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[12h]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-write-response-errors"}, }, { Record: "apiserver_request:burnrate2d", - Expr: intstr.FromString(`sum by(verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2d])) / sum by(verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2d]))`), + Expr: intstr.FromString(`sum by (verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2d])) / sum by (verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2d]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-write-response-errors"}, }, { Alert: "ErrorBudgetBurn", @@ -750,31 +750,31 @@ func TestObjective_Burnrates(t *testing.T) { Interval: "30s", Rules: []monitoringv1.Rule{{ Record: "apiserver_request:burnrate3m", - Expr: intstr.FromString(`sum by(verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3m])) / sum by(verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3m]))`), + Expr: intstr.FromString(`sum by (verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3m])) / sum by (verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3m]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-write-response-errors"}, }, { Record: "apiserver_request:burnrate15m", - Expr: intstr.FromString(`sum by(verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[15m])) / sum by(verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[15m]))`), + Expr: intstr.FromString(`sum by (verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[15m])) / sum by (verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[15m]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-write-response-errors"}, }, { Record: "apiserver_request:burnrate30m", - Expr: intstr.FromString(`sum by(verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m])) / sum by(verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m]))`), + Expr: intstr.FromString(`sum by (verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m])) / sum by (verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-write-response-errors"}, }, { Record: "apiserver_request:burnrate1h", - Expr: intstr.FromString(`sum by(verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h])) / sum by(verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h]))`), + Expr: intstr.FromString(`sum by (verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h])) / sum by (verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-write-response-errors"}, }, { Record: "apiserver_request:burnrate3h", - Expr: intstr.FromString(`sum by(verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3h])) / sum by(verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3h]))`), + Expr: intstr.FromString(`sum by (verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3h])) / sum by (verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3h]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-write-response-errors"}, }, { Record: "apiserver_request:burnrate12h", - Expr: intstr.FromString(`sum by(verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[12h])) / sum by(verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[12h]))`), + Expr: intstr.FromString(`sum by (verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[12h])) / sum by (verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[12h]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-write-response-errors"}, }, { Record: "apiserver_request:burnrate2d", - Expr: intstr.FromString(`sum by(verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2d])) / sum by(verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2d]))`), + Expr: intstr.FromString(`sum by (verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2d])) / sum by (verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2d]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-write-response-errors"}, }}, }, @@ -786,31 +786,31 @@ func TestObjective_Burnrates(t *testing.T) { Interval: "30s", Rules: []monitoringv1.Rule{{ Record: "apiserver_request_duration_seconds:burnrate3m", - Expr: intstr.FromString(`(sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[3m])) - sum by(resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[3m]))) / sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[3m]))`), + Expr: intstr.FromString(`(sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[3m])) - sum by (resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[3m]))) / sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[3m]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-read-resource-latency"}, }, { Record: "apiserver_request_duration_seconds:burnrate15m", - Expr: intstr.FromString(`(sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[15m])) - sum by(resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[15m]))) / sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[15m]))`), + Expr: intstr.FromString(`(sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[15m])) - sum by (resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[15m]))) / sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[15m]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-read-resource-latency"}, }, { Record: "apiserver_request_duration_seconds:burnrate30m", - Expr: intstr.FromString(`(sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[30m])) - sum by(resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[30m]))) / sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[30m]))`), + Expr: intstr.FromString(`(sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[30m])) - sum by (resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[30m]))) / sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[30m]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-read-resource-latency"}, }, { Record: "apiserver_request_duration_seconds:burnrate1h", - Expr: intstr.FromString(`(sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[1h])) - sum by(resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[1h]))) / sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[1h]))`), + Expr: intstr.FromString(`(sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[1h])) - sum by (resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[1h]))) / sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[1h]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-read-resource-latency"}, }, { Record: "apiserver_request_duration_seconds:burnrate3h", - Expr: intstr.FromString(`(sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[3h])) - sum by(resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[3h]))) / sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[3h]))`), + Expr: intstr.FromString(`(sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[3h])) - sum by (resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[3h]))) / sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[3h]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-read-resource-latency"}, }, { Record: "apiserver_request_duration_seconds:burnrate12h", - Expr: intstr.FromString(`(sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[12h])) - sum by(resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[12h]))) / sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[12h]))`), + Expr: intstr.FromString(`(sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[12h])) - sum by (resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[12h]))) / sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[12h]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-read-resource-latency"}, }, { Record: "apiserver_request_duration_seconds:burnrate2d", - Expr: intstr.FromString(`(sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[2d])) - sum by(resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[2d]))) / sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[2d]))`), + Expr: intstr.FromString(`(sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[2d])) - sum by (resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[2d]))) / sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[2d]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-read-resource-latency"}, }, { Alert: "ErrorBudgetBurn", @@ -842,31 +842,31 @@ func TestObjective_Burnrates(t *testing.T) { Interval: "30s", Rules: []monitoringv1.Rule{{ Record: "apiserver_request_duration_seconds:burnrate3m", - Expr: intstr.FromString(`(sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[3m])) - sum by(resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[3m]))) / sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[3m]))`), + Expr: intstr.FromString(`(sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[3m])) - sum by (resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[3m]))) / sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[3m]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-read-resource-latency"}, }, { Record: "apiserver_request_duration_seconds:burnrate15m", - Expr: intstr.FromString(`(sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[15m])) - sum by(resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[15m]))) / sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[15m]))`), + Expr: intstr.FromString(`(sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[15m])) - sum by (resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[15m]))) / sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[15m]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-read-resource-latency"}, }, { Record: "apiserver_request_duration_seconds:burnrate30m", - Expr: intstr.FromString(`(sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[30m])) - sum by(resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[30m]))) / sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[30m]))`), + Expr: intstr.FromString(`(sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[30m])) - sum by (resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[30m]))) / sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[30m]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-read-resource-latency"}, }, { Record: "apiserver_request_duration_seconds:burnrate1h", - Expr: intstr.FromString(`(sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[1h])) - sum by(resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[1h]))) / sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[1h]))`), + Expr: intstr.FromString(`(sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[1h])) - sum by (resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[1h]))) / sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[1h]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-read-resource-latency"}, }, { Record: "apiserver_request_duration_seconds:burnrate3h", - Expr: intstr.FromString(`(sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[3h])) - sum by(resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[3h]))) / sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[3h]))`), + Expr: intstr.FromString(`(sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[3h])) - sum by (resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[3h]))) / sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[3h]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-read-resource-latency"}, }, { Record: "apiserver_request_duration_seconds:burnrate12h", - Expr: intstr.FromString(`(sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[12h])) - sum by(resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[12h]))) / sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[12h]))`), + Expr: intstr.FromString(`(sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[12h])) - sum by (resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[12h]))) / sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[12h]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-read-resource-latency"}, }, { Record: "apiserver_request_duration_seconds:burnrate2d", - Expr: intstr.FromString(`(sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[2d])) - sum by(resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[2d]))) / sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[2d]))`), + Expr: intstr.FromString(`(sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[2d])) - sum by (resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[2d]))) / sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[2d]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-read-resource-latency"}, }}, }, @@ -878,31 +878,31 @@ func TestObjective_Burnrates(t *testing.T) { Interval: "30s", Rules: []monitoringv1.Rule{{ Record: "apiserver_request_duration_seconds:burnrate3m", - Expr: intstr.FromString(`(sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[3m])) - sum by(resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[3m]))) / sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[3m]))`), + Expr: intstr.FromString(`(sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[3m])) - sum by (resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[3m]))) / sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[3m]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-read-resource-latency"}, }, { Record: "apiserver_request_duration_seconds:burnrate15m", - Expr: intstr.FromString(`(sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[15m])) - sum by(resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[15m]))) / sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[15m]))`), + Expr: intstr.FromString(`(sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[15m])) - sum by (resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[15m]))) / sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[15m]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-read-resource-latency"}, }, { Record: "apiserver_request_duration_seconds:burnrate30m", - Expr: intstr.FromString(`(sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[30m])) - sum by(resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[30m]))) / sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[30m]))`), + Expr: intstr.FromString(`(sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[30m])) - sum by (resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[30m]))) / sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[30m]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-read-resource-latency"}, }, { Record: "apiserver_request_duration_seconds:burnrate1h", - Expr: intstr.FromString(`(sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[1h])) - sum by(resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[1h]))) / sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[1h]))`), + Expr: intstr.FromString(`(sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[1h])) - sum by (resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[1h]))) / sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[1h]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-read-resource-latency"}, }, { Record: "apiserver_request_duration_seconds:burnrate3h", - Expr: intstr.FromString(`(sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[3h])) - sum by(resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[3h]))) / sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[3h]))`), + Expr: intstr.FromString(`(sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[3h])) - sum by (resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[3h]))) / sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[3h]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-read-resource-latency"}, }, { Record: "apiserver_request_duration_seconds:burnrate12h", - Expr: intstr.FromString(`(sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[12h])) - sum by(resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[12h]))) / sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[12h]))`), + Expr: intstr.FromString(`(sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[12h])) - sum by (resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[12h]))) / sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[12h]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-read-resource-latency"}, }, { Record: "apiserver_request_duration_seconds:burnrate2d", - Expr: intstr.FromString(`(sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[2d])) - sum by(resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[2d]))) / sum by(resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[2d]))`), + Expr: intstr.FromString(`(sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[2d])) - sum by (resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[2d]))) / sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[2d]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-read-resource-latency"}, }, { Alert: "APIServerLatencyErrorBudgetBurn", @@ -1021,7 +1021,7 @@ func TestObjective_IncreaseRules(t *testing.T) { Interval: "2m30s", Rules: []monitoringv1.Rule{{ Record: "http_requests:increase4w", - Expr: intstr.FromString(`sum by(code) (increase(http_requests_total{job="thanos-receive-default"}[4w]))`), + Expr: intstr.FromString(`sum by (code) (increase(http_requests_total{job="thanos-receive-default"}[4w]))`), Labels: map[string]string{"job": "thanos-receive-default", "slo": "monitoring-http-errors"}, }, { Alert: "SLOMetricAbsent", @@ -1038,7 +1038,7 @@ func TestObjective_IncreaseRules(t *testing.T) { Interval: "2m30s", Rules: []monitoringv1.Rule{{ Record: "http_requests:increase4w", - Expr: intstr.FromString(`sum by(code, handler, job) (increase(http_requests_total{job="thanos-receive-default"}[4w]))`), + Expr: intstr.FromString(`sum by (code, handler, job) (increase(http_requests_total{job="thanos-receive-default"}[4w]))`), Labels: map[string]string{"slo": "monitoring-http-errors"}, }, { Alert: "SLOMetricAbsent", @@ -1055,7 +1055,7 @@ func TestObjective_IncreaseRules(t *testing.T) { Interval: "2m30s", Rules: []monitoringv1.Rule{{ Record: "http_requests:increase4w", - Expr: intstr.FromString(`sum by(code, handler, job) (increase(http_requests_total{handler=~"/api.*",job="thanos-receive-default"}[4w]))`), + Expr: intstr.FromString(`sum by (code, handler, job) (increase(http_requests_total{handler=~"/api.*",job="thanos-receive-default"}[4w]))`), Labels: map[string]string{"slo": "monitoring-http-errors"}, }, { Alert: "SLOMetricAbsent", @@ -1072,7 +1072,7 @@ func TestObjective_IncreaseRules(t *testing.T) { Interval: "2m30s", Rules: []monitoringv1.Rule{{ Record: "grpc_server_handled:increase4w", - Expr: intstr.FromString(`sum by(grpc_code) (increase(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[4w]))`), + Expr: intstr.FromString(`sum by (grpc_code) (increase(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[4w]))`), Labels: map[string]string{"grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "job": "api", "slo": "monitoring-grpc-errors"}, }, { Alert: "SLOMetricAbsent", @@ -1089,7 +1089,7 @@ func TestObjective_IncreaseRules(t *testing.T) { Interval: "2m30s", Rules: []monitoringv1.Rule{{ Record: "grpc_server_handled:increase4w", - Expr: intstr.FromString(`sum by(grpc_code, handler, job) (increase(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[4w]))`), + Expr: intstr.FromString(`sum by (grpc_code, handler, job) (increase(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[4w]))`), Labels: map[string]string{"grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "slo": "monitoring-grpc-errors"}, }, { Alert: "SLOMetricAbsent", @@ -1106,11 +1106,11 @@ func TestObjective_IncreaseRules(t *testing.T) { Interval: "2m30s", Rules: []monitoringv1.Rule{{ Record: "http_request_duration_seconds:increase4w", - Expr: intstr.FromString(`sum by(code) (increase(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[4w]))`), + Expr: intstr.FromString(`sum by (code) (increase(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[4w]))`), Labels: map[string]string{"job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency"}, }, { Record: "http_request_duration_seconds:increase4w", - Expr: intstr.FromString(`sum by(code) (increase(http_request_duration_seconds_bucket{code=~"2..",job="metrics-service-thanos-receive-default",le="1"}[4w]))`), + Expr: intstr.FromString(`sum by (code) (increase(http_request_duration_seconds_bucket{code=~"2..",job="metrics-service-thanos-receive-default",le="1"}[4w]))`), Labels: map[string]string{"job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "le": "1"}, }, { Alert: "SLOMetricAbsent", @@ -1132,11 +1132,11 @@ func TestObjective_IncreaseRules(t *testing.T) { Interval: "2m30s", Rules: []monitoringv1.Rule{{ Record: "http_request_duration_seconds:increase4w", - Expr: intstr.FromString(`sum by(code, handler, job) (increase(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[4w]))`), + Expr: intstr.FromString(`sum by (code, handler, job) (increase(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[4w]))`), Labels: map[string]string{"slo": "monitoring-http-latency"}, }, { Record: "http_request_duration_seconds:increase4w", - Expr: intstr.FromString(`sum by(code, handler, job) (increase(http_request_duration_seconds_bucket{code=~"2..",job="metrics-service-thanos-receive-default",le="1"}[4w]))`), + Expr: intstr.FromString(`sum by (code, handler, job) (increase(http_request_duration_seconds_bucket{code=~"2..",job="metrics-service-thanos-receive-default",le="1"}[4w]))`), Labels: map[string]string{"slo": "monitoring-http-latency", "le": "1"}, }, { Alert: "SLOMetricAbsent", @@ -1158,11 +1158,11 @@ func TestObjective_IncreaseRules(t *testing.T) { Interval: "2m30s", Rules: []monitoringv1.Rule{{ Record: "http_request_duration_seconds:increase4w", - Expr: intstr.FromString(`sum by(code, handler, job) (increase(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[4w]))`), + Expr: intstr.FromString(`sum by (code, handler, job) (increase(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[4w]))`), Labels: map[string]string{"slo": "monitoring-http-latency"}, }, { Record: "http_request_duration_seconds:increase4w", - Expr: intstr.FromString(`sum by(code, handler, job) (increase(http_request_duration_seconds_bucket{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default",le="1"}[4w]))`), + Expr: intstr.FromString(`sum by (code, handler, job) (increase(http_request_duration_seconds_bucket{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default",le="1"}[4w]))`), Labels: map[string]string{"slo": "monitoring-http-latency", "le": "1"}, }, { Alert: "SLOMetricAbsent", @@ -1210,11 +1210,11 @@ func TestObjective_IncreaseRules(t *testing.T) { Interval: "1m", Rules: []monitoringv1.Rule{{ Record: "grpc_server_handling_seconds:increase1w", - Expr: intstr.FromString(`sum by(handler, job) (increase(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1w]))`), + Expr: intstr.FromString(`sum by (handler, job) (increase(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1w]))`), Labels: map[string]string{"slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore"}, }, { Record: "grpc_server_handling_seconds:increase1w", - Expr: intstr.FromString(`sum by(handler, job) (increase(grpc_server_handling_seconds_bucket{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",le="0.6"}[1w]))`), + Expr: intstr.FromString(`sum by (handler, job) (increase(grpc_server_handling_seconds_bucket{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",le="0.6"}[1w]))`), Labels: map[string]string{"slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "le": "0.6"}, }, { Alert: "SLOMetricAbsent", @@ -1262,7 +1262,7 @@ func TestObjective_IncreaseRules(t *testing.T) { Interval: "1m30s", Rules: []monitoringv1.Rule{{ Record: "prometheus_operator_reconcile_operations:increase2w", - Expr: intstr.FromString(`sum by(namespace) (increase(prometheus_operator_reconcile_operations_total[2w]))`), + Expr: intstr.FromString(`sum by (namespace) (increase(prometheus_operator_reconcile_operations_total[2w]))`), Labels: map[string]string{"slo": "monitoring-prometheus-operator-errors"}, }, { Alert: "SLOMetricAbsent", @@ -1271,7 +1271,7 @@ func TestObjective_IncreaseRules(t *testing.T) { Labels: map[string]string{"slo": "monitoring-prometheus-operator-errors", "severity": "critical"}, }, { Record: "prometheus_operator_reconcile_errors:increase2w", - Expr: intstr.FromString(`sum by(namespace) (increase(prometheus_operator_reconcile_errors_total[2w]))`), + Expr: intstr.FromString(`sum by (namespace) (increase(prometheus_operator_reconcile_errors_total[2w]))`), Labels: map[string]string{"slo": "monitoring-prometheus-operator-errors"}, }, { Alert: "SLOMetricAbsent", @@ -1288,7 +1288,7 @@ func TestObjective_IncreaseRules(t *testing.T) { Interval: "1m30s", Rules: []monitoringv1.Rule{{ Record: "apiserver_request:increase2w", - Expr: intstr.FromString(`sum by(code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2w]))`), + Expr: intstr.FromString(`sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2w]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-write-response-errors"}, }, { Alert: "SLOMetricAbsent", @@ -1305,11 +1305,11 @@ func TestObjective_IncreaseRules(t *testing.T) { Interval: "1m30s", Rules: []monitoringv1.Rule{{ Record: "apiserver_request_duration_seconds:increase2w", - Expr: intstr.FromString(`sum by(resource, verb) (increase(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[2w]))`), + Expr: intstr.FromString(`sum by (resource, verb) (increase(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[2w]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-read-resource-latency"}, }, { Record: "apiserver_request_duration_seconds:increase2w", - Expr: intstr.FromString(`sum by(resource, verb) (increase(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[2w]))`), + Expr: intstr.FromString(`sum by (resource, verb) (increase(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[2w]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-read-resource-latency", "le": "0.1"}, }, { Alert: "SLOMetricAbsent",