From 227b099cfef81e4efa568a40451714246f3c81d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel=20Ortu=C3=B1o?= Date: Wed, 4 May 2022 17:39:51 +0200 Subject: [PATCH 1/2] ruler: use dskit grpc client for remote evaluation --- CHANGELOG.md | 11 +- cmd/mimir/config-descriptor.json | 243 +++++++++++++----- cmd/mimir/help-all.txt.tmpl | 20 +- .../index.md | 94 +++++-- pkg/ruler/remotequerier.go | 51 ++-- 5 files changed, 293 insertions(+), 126 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 752e3ed541..95024b40b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,9 +32,18 @@ * `cortex_distributor_ingester_append_failures_total` * [FEATURE] Querier: Added support for [streaming remote read](https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/). Should be noted that benefits of chunking the response are partial here, since in a typical `query-frontend` setup responses will be buffered until they've been completed. #1735 * [FEATURE] Ruler: Allow setting `evaluation_delay` for each rule group via rules group configuration file. #1474 -* [FEATURE] Ruler: Added support for expression remote evaluation. #1536 +* [FEATURE] Ruler: Added support for expression remote evaluation. #1536 #1818 * The following CLI flags (and their respective YAML config options) have been added: * `-ruler.query-frontend.address` + * `-ruler.query-frontend.grpc-max-recv-msg-size` + * `-ruler.query-frontend.grpc-max-send-msg-size` + * `-ruler.query-frontend.grpc-compression` + * `-ruler.query-frontend.grpc-client-rate-limit` + * `-ruler.query-frontend.grpc-client-rate-limit-burst` + * `-ruler.query-frontend.backoff-on-ratelimits` + * `-ruler.query-frontend.backoff-min-period` + * `-ruler.query-frontend.backoff-max-period` + * `-ruler.query-frontend.backoff-retries` * `-ruler.query-frontend.tls-enabled` * `-ruler.query-frontend.tls-ca-path` * `-ruler.query-frontend.tls-cert-path` diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index 36ce146ab7..11a86c15f1 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -7327,70 +7327,189 @@ "fieldType": "string" }, { - "kind": "field", - "name": "tls_enabled", - "required": false, - "desc": "Set to true if query-frontend connection requires TLS.", - "fieldValue": null, - "fieldDefaultValue": false, - "fieldFlag": "ruler.query-frontend.tls-enabled", - "fieldType": "boolean", - "fieldCategory": "advanced" - }, - { - "kind": "field", - "name": "tls_cert_path", - "required": false, - "desc": "Path to the client certificate file, which will be used for authenticating with the server. Also requires the key path to be configured.", - "fieldValue": null, - "fieldDefaultValue": "", - "fieldFlag": "ruler.query-frontend.tls-cert-path", - "fieldType": "string", - "fieldCategory": "advanced" - }, - { - "kind": "field", - "name": "tls_key_path", - "required": false, - "desc": "Path to the key file for the client certificate. Also requires the client certificate to be configured.", - "fieldValue": null, - "fieldDefaultValue": "", - "fieldFlag": "ruler.query-frontend.tls-key-path", - "fieldType": "string", - "fieldCategory": "advanced" - }, - { - "kind": "field", - "name": "tls_ca_path", - "required": false, - "desc": "Path to the CA certificates file to validate server certificate against. If not set, the host's root CA certificates are used.", - "fieldValue": null, - "fieldDefaultValue": "", - "fieldFlag": "ruler.query-frontend.tls-ca-path", - "fieldType": "string", - "fieldCategory": "advanced" - }, - { - "kind": "field", - "name": "tls_server_name", - "required": false, - "desc": "Override the expected name on the server certificate.", - "fieldValue": null, - "fieldDefaultValue": "", - "fieldFlag": "ruler.query-frontend.tls-server-name", - "fieldType": "string", - "fieldCategory": "advanced" - }, - { - "kind": "field", - "name": "tls_insecure_skip_verify", + "kind": "block", + "name": "grpc_client_config", "required": false, - "desc": "Skip validating server certificate.", + "desc": "", + "blockEntries": [ + { + "kind": "field", + "name": "max_recv_msg_size", + "required": false, + "desc": "gRPC client max receive message size (bytes).", + "fieldValue": null, + "fieldDefaultValue": 104857600, + "fieldFlag": "ruler.query-frontend.grpc-max-recv-msg-size", + "fieldType": "int", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "max_send_msg_size", + "required": false, + "desc": "gRPC client max send message size (bytes).", + "fieldValue": null, + "fieldDefaultValue": 104857600, + "fieldFlag": "ruler.query-frontend.grpc-max-send-msg-size", + "fieldType": "int", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "grpc_compression", + "required": false, + "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "ruler.query-frontend.grpc-compression", + "fieldType": "string", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "rate_limit", + "required": false, + "desc": "Rate limit for gRPC client; 0 means disabled.", + "fieldValue": null, + "fieldDefaultValue": 0, + "fieldFlag": "ruler.query-frontend.grpc-client-rate-limit", + "fieldType": "float", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "rate_limit_burst", + "required": false, + "desc": "Rate limit burst for gRPC client.", + "fieldValue": null, + "fieldDefaultValue": 0, + "fieldFlag": "ruler.query-frontend.grpc-client-rate-limit-burst", + "fieldType": "int", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "backoff_on_ratelimits", + "required": false, + "desc": "Enable backoff and retry when we hit ratelimits.", + "fieldValue": null, + "fieldDefaultValue": false, + "fieldFlag": "ruler.query-frontend.backoff-on-ratelimits", + "fieldType": "boolean", + "fieldCategory": "advanced" + }, + { + "kind": "block", + "name": "backoff_config", + "required": false, + "desc": "", + "blockEntries": [ + { + "kind": "field", + "name": "min_period", + "required": false, + "desc": "Minimum delay when backing off.", + "fieldValue": null, + "fieldDefaultValue": 100000000, + "fieldFlag": "ruler.query-frontend.backoff-min-period", + "fieldType": "duration", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "max_period", + "required": false, + "desc": "Maximum delay when backing off.", + "fieldValue": null, + "fieldDefaultValue": 10000000000, + "fieldFlag": "ruler.query-frontend.backoff-max-period", + "fieldType": "duration", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "max_retries", + "required": false, + "desc": "Number of times to backoff and retry before failing.", + "fieldValue": null, + "fieldDefaultValue": 10, + "fieldFlag": "ruler.query-frontend.backoff-retries", + "fieldType": "int", + "fieldCategory": "advanced" + } + ], + "fieldValue": null, + "fieldDefaultValue": null + }, + { + "kind": "field", + "name": "tls_enabled", + "required": false, + "desc": "Enable TLS in the GRPC client. This flag needs to be enabled when any other TLS flag is set. If set to false, insecure connection to gRPC server will be used.", + "fieldValue": null, + "fieldDefaultValue": false, + "fieldFlag": "ruler.query-frontend.tls-enabled", + "fieldType": "boolean", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "tls_cert_path", + "required": false, + "desc": "Path to the client certificate file, which will be used for authenticating with the server. Also requires the key path to be configured.", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "ruler.query-frontend.tls-cert-path", + "fieldType": "string", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "tls_key_path", + "required": false, + "desc": "Path to the key file for the client certificate. Also requires the client certificate to be configured.", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "ruler.query-frontend.tls-key-path", + "fieldType": "string", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "tls_ca_path", + "required": false, + "desc": "Path to the CA certificates file to validate server certificate against. If not set, the host's root CA certificates are used.", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "ruler.query-frontend.tls-ca-path", + "fieldType": "string", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "tls_server_name", + "required": false, + "desc": "Override the expected name on the server certificate.", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "ruler.query-frontend.tls-server-name", + "fieldType": "string", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "tls_insecure_skip_verify", + "required": false, + "desc": "Skip validating server certificate.", + "fieldValue": null, + "fieldDefaultValue": false, + "fieldFlag": "ruler.query-frontend.tls-insecure-skip-verify", + "fieldType": "boolean", + "fieldCategory": "advanced" + } + ], "fieldValue": null, - "fieldDefaultValue": false, - "fieldFlag": "ruler.query-frontend.tls-insecure-skip-verify", - "fieldType": "boolean", - "fieldCategory": "advanced" + "fieldDefaultValue": null } ], "fieldValue": null, diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index 8964a5f29c..dc19e12fa5 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -1406,12 +1406,30 @@ Usage of ./cmd/mimir/mimir: How frequently to poll for rule changes (default 1m0s) -ruler.query-frontend.address string GRPC listen address of the query-frontend(s). Must be a DNS address (prefixed with dns:///) to enable client side load balancing. + -ruler.query-frontend.backoff-max-period duration + Maximum delay when backing off. (default 10s) + -ruler.query-frontend.backoff-min-period duration + Minimum delay when backing off. (default 100ms) + -ruler.query-frontend.backoff-on-ratelimits + Enable backoff and retry when we hit ratelimits. + -ruler.query-frontend.backoff-retries int + Number of times to backoff and retry before failing. (default 10) + -ruler.query-frontend.grpc-client-rate-limit float + Rate limit for gRPC client; 0 means disabled. + -ruler.query-frontend.grpc-client-rate-limit-burst int + Rate limit burst for gRPC client. + -ruler.query-frontend.grpc-compression string + Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression) + -ruler.query-frontend.grpc-max-recv-msg-size int + gRPC client max receive message size (bytes). (default 104857600) + -ruler.query-frontend.grpc-max-send-msg-size int + gRPC client max send message size (bytes). (default 104857600) -ruler.query-frontend.tls-ca-path string Path to the CA certificates file to validate server certificate against. If not set, the host's root CA certificates are used. -ruler.query-frontend.tls-cert-path string Path to the client certificate file, which will be used for authenticating with the server. Also requires the key path to be configured. -ruler.query-frontend.tls-enabled - Set to true if query-frontend connection requires TLS. + Enable TLS in the GRPC client. This flag needs to be enabled when any other TLS flag is set. If set to false, insecure connection to gRPC server will be used. -ruler.query-frontend.tls-insecure-skip-verify Skip validating server certificate. -ruler.query-frontend.tls-key-path string diff --git a/docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md b/docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md index 34531c9760..9da451e773 100644 --- a/docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md +++ b/docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md @@ -1408,32 +1408,74 @@ query_frontend: # CLI flag: -ruler.query-frontend.address [address: | default = ""] - # (advanced) Set to true if query-frontend connection requires TLS. - # CLI flag: -ruler.query-frontend.tls-enabled - [tls_enabled: | default = false] - - # (advanced) Path to the client certificate file, which will be used for - # authenticating with the server. Also requires the key path to be configured. - # CLI flag: -ruler.query-frontend.tls-cert-path - [tls_cert_path: | default = ""] - - # (advanced) Path to the key file for the client certificate. Also requires - # the client certificate to be configured. - # CLI flag: -ruler.query-frontend.tls-key-path - [tls_key_path: | default = ""] - - # (advanced) Path to the CA certificates file to validate server certificate - # against. If not set, the host's root CA certificates are used. - # CLI flag: -ruler.query-frontend.tls-ca-path - [tls_ca_path: | default = ""] - - # (advanced) Override the expected name on the server certificate. - # CLI flag: -ruler.query-frontend.tls-server-name - [tls_server_name: | default = ""] - - # (advanced) Skip validating server certificate. - # CLI flag: -ruler.query-frontend.tls-insecure-skip-verify - [tls_insecure_skip_verify: | default = false] + grpc_client_config: + # (advanced) gRPC client max receive message size (bytes). + # CLI flag: -ruler.query-frontend.grpc-max-recv-msg-size + [max_recv_msg_size: | default = 104857600] + + # (advanced) gRPC client max send message size (bytes). + # CLI flag: -ruler.query-frontend.grpc-max-send-msg-size + [max_send_msg_size: | default = 104857600] + + # (advanced) Use compression when sending messages. Supported values are: + # 'gzip', 'snappy' and '' (disable compression) + # CLI flag: -ruler.query-frontend.grpc-compression + [grpc_compression: | default = ""] + + # (advanced) Rate limit for gRPC client; 0 means disabled. + # CLI flag: -ruler.query-frontend.grpc-client-rate-limit + [rate_limit: | default = 0] + + # (advanced) Rate limit burst for gRPC client. + # CLI flag: -ruler.query-frontend.grpc-client-rate-limit-burst + [rate_limit_burst: | default = 0] + + # (advanced) Enable backoff and retry when we hit ratelimits. + # CLI flag: -ruler.query-frontend.backoff-on-ratelimits + [backoff_on_ratelimits: | default = false] + + backoff_config: + # (advanced) Minimum delay when backing off. + # CLI flag: -ruler.query-frontend.backoff-min-period + [min_period: | default = 100ms] + + # (advanced) Maximum delay when backing off. + # CLI flag: -ruler.query-frontend.backoff-max-period + [max_period: | default = 10s] + + # (advanced) Number of times to backoff and retry before failing. + # CLI flag: -ruler.query-frontend.backoff-retries + [max_retries: | default = 10] + + # (advanced) Enable TLS in the GRPC client. This flag needs to be enabled + # when any other TLS flag is set. If set to false, insecure connection to + # gRPC server will be used. + # CLI flag: -ruler.query-frontend.tls-enabled + [tls_enabled: | default = false] + + # (advanced) Path to the client certificate file, which will be used for + # authenticating with the server. Also requires the key path to be + # configured. + # CLI flag: -ruler.query-frontend.tls-cert-path + [tls_cert_path: | default = ""] + + # (advanced) Path to the key file for the client certificate. Also requires + # the client certificate to be configured. + # CLI flag: -ruler.query-frontend.tls-key-path + [tls_key_path: | default = ""] + + # (advanced) Path to the CA certificates file to validate server certificate + # against. If not set, the host's root CA certificates are used. + # CLI flag: -ruler.query-frontend.tls-ca-path + [tls_ca_path: | default = ""] + + # (advanced) Override the expected name on the server certificate. + # CLI flag: -ruler.query-frontend.tls-server-name + [tls_server_name: | default = ""] + + # (advanced) Skip validating server certificate. + # CLI flag: -ruler.query-frontend.tls-insecure-skip-verify + [tls_insecure_skip_verify: | default = false] tenant_federation: # Enable running rule groups against multiple tenants. The tenant IDs involved diff --git a/pkg/ruler/remotequerier.go b/pkg/ruler/remotequerier.go index fb3387c828..df6dfcbd36 100644 --- a/pkg/ruler/remotequerier.go +++ b/pkg/ruler/remotequerier.go @@ -18,7 +18,7 @@ import ( "github.com/go-kit/log/level" "github.com/gogo/protobuf/proto" "github.com/golang/snappy" - "github.com/grafana/dskit/crypto/tls" + "github.com/grafana/dskit/grpcclient" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" otgrpc "github.com/opentracing-contrib/go-grpc" "github.com/opentracing/opentracing-go" @@ -32,16 +32,12 @@ import ( "github.com/weaveworks/common/middleware" "github.com/weaveworks/common/user" "google.golang.org/grpc" - "google.golang.org/grpc/keepalive" "github.com/grafana/mimir/pkg/util/spanlogger" "github.com/grafana/mimir/pkg/util/version" ) const ( - keepAlive = time.Second * 10 - keepAliveTimeout = time.Second * 5 - serviceConfig = `{"loadBalancingPolicy": "round_robin"}` readEndpointPath = "/api/v1/read" @@ -56,14 +52,11 @@ var userAgent = fmt.Sprintf("mimir/%s", version.Version) // QueryFrontendConfig defines query-frontend transport configuration. type QueryFrontendConfig struct { - // The address of the remote querier to connect to. + // Address is the address of the query-frontend to connect to. Address string `yaml:"address"` - // TLSEnabled tells whether TLS should be used to establish remote connection. - TLSEnabled bool `yaml:"tls_enabled" category:"advanced"` - - // TLS is the config for client TLS. - TLS tls.ClientConfig `yaml:",inline"` + // GRPCClientConfig contains gRPC specific config options. + GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config"` } func (c *QueryFrontendConfig) RegisterFlags(f *flag.FlagSet) { @@ -73,38 +66,24 @@ func (c *QueryFrontendConfig) RegisterFlags(f *flag.FlagSet) { "GRPC listen address of the query-frontend(s). Must be a DNS address (prefixed with dns:///) "+ "to enable client side load balancing.") - f.BoolVar(&c.TLSEnabled, "ruler.query-frontend.tls-enabled", false, "Set to true if query-frontend connection requires TLS.") - - c.TLS.RegisterFlagsWithPrefix("ruler.query-frontend", f) + c.GRPCClientConfig.RegisterFlagsWithPrefix("ruler.query-frontend", f) } // DialQueryFrontend creates and initializes a new httpgrpc.HTTPClient taking a QueryFrontendConfig configuration. func DialQueryFrontend(cfg QueryFrontendConfig) (httpgrpc.HTTPClient, error) { - tlsDialOptions, err := cfg.TLS.GetGRPCDialOptions(cfg.TLSEnabled) + opts, err := cfg.GRPCClientConfig.DialOption(nil, nil) if err != nil { return nil, err } - dialOptions := append( - []grpc.DialOption{ - grpc.WithKeepaliveParams( - keepalive.ClientParameters{ - Time: keepAlive, - Timeout: keepAliveTimeout, - PermitWithoutStream: true, - }, - ), - grpc.WithUnaryInterceptor( - grpc_middleware.ChainUnaryClient( - otgrpc.OpenTracingClientInterceptor(opentracing.GlobalTracer()), - middleware.ClientUserHeaderInterceptor, - ), - ), - grpc.WithDefaultServiceConfig(serviceConfig), - }, - tlsDialOptions..., - ) - - conn, err := grpc.Dial(cfg.Address, dialOptions...) + opts = append(opts, grpc.WithUnaryInterceptor( + grpc_middleware.ChainUnaryClient( + otgrpc.OpenTracingClientInterceptor(opentracing.GlobalTracer()), + middleware.ClientUserHeaderInterceptor, + ), + )) + opts = append(opts, grpc.WithDefaultServiceConfig(serviceConfig)) + + conn, err := grpc.Dial(cfg.Address, opts...) if err != nil { return nil, err } From e4f0a3c12f81de93515431a881d33e1d3df12c65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel=20Ortu=C3=B1o?= Date: Thu, 5 May 2022 17:26:56 +0200 Subject: [PATCH 2/2] addressed PR feedback --- CHANGELOG.md | 30 +++++++++---------- cmd/mimir/config-descriptor.json | 30 +++++++++---------- cmd/mimir/help-all.txt.tmpl | 30 +++++++++---------- .../index.md | 30 +++++++++---------- go.mod | 2 +- pkg/ruler/remotequerier.go | 14 ++++----- 6 files changed, 66 insertions(+), 70 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95024b40b1..0d2902da3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,21 +35,21 @@ * [FEATURE] Ruler: Added support for expression remote evaluation. #1536 #1818 * The following CLI flags (and their respective YAML config options) have been added: * `-ruler.query-frontend.address` - * `-ruler.query-frontend.grpc-max-recv-msg-size` - * `-ruler.query-frontend.grpc-max-send-msg-size` - * `-ruler.query-frontend.grpc-compression` - * `-ruler.query-frontend.grpc-client-rate-limit` - * `-ruler.query-frontend.grpc-client-rate-limit-burst` - * `-ruler.query-frontend.backoff-on-ratelimits` - * `-ruler.query-frontend.backoff-min-period` - * `-ruler.query-frontend.backoff-max-period` - * `-ruler.query-frontend.backoff-retries` - * `-ruler.query-frontend.tls-enabled` - * `-ruler.query-frontend.tls-ca-path` - * `-ruler.query-frontend.tls-cert-path` - * `-ruler.query-frontend.tls-key-path` - * `-ruler.query-frontend.tls-server-name` - * `-ruler.query-frontend.tls-insecure-skip-verify` + * `-ruler.query-frontend.grpc-client-config.grpc-max-recv-msg-size` + * `-ruler.query-frontend.grpc-client-config.grpc-max-send-msg-size` + * `-ruler.query-frontend.grpc-client-config.grpc-compression` + * `-ruler.query-frontend.grpc-client-config.grpc-client-rate-limit` + * `-ruler.query-frontend.grpc-client-config.grpc-client-rate-limit-burst` + * `-ruler.query-frontend.grpc-client-config.backoff-on-ratelimits` + * `-ruler.query-frontend.grpc-client-config.backoff-min-period` + * `-ruler.query-frontend.grpc-client-config.backoff-max-period` + * `-ruler.query-frontend.grpc-client-config.backoff-retries` + * `-ruler.query-frontend.grpc-client-config.tls-enabled` + * `-ruler.query-frontend.grpc-client-config.tls-ca-path` + * `-ruler.query-frontend.grpc-client-config.tls-cert-path` + * `-ruler.query-frontend.grpc-client-config.tls-key-path` + * `-ruler.query-frontend.grpc-client-config.tls-server-name` + * `-ruler.query-frontend.grpc-client-config.tls-insecure-skip-verify` * [FEATURE] Distributor: Added the ability to forward specifics metrics to alternative remote_write API endpoints. #1052 * [FEATURE] Ingester: Active series custom trackers now supports runtime tenant-specific overrides. The configuration has been moved to limit config, the ingester config has been deprecated. #1188 * [ENHANCEMENT] Alertmanager API: Concurrency limit for GET requests is now configurable using `-alertmanager.max-concurrent-get-requests-per-tenant`. #1547 diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index 11a86c15f1..5eaede5f28 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -7339,7 +7339,7 @@ "desc": "gRPC client max receive message size (bytes).", "fieldValue": null, "fieldDefaultValue": 104857600, - "fieldFlag": "ruler.query-frontend.grpc-max-recv-msg-size", + "fieldFlag": "ruler.query-frontend.grpc-client-config.grpc-max-recv-msg-size", "fieldType": "int", "fieldCategory": "advanced" }, @@ -7350,7 +7350,7 @@ "desc": "gRPC client max send message size (bytes).", "fieldValue": null, "fieldDefaultValue": 104857600, - "fieldFlag": "ruler.query-frontend.grpc-max-send-msg-size", + "fieldFlag": "ruler.query-frontend.grpc-client-config.grpc-max-send-msg-size", "fieldType": "int", "fieldCategory": "advanced" }, @@ -7361,7 +7361,7 @@ "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)", "fieldValue": null, "fieldDefaultValue": "", - "fieldFlag": "ruler.query-frontend.grpc-compression", + "fieldFlag": "ruler.query-frontend.grpc-client-config.grpc-compression", "fieldType": "string", "fieldCategory": "advanced" }, @@ -7372,7 +7372,7 @@ "desc": "Rate limit for gRPC client; 0 means disabled.", "fieldValue": null, "fieldDefaultValue": 0, - "fieldFlag": "ruler.query-frontend.grpc-client-rate-limit", + "fieldFlag": "ruler.query-frontend.grpc-client-config.grpc-client-rate-limit", "fieldType": "float", "fieldCategory": "advanced" }, @@ -7383,7 +7383,7 @@ "desc": "Rate limit burst for gRPC client.", "fieldValue": null, "fieldDefaultValue": 0, - "fieldFlag": "ruler.query-frontend.grpc-client-rate-limit-burst", + "fieldFlag": "ruler.query-frontend.grpc-client-config.grpc-client-rate-limit-burst", "fieldType": "int", "fieldCategory": "advanced" }, @@ -7394,7 +7394,7 @@ "desc": "Enable backoff and retry when we hit ratelimits.", "fieldValue": null, "fieldDefaultValue": false, - "fieldFlag": "ruler.query-frontend.backoff-on-ratelimits", + "fieldFlag": "ruler.query-frontend.grpc-client-config.backoff-on-ratelimits", "fieldType": "boolean", "fieldCategory": "advanced" }, @@ -7411,7 +7411,7 @@ "desc": "Minimum delay when backing off.", "fieldValue": null, "fieldDefaultValue": 100000000, - "fieldFlag": "ruler.query-frontend.backoff-min-period", + "fieldFlag": "ruler.query-frontend.grpc-client-config.backoff-min-period", "fieldType": "duration", "fieldCategory": "advanced" }, @@ -7422,7 +7422,7 @@ "desc": "Maximum delay when backing off.", "fieldValue": null, "fieldDefaultValue": 10000000000, - "fieldFlag": "ruler.query-frontend.backoff-max-period", + "fieldFlag": "ruler.query-frontend.grpc-client-config.backoff-max-period", "fieldType": "duration", "fieldCategory": "advanced" }, @@ -7433,7 +7433,7 @@ "desc": "Number of times to backoff and retry before failing.", "fieldValue": null, "fieldDefaultValue": 10, - "fieldFlag": "ruler.query-frontend.backoff-retries", + "fieldFlag": "ruler.query-frontend.grpc-client-config.backoff-retries", "fieldType": "int", "fieldCategory": "advanced" } @@ -7448,7 +7448,7 @@ "desc": "Enable TLS in the GRPC client. This flag needs to be enabled when any other TLS flag is set. If set to false, insecure connection to gRPC server will be used.", "fieldValue": null, "fieldDefaultValue": false, - "fieldFlag": "ruler.query-frontend.tls-enabled", + "fieldFlag": "ruler.query-frontend.grpc-client-config.tls-enabled", "fieldType": "boolean", "fieldCategory": "advanced" }, @@ -7459,7 +7459,7 @@ "desc": "Path to the client certificate file, which will be used for authenticating with the server. Also requires the key path to be configured.", "fieldValue": null, "fieldDefaultValue": "", - "fieldFlag": "ruler.query-frontend.tls-cert-path", + "fieldFlag": "ruler.query-frontend.grpc-client-config.tls-cert-path", "fieldType": "string", "fieldCategory": "advanced" }, @@ -7470,7 +7470,7 @@ "desc": "Path to the key file for the client certificate. Also requires the client certificate to be configured.", "fieldValue": null, "fieldDefaultValue": "", - "fieldFlag": "ruler.query-frontend.tls-key-path", + "fieldFlag": "ruler.query-frontend.grpc-client-config.tls-key-path", "fieldType": "string", "fieldCategory": "advanced" }, @@ -7481,7 +7481,7 @@ "desc": "Path to the CA certificates file to validate server certificate against. If not set, the host's root CA certificates are used.", "fieldValue": null, "fieldDefaultValue": "", - "fieldFlag": "ruler.query-frontend.tls-ca-path", + "fieldFlag": "ruler.query-frontend.grpc-client-config.tls-ca-path", "fieldType": "string", "fieldCategory": "advanced" }, @@ -7492,7 +7492,7 @@ "desc": "Override the expected name on the server certificate.", "fieldValue": null, "fieldDefaultValue": "", - "fieldFlag": "ruler.query-frontend.tls-server-name", + "fieldFlag": "ruler.query-frontend.grpc-client-config.tls-server-name", "fieldType": "string", "fieldCategory": "advanced" }, @@ -7503,7 +7503,7 @@ "desc": "Skip validating server certificate.", "fieldValue": null, "fieldDefaultValue": false, - "fieldFlag": "ruler.query-frontend.tls-insecure-skip-verify", + "fieldFlag": "ruler.query-frontend.grpc-client-config.tls-insecure-skip-verify", "fieldType": "boolean", "fieldCategory": "advanced" } diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index dc19e12fa5..b7632305b1 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -1406,35 +1406,35 @@ Usage of ./cmd/mimir/mimir: How frequently to poll for rule changes (default 1m0s) -ruler.query-frontend.address string GRPC listen address of the query-frontend(s). Must be a DNS address (prefixed with dns:///) to enable client side load balancing. - -ruler.query-frontend.backoff-max-period duration + -ruler.query-frontend.grpc-client-config.backoff-max-period duration Maximum delay when backing off. (default 10s) - -ruler.query-frontend.backoff-min-period duration + -ruler.query-frontend.grpc-client-config.backoff-min-period duration Minimum delay when backing off. (default 100ms) - -ruler.query-frontend.backoff-on-ratelimits + -ruler.query-frontend.grpc-client-config.backoff-on-ratelimits Enable backoff and retry when we hit ratelimits. - -ruler.query-frontend.backoff-retries int + -ruler.query-frontend.grpc-client-config.backoff-retries int Number of times to backoff and retry before failing. (default 10) - -ruler.query-frontend.grpc-client-rate-limit float + -ruler.query-frontend.grpc-client-config.grpc-client-rate-limit float Rate limit for gRPC client; 0 means disabled. - -ruler.query-frontend.grpc-client-rate-limit-burst int + -ruler.query-frontend.grpc-client-config.grpc-client-rate-limit-burst int Rate limit burst for gRPC client. - -ruler.query-frontend.grpc-compression string + -ruler.query-frontend.grpc-client-config.grpc-compression string Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression) - -ruler.query-frontend.grpc-max-recv-msg-size int + -ruler.query-frontend.grpc-client-config.grpc-max-recv-msg-size int gRPC client max receive message size (bytes). (default 104857600) - -ruler.query-frontend.grpc-max-send-msg-size int + -ruler.query-frontend.grpc-client-config.grpc-max-send-msg-size int gRPC client max send message size (bytes). (default 104857600) - -ruler.query-frontend.tls-ca-path string + -ruler.query-frontend.grpc-client-config.tls-ca-path string Path to the CA certificates file to validate server certificate against. If not set, the host's root CA certificates are used. - -ruler.query-frontend.tls-cert-path string + -ruler.query-frontend.grpc-client-config.tls-cert-path string Path to the client certificate file, which will be used for authenticating with the server. Also requires the key path to be configured. - -ruler.query-frontend.tls-enabled + -ruler.query-frontend.grpc-client-config.tls-enabled Enable TLS in the GRPC client. This flag needs to be enabled when any other TLS flag is set. If set to false, insecure connection to gRPC server will be used. - -ruler.query-frontend.tls-insecure-skip-verify + -ruler.query-frontend.grpc-client-config.tls-insecure-skip-verify Skip validating server certificate. - -ruler.query-frontend.tls-key-path string + -ruler.query-frontend.grpc-client-config.tls-key-path string Path to the key file for the client certificate. Also requires the client certificate to be configured. - -ruler.query-frontend.tls-server-name string + -ruler.query-frontend.grpc-client-config.tls-server-name string Override the expected name on the server certificate. -ruler.query-stats-enabled Report the wall time for ruler queries to complete as a per-tenant metric and as an info level log message. diff --git a/docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md b/docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md index 9da451e773..74733e57bb 100644 --- a/docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md +++ b/docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md @@ -1410,71 +1410,71 @@ query_frontend: grpc_client_config: # (advanced) gRPC client max receive message size (bytes). - # CLI flag: -ruler.query-frontend.grpc-max-recv-msg-size + # CLI flag: -ruler.query-frontend.grpc-client-config.grpc-max-recv-msg-size [max_recv_msg_size: | default = 104857600] # (advanced) gRPC client max send message size (bytes). - # CLI flag: -ruler.query-frontend.grpc-max-send-msg-size + # CLI flag: -ruler.query-frontend.grpc-client-config.grpc-max-send-msg-size [max_send_msg_size: | default = 104857600] # (advanced) Use compression when sending messages. Supported values are: # 'gzip', 'snappy' and '' (disable compression) - # CLI flag: -ruler.query-frontend.grpc-compression + # CLI flag: -ruler.query-frontend.grpc-client-config.grpc-compression [grpc_compression: | default = ""] # (advanced) Rate limit for gRPC client; 0 means disabled. - # CLI flag: -ruler.query-frontend.grpc-client-rate-limit + # CLI flag: -ruler.query-frontend.grpc-client-config.grpc-client-rate-limit [rate_limit: | default = 0] # (advanced) Rate limit burst for gRPC client. - # CLI flag: -ruler.query-frontend.grpc-client-rate-limit-burst + # CLI flag: -ruler.query-frontend.grpc-client-config.grpc-client-rate-limit-burst [rate_limit_burst: | default = 0] # (advanced) Enable backoff and retry when we hit ratelimits. - # CLI flag: -ruler.query-frontend.backoff-on-ratelimits + # CLI flag: -ruler.query-frontend.grpc-client-config.backoff-on-ratelimits [backoff_on_ratelimits: | default = false] backoff_config: # (advanced) Minimum delay when backing off. - # CLI flag: -ruler.query-frontend.backoff-min-period + # CLI flag: -ruler.query-frontend.grpc-client-config.backoff-min-period [min_period: | default = 100ms] # (advanced) Maximum delay when backing off. - # CLI flag: -ruler.query-frontend.backoff-max-period + # CLI flag: -ruler.query-frontend.grpc-client-config.backoff-max-period [max_period: | default = 10s] # (advanced) Number of times to backoff and retry before failing. - # CLI flag: -ruler.query-frontend.backoff-retries + # CLI flag: -ruler.query-frontend.grpc-client-config.backoff-retries [max_retries: | default = 10] # (advanced) Enable TLS in the GRPC client. This flag needs to be enabled # when any other TLS flag is set. If set to false, insecure connection to # gRPC server will be used. - # CLI flag: -ruler.query-frontend.tls-enabled + # CLI flag: -ruler.query-frontend.grpc-client-config.tls-enabled [tls_enabled: | default = false] # (advanced) Path to the client certificate file, which will be used for # authenticating with the server. Also requires the key path to be # configured. - # CLI flag: -ruler.query-frontend.tls-cert-path + # CLI flag: -ruler.query-frontend.grpc-client-config.tls-cert-path [tls_cert_path: | default = ""] # (advanced) Path to the key file for the client certificate. Also requires # the client certificate to be configured. - # CLI flag: -ruler.query-frontend.tls-key-path + # CLI flag: -ruler.query-frontend.grpc-client-config.tls-key-path [tls_key_path: | default = ""] # (advanced) Path to the CA certificates file to validate server certificate # against. If not set, the host's root CA certificates are used. - # CLI flag: -ruler.query-frontend.tls-ca-path + # CLI flag: -ruler.query-frontend.grpc-client-config.tls-ca-path [tls_ca_path: | default = ""] # (advanced) Override the expected name on the server certificate. - # CLI flag: -ruler.query-frontend.tls-server-name + # CLI flag: -ruler.query-frontend.grpc-client-config.tls-server-name [tls_server_name: | default = ""] # (advanced) Skip validating server certificate. - # CLI flag: -ruler.query-frontend.tls-insecure-skip-verify + # CLI flag: -ruler.query-frontend.grpc-client-config.tls-insecure-skip-verify [tls_insecure_skip_verify: | default = false] tenant_federation: diff --git a/go.mod b/go.mod index 023700bda7..ab20276fcf 100644 --- a/go.mod +++ b/go.mod @@ -58,7 +58,6 @@ require ( github.com/google/go-github/v32 v32.1.0 github.com/grafana-tools/sdk v0.0.0-20211220201350-966b3088eec9 github.com/grafana/regexp v0.0.0-20220304095617-2e8d9baf4ac2 - github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db gopkg.in/alecthomas/kingpin.v2 v2.2.6 ) @@ -139,6 +138,7 @@ require ( github.com/googleapis/gax-go/v2 v2.2.0 // indirect github.com/gopherjs/gopherjs v0.0.0-20191106031601-ce3c9ade29de // indirect github.com/gosimple/slug v1.1.1 // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7 // indirect github.com/hashicorp/consul/api v1.12.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect diff --git a/pkg/ruler/remotequerier.go b/pkg/ruler/remotequerier.go index df6dfcbd36..419f06a1ae 100644 --- a/pkg/ruler/remotequerier.go +++ b/pkg/ruler/remotequerier.go @@ -19,7 +19,6 @@ import ( "github.com/gogo/protobuf/proto" "github.com/golang/snappy" "github.com/grafana/dskit/grpcclient" - grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" otgrpc "github.com/opentracing-contrib/go-grpc" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" @@ -66,21 +65,18 @@ func (c *QueryFrontendConfig) RegisterFlags(f *flag.FlagSet) { "GRPC listen address of the query-frontend(s). Must be a DNS address (prefixed with dns:///) "+ "to enable client side load balancing.") - c.GRPCClientConfig.RegisterFlagsWithPrefix("ruler.query-frontend", f) + c.GRPCClientConfig.RegisterFlagsWithPrefix("ruler.query-frontend.grpc-client-config", f) } // DialQueryFrontend creates and initializes a new httpgrpc.HTTPClient taking a QueryFrontendConfig configuration. func DialQueryFrontend(cfg QueryFrontendConfig) (httpgrpc.HTTPClient, error) { - opts, err := cfg.GRPCClientConfig.DialOption(nil, nil) + opts, err := cfg.GRPCClientConfig.DialOption([]grpc.UnaryClientInterceptor{ + otgrpc.OpenTracingClientInterceptor(opentracing.GlobalTracer()), + middleware.ClientUserHeaderInterceptor, + }, nil) if err != nil { return nil, err } - opts = append(opts, grpc.WithUnaryInterceptor( - grpc_middleware.ChainUnaryClient( - otgrpc.OpenTracingClientInterceptor(opentracing.GlobalTracer()), - middleware.ClientUserHeaderInterceptor, - ), - )) opts = append(opts, grpc.WithDefaultServiceConfig(serviceConfig)) conn, err := grpc.Dial(cfg.Address, opts...)