diff --git a/CHANGELOG.md b/CHANGELOG.md index 752e3ed541a..0d2902da3fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,15 +32,24 @@ * `cortex_distributor_ingester_append_failures_total` * [FEATURE] Querier: Added support for [streaming remote read](https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/). Should be noted that benefits of chunking the response are partial here, since in a typical `query-frontend` setup responses will be buffered until they've been completed. #1735 * [FEATURE] Ruler: Allow setting `evaluation_delay` for each rule group via rules group configuration file. #1474 -* [FEATURE] Ruler: Added support for expression remote evaluation. #1536 +* [FEATURE] Ruler: Added support for expression remote evaluation. #1536 #1818 * The following CLI flags (and their respective YAML config options) have been added: * `-ruler.query-frontend.address` - * `-ruler.query-frontend.tls-enabled` - * `-ruler.query-frontend.tls-ca-path` - * `-ruler.query-frontend.tls-cert-path` - * `-ruler.query-frontend.tls-key-path` - * `-ruler.query-frontend.tls-server-name` - * `-ruler.query-frontend.tls-insecure-skip-verify` + * `-ruler.query-frontend.grpc-client-config.grpc-max-recv-msg-size` + * `-ruler.query-frontend.grpc-client-config.grpc-max-send-msg-size` + * `-ruler.query-frontend.grpc-client-config.grpc-compression` + * `-ruler.query-frontend.grpc-client-config.grpc-client-rate-limit` + * `-ruler.query-frontend.grpc-client-config.grpc-client-rate-limit-burst` + * `-ruler.query-frontend.grpc-client-config.backoff-on-ratelimits` + * `-ruler.query-frontend.grpc-client-config.backoff-min-period` + * `-ruler.query-frontend.grpc-client-config.backoff-max-period` + * `-ruler.query-frontend.grpc-client-config.backoff-retries` + * `-ruler.query-frontend.grpc-client-config.tls-enabled` + * `-ruler.query-frontend.grpc-client-config.tls-ca-path` + * `-ruler.query-frontend.grpc-client-config.tls-cert-path` + * `-ruler.query-frontend.grpc-client-config.tls-key-path` + * `-ruler.query-frontend.grpc-client-config.tls-server-name` + * `-ruler.query-frontend.grpc-client-config.tls-insecure-skip-verify` * [FEATURE] Distributor: Added the ability to forward specifics metrics to alternative remote_write API endpoints. #1052 * [FEATURE] Ingester: Active series custom trackers now supports runtime tenant-specific overrides. The configuration has been moved to limit config, the ingester config has been deprecated. #1188 * [ENHANCEMENT] Alertmanager API: Concurrency limit for GET requests is now configurable using `-alertmanager.max-concurrent-get-requests-per-tenant`. #1547 diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index 36ce146ab76..5eaede5f288 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -7327,70 +7327,189 @@ "fieldType": "string" }, { - "kind": "field", - "name": "tls_enabled", - "required": false, - "desc": "Set to true if query-frontend connection requires TLS.", - "fieldValue": null, - "fieldDefaultValue": false, - "fieldFlag": "ruler.query-frontend.tls-enabled", - "fieldType": "boolean", - "fieldCategory": "advanced" - }, - { - "kind": "field", - "name": "tls_cert_path", - "required": false, - "desc": "Path to the client certificate file, which will be used for authenticating with the server. Also requires the key path to be configured.", - "fieldValue": null, - "fieldDefaultValue": "", - "fieldFlag": "ruler.query-frontend.tls-cert-path", - "fieldType": "string", - "fieldCategory": "advanced" - }, - { - "kind": "field", - "name": "tls_key_path", - "required": false, - "desc": "Path to the key file for the client certificate. Also requires the client certificate to be configured.", - "fieldValue": null, - "fieldDefaultValue": "", - "fieldFlag": "ruler.query-frontend.tls-key-path", - "fieldType": "string", - "fieldCategory": "advanced" - }, - { - "kind": "field", - "name": "tls_ca_path", - "required": false, - "desc": "Path to the CA certificates file to validate server certificate against. If not set, the host's root CA certificates are used.", - "fieldValue": null, - "fieldDefaultValue": "", - "fieldFlag": "ruler.query-frontend.tls-ca-path", - "fieldType": "string", - "fieldCategory": "advanced" - }, - { - "kind": "field", - "name": "tls_server_name", - "required": false, - "desc": "Override the expected name on the server certificate.", - "fieldValue": null, - "fieldDefaultValue": "", - "fieldFlag": "ruler.query-frontend.tls-server-name", - "fieldType": "string", - "fieldCategory": "advanced" - }, - { - "kind": "field", - "name": "tls_insecure_skip_verify", + "kind": "block", + "name": "grpc_client_config", "required": false, - "desc": "Skip validating server certificate.", + "desc": "", + "blockEntries": [ + { + "kind": "field", + "name": "max_recv_msg_size", + "required": false, + "desc": "gRPC client max receive message size (bytes).", + "fieldValue": null, + "fieldDefaultValue": 104857600, + "fieldFlag": "ruler.query-frontend.grpc-client-config.grpc-max-recv-msg-size", + "fieldType": "int", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "max_send_msg_size", + "required": false, + "desc": "gRPC client max send message size (bytes).", + "fieldValue": null, + "fieldDefaultValue": 104857600, + "fieldFlag": "ruler.query-frontend.grpc-client-config.grpc-max-send-msg-size", + "fieldType": "int", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "grpc_compression", + "required": false, + "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "ruler.query-frontend.grpc-client-config.grpc-compression", + "fieldType": "string", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "rate_limit", + "required": false, + "desc": "Rate limit for gRPC client; 0 means disabled.", + "fieldValue": null, + "fieldDefaultValue": 0, + "fieldFlag": "ruler.query-frontend.grpc-client-config.grpc-client-rate-limit", + "fieldType": "float", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "rate_limit_burst", + "required": false, + "desc": "Rate limit burst for gRPC client.", + "fieldValue": null, + "fieldDefaultValue": 0, + "fieldFlag": "ruler.query-frontend.grpc-client-config.grpc-client-rate-limit-burst", + "fieldType": "int", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "backoff_on_ratelimits", + "required": false, + "desc": "Enable backoff and retry when we hit ratelimits.", + "fieldValue": null, + "fieldDefaultValue": false, + "fieldFlag": "ruler.query-frontend.grpc-client-config.backoff-on-ratelimits", + "fieldType": "boolean", + "fieldCategory": "advanced" + }, + { + "kind": "block", + "name": "backoff_config", + "required": false, + "desc": "", + "blockEntries": [ + { + "kind": "field", + "name": "min_period", + "required": false, + "desc": "Minimum delay when backing off.", + "fieldValue": null, + "fieldDefaultValue": 100000000, + "fieldFlag": "ruler.query-frontend.grpc-client-config.backoff-min-period", + "fieldType": "duration", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "max_period", + "required": false, + "desc": "Maximum delay when backing off.", + "fieldValue": null, + "fieldDefaultValue": 10000000000, + "fieldFlag": "ruler.query-frontend.grpc-client-config.backoff-max-period", + "fieldType": "duration", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "max_retries", + "required": false, + "desc": "Number of times to backoff and retry before failing.", + "fieldValue": null, + "fieldDefaultValue": 10, + "fieldFlag": "ruler.query-frontend.grpc-client-config.backoff-retries", + "fieldType": "int", + "fieldCategory": "advanced" + } + ], + "fieldValue": null, + "fieldDefaultValue": null + }, + { + "kind": "field", + "name": "tls_enabled", + "required": false, + "desc": "Enable TLS in the GRPC client. This flag needs to be enabled when any other TLS flag is set. If set to false, insecure connection to gRPC server will be used.", + "fieldValue": null, + "fieldDefaultValue": false, + "fieldFlag": "ruler.query-frontend.grpc-client-config.tls-enabled", + "fieldType": "boolean", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "tls_cert_path", + "required": false, + "desc": "Path to the client certificate file, which will be used for authenticating with the server. Also requires the key path to be configured.", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "ruler.query-frontend.grpc-client-config.tls-cert-path", + "fieldType": "string", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "tls_key_path", + "required": false, + "desc": "Path to the key file for the client certificate. Also requires the client certificate to be configured.", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "ruler.query-frontend.grpc-client-config.tls-key-path", + "fieldType": "string", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "tls_ca_path", + "required": false, + "desc": "Path to the CA certificates file to validate server certificate against. If not set, the host's root CA certificates are used.", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "ruler.query-frontend.grpc-client-config.tls-ca-path", + "fieldType": "string", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "tls_server_name", + "required": false, + "desc": "Override the expected name on the server certificate.", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "ruler.query-frontend.grpc-client-config.tls-server-name", + "fieldType": "string", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "tls_insecure_skip_verify", + "required": false, + "desc": "Skip validating server certificate.", + "fieldValue": null, + "fieldDefaultValue": false, + "fieldFlag": "ruler.query-frontend.grpc-client-config.tls-insecure-skip-verify", + "fieldType": "boolean", + "fieldCategory": "advanced" + } + ], "fieldValue": null, - "fieldDefaultValue": false, - "fieldFlag": "ruler.query-frontend.tls-insecure-skip-verify", - "fieldType": "boolean", - "fieldCategory": "advanced" + "fieldDefaultValue": null } ], "fieldValue": null, diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index 8964a5f29c6..b7632305b10 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -1406,17 +1406,35 @@ Usage of ./cmd/mimir/mimir: How frequently to poll for rule changes (default 1m0s) -ruler.query-frontend.address string GRPC listen address of the query-frontend(s). Must be a DNS address (prefixed with dns:///) to enable client side load balancing. - -ruler.query-frontend.tls-ca-path string + -ruler.query-frontend.grpc-client-config.backoff-max-period duration + Maximum delay when backing off. (default 10s) + -ruler.query-frontend.grpc-client-config.backoff-min-period duration + Minimum delay when backing off. (default 100ms) + -ruler.query-frontend.grpc-client-config.backoff-on-ratelimits + Enable backoff and retry when we hit ratelimits. + -ruler.query-frontend.grpc-client-config.backoff-retries int + Number of times to backoff and retry before failing. (default 10) + -ruler.query-frontend.grpc-client-config.grpc-client-rate-limit float + Rate limit for gRPC client; 0 means disabled. + -ruler.query-frontend.grpc-client-config.grpc-client-rate-limit-burst int + Rate limit burst for gRPC client. + -ruler.query-frontend.grpc-client-config.grpc-compression string + Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression) + -ruler.query-frontend.grpc-client-config.grpc-max-recv-msg-size int + gRPC client max receive message size (bytes). (default 104857600) + -ruler.query-frontend.grpc-client-config.grpc-max-send-msg-size int + gRPC client max send message size (bytes). (default 104857600) + -ruler.query-frontend.grpc-client-config.tls-ca-path string Path to the CA certificates file to validate server certificate against. If not set, the host's root CA certificates are used. - -ruler.query-frontend.tls-cert-path string + -ruler.query-frontend.grpc-client-config.tls-cert-path string Path to the client certificate file, which will be used for authenticating with the server. Also requires the key path to be configured. - -ruler.query-frontend.tls-enabled - Set to true if query-frontend connection requires TLS. - -ruler.query-frontend.tls-insecure-skip-verify + -ruler.query-frontend.grpc-client-config.tls-enabled + Enable TLS in the GRPC client. This flag needs to be enabled when any other TLS flag is set. If set to false, insecure connection to gRPC server will be used. + -ruler.query-frontend.grpc-client-config.tls-insecure-skip-verify Skip validating server certificate. - -ruler.query-frontend.tls-key-path string + -ruler.query-frontend.grpc-client-config.tls-key-path string Path to the key file for the client certificate. Also requires the client certificate to be configured. - -ruler.query-frontend.tls-server-name string + -ruler.query-frontend.grpc-client-config.tls-server-name string Override the expected name on the server certificate. -ruler.query-stats-enabled Report the wall time for ruler queries to complete as a per-tenant metric and as an info level log message. diff --git a/docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md b/docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md index 34531c97608..74733e57bba 100644 --- a/docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md +++ b/docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md @@ -1408,32 +1408,74 @@ query_frontend: # CLI flag: -ruler.query-frontend.address [address: | default = ""] - # (advanced) Set to true if query-frontend connection requires TLS. - # CLI flag: -ruler.query-frontend.tls-enabled - [tls_enabled: | default = false] - - # (advanced) Path to the client certificate file, which will be used for - # authenticating with the server. Also requires the key path to be configured. - # CLI flag: -ruler.query-frontend.tls-cert-path - [tls_cert_path: | default = ""] - - # (advanced) Path to the key file for the client certificate. Also requires - # the client certificate to be configured. - # CLI flag: -ruler.query-frontend.tls-key-path - [tls_key_path: | default = ""] - - # (advanced) Path to the CA certificates file to validate server certificate - # against. If not set, the host's root CA certificates are used. - # CLI flag: -ruler.query-frontend.tls-ca-path - [tls_ca_path: | default = ""] - - # (advanced) Override the expected name on the server certificate. - # CLI flag: -ruler.query-frontend.tls-server-name - [tls_server_name: | default = ""] - - # (advanced) Skip validating server certificate. - # CLI flag: -ruler.query-frontend.tls-insecure-skip-verify - [tls_insecure_skip_verify: | default = false] + grpc_client_config: + # (advanced) gRPC client max receive message size (bytes). + # CLI flag: -ruler.query-frontend.grpc-client-config.grpc-max-recv-msg-size + [max_recv_msg_size: | default = 104857600] + + # (advanced) gRPC client max send message size (bytes). + # CLI flag: -ruler.query-frontend.grpc-client-config.grpc-max-send-msg-size + [max_send_msg_size: | default = 104857600] + + # (advanced) Use compression when sending messages. Supported values are: + # 'gzip', 'snappy' and '' (disable compression) + # CLI flag: -ruler.query-frontend.grpc-client-config.grpc-compression + [grpc_compression: | default = ""] + + # (advanced) Rate limit for gRPC client; 0 means disabled. + # CLI flag: -ruler.query-frontend.grpc-client-config.grpc-client-rate-limit + [rate_limit: | default = 0] + + # (advanced) Rate limit burst for gRPC client. + # CLI flag: -ruler.query-frontend.grpc-client-config.grpc-client-rate-limit-burst + [rate_limit_burst: | default = 0] + + # (advanced) Enable backoff and retry when we hit ratelimits. + # CLI flag: -ruler.query-frontend.grpc-client-config.backoff-on-ratelimits + [backoff_on_ratelimits: | default = false] + + backoff_config: + # (advanced) Minimum delay when backing off. + # CLI flag: -ruler.query-frontend.grpc-client-config.backoff-min-period + [min_period: | default = 100ms] + + # (advanced) Maximum delay when backing off. + # CLI flag: -ruler.query-frontend.grpc-client-config.backoff-max-period + [max_period: | default = 10s] + + # (advanced) Number of times to backoff and retry before failing. + # CLI flag: -ruler.query-frontend.grpc-client-config.backoff-retries + [max_retries: | default = 10] + + # (advanced) Enable TLS in the GRPC client. This flag needs to be enabled + # when any other TLS flag is set. If set to false, insecure connection to + # gRPC server will be used. + # CLI flag: -ruler.query-frontend.grpc-client-config.tls-enabled + [tls_enabled: | default = false] + + # (advanced) Path to the client certificate file, which will be used for + # authenticating with the server. Also requires the key path to be + # configured. + # CLI flag: -ruler.query-frontend.grpc-client-config.tls-cert-path + [tls_cert_path: | default = ""] + + # (advanced) Path to the key file for the client certificate. Also requires + # the client certificate to be configured. + # CLI flag: -ruler.query-frontend.grpc-client-config.tls-key-path + [tls_key_path: | default = ""] + + # (advanced) Path to the CA certificates file to validate server certificate + # against. If not set, the host's root CA certificates are used. + # CLI flag: -ruler.query-frontend.grpc-client-config.tls-ca-path + [tls_ca_path: | default = ""] + + # (advanced) Override the expected name on the server certificate. + # CLI flag: -ruler.query-frontend.grpc-client-config.tls-server-name + [tls_server_name: | default = ""] + + # (advanced) Skip validating server certificate. + # CLI flag: -ruler.query-frontend.grpc-client-config.tls-insecure-skip-verify + [tls_insecure_skip_verify: | default = false] tenant_federation: # Enable running rule groups against multiple tenants. The tenant IDs involved diff --git a/go.mod b/go.mod index 023700bda78..ab20276fcfc 100644 --- a/go.mod +++ b/go.mod @@ -58,7 +58,6 @@ require ( github.com/google/go-github/v32 v32.1.0 github.com/grafana-tools/sdk v0.0.0-20211220201350-966b3088eec9 github.com/grafana/regexp v0.0.0-20220304095617-2e8d9baf4ac2 - github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db gopkg.in/alecthomas/kingpin.v2 v2.2.6 ) @@ -139,6 +138,7 @@ require ( github.com/googleapis/gax-go/v2 v2.2.0 // indirect github.com/gopherjs/gopherjs v0.0.0-20191106031601-ce3c9ade29de // indirect github.com/gosimple/slug v1.1.1 // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7 // indirect github.com/hashicorp/consul/api v1.12.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect diff --git a/pkg/ruler/remotequerier.go b/pkg/ruler/remotequerier.go index fb3387c8287..419f06a1aeb 100644 --- a/pkg/ruler/remotequerier.go +++ b/pkg/ruler/remotequerier.go @@ -18,8 +18,7 @@ import ( "github.com/go-kit/log/level" "github.com/gogo/protobuf/proto" "github.com/golang/snappy" - "github.com/grafana/dskit/crypto/tls" - grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" + "github.com/grafana/dskit/grpcclient" otgrpc "github.com/opentracing-contrib/go-grpc" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" @@ -32,16 +31,12 @@ import ( "github.com/weaveworks/common/middleware" "github.com/weaveworks/common/user" "google.golang.org/grpc" - "google.golang.org/grpc/keepalive" "github.com/grafana/mimir/pkg/util/spanlogger" "github.com/grafana/mimir/pkg/util/version" ) const ( - keepAlive = time.Second * 10 - keepAliveTimeout = time.Second * 5 - serviceConfig = `{"loadBalancingPolicy": "round_robin"}` readEndpointPath = "/api/v1/read" @@ -56,14 +51,11 @@ var userAgent = fmt.Sprintf("mimir/%s", version.Version) // QueryFrontendConfig defines query-frontend transport configuration. type QueryFrontendConfig struct { - // The address of the remote querier to connect to. + // Address is the address of the query-frontend to connect to. Address string `yaml:"address"` - // TLSEnabled tells whether TLS should be used to establish remote connection. - TLSEnabled bool `yaml:"tls_enabled" category:"advanced"` - - // TLS is the config for client TLS. - TLS tls.ClientConfig `yaml:",inline"` + // GRPCClientConfig contains gRPC specific config options. + GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config"` } func (c *QueryFrontendConfig) RegisterFlags(f *flag.FlagSet) { @@ -73,38 +65,21 @@ func (c *QueryFrontendConfig) RegisterFlags(f *flag.FlagSet) { "GRPC listen address of the query-frontend(s). Must be a DNS address (prefixed with dns:///) "+ "to enable client side load balancing.") - f.BoolVar(&c.TLSEnabled, "ruler.query-frontend.tls-enabled", false, "Set to true if query-frontend connection requires TLS.") - - c.TLS.RegisterFlagsWithPrefix("ruler.query-frontend", f) + c.GRPCClientConfig.RegisterFlagsWithPrefix("ruler.query-frontend.grpc-client-config", f) } // DialQueryFrontend creates and initializes a new httpgrpc.HTTPClient taking a QueryFrontendConfig configuration. func DialQueryFrontend(cfg QueryFrontendConfig) (httpgrpc.HTTPClient, error) { - tlsDialOptions, err := cfg.TLS.GetGRPCDialOptions(cfg.TLSEnabled) + opts, err := cfg.GRPCClientConfig.DialOption([]grpc.UnaryClientInterceptor{ + otgrpc.OpenTracingClientInterceptor(opentracing.GlobalTracer()), + middleware.ClientUserHeaderInterceptor, + }, nil) if err != nil { return nil, err } - dialOptions := append( - []grpc.DialOption{ - grpc.WithKeepaliveParams( - keepalive.ClientParameters{ - Time: keepAlive, - Timeout: keepAliveTimeout, - PermitWithoutStream: true, - }, - ), - grpc.WithUnaryInterceptor( - grpc_middleware.ChainUnaryClient( - otgrpc.OpenTracingClientInterceptor(opentracing.GlobalTracer()), - middleware.ClientUserHeaderInterceptor, - ), - ), - grpc.WithDefaultServiceConfig(serviceConfig), - }, - tlsDialOptions..., - ) + opts = append(opts, grpc.WithDefaultServiceConfig(serviceConfig)) - conn, err := grpc.Dial(cfg.Address, dialOptions...) + conn, err := grpc.Dial(cfg.Address, opts...) if err != nil { return nil, err }