Skip to content

Commit

Permalink
Distributor: Support for forwarding request via httpgrpc (#2996)
Browse files Browse the repository at this point in the history 
Signed-off-by: Peter Štibraný <pstibrany@gmail.com>
Co-authored-by: Mauro Stettler <mauro.stettler@gmail.com>
  • Loading branch information
@pstibrany @replay
pstibrany and replay committed Sep 22, 2022
1 parent 85ba77c commit 12db6c0
Show file tree
Hide file tree
Showing 7 changed files with 640 additions and 28 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
* [ENHANCEMENT] Query-frontend: Add `cortex_frontend_query_result_cache_skipped_total` and `cortex_frontend_query_result_cache_attempted_total` metrics to track the reason why query results are not cached. #2855
* [ENHANCEMENT] Distributor: pool more connections per host when forwarding request. Mark requests as idempotent so they can be retried under some conditions. #2968
* [ENHANCEMENT] Distributor: failure to send request to forwarding target now also increments `cortex_distributor_forward_errors_total`, with `status_code="failed"`. #2968
* [ENHANCEMENT] Distributor: added support forwarding push requests via gRPC, using `httpgrpc` messages from weaveworks/common library. #2996
* [BUGFIX] Querier: Fix 400 response while handling streaming remote read. #2963
* [BUGFIX] Fix a bug causing query-frontend, query-scheduler, and querier not failing if one of their internal components fail. #2978

Expand Down
185 changes: 185 additions & 0 deletions cmd/mimir/config-descriptor.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1398,6 +1398,191 @@
"fieldFlag": "distributor.forwarding.propagate-errors",
"fieldType": "boolean",
"fieldCategory": "experimental"
},
{
"kind": "block",
"name": "grpc_client",
"required": false,
"desc": "",
"blockEntries": [
{
"kind": "field",
"name": "max_recv_msg_size",
"required": false,
"desc": "gRPC client max receive message size (bytes).",
"fieldValue": null,
"fieldDefaultValue": 104857600,
"fieldFlag": "distributor.forwarding.grpc-client.grpc-max-recv-msg-size",
"fieldType": "int",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "max_send_msg_size",
"required": false,
"desc": "gRPC client max send message size (bytes).",
"fieldValue": null,
"fieldDefaultValue": 104857600,
"fieldFlag": "distributor.forwarding.grpc-client.grpc-max-send-msg-size",
"fieldType": "int",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "grpc_compression",
"required": false,
"desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "distributor.forwarding.grpc-client.grpc-compression",
"fieldType": "string",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "rate_limit",
"required": false,
"desc": "Rate limit for gRPC client; 0 means disabled.",
"fieldValue": null,
"fieldDefaultValue": 0,
"fieldFlag": "distributor.forwarding.grpc-client.grpc-client-rate-limit",
"fieldType": "float",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "rate_limit_burst",
"required": false,
"desc": "Rate limit burst for gRPC client.",
"fieldValue": null,
"fieldDefaultValue": 0,
"fieldFlag": "distributor.forwarding.grpc-client.grpc-client-rate-limit-burst",
"fieldType": "int",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "backoff_on_ratelimits",
"required": false,
"desc": "Enable backoff and retry when we hit ratelimits.",
"fieldValue": null,
"fieldDefaultValue": false,
"fieldFlag": "distributor.forwarding.grpc-client.backoff-on-ratelimits",
"fieldType": "boolean",
"fieldCategory": "advanced"
},
{
"kind": "block",
"name": "backoff_config",
"required": false,
"desc": "",
"blockEntries": [
{
"kind": "field",
"name": "min_period",
"required": false,
"desc": "Minimum delay when backing off.",
"fieldValue": null,
"fieldDefaultValue": 100000000,
"fieldFlag": "distributor.forwarding.grpc-client.backoff-min-period",
"fieldType": "duration",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "max_period",
"required": false,
"desc": "Maximum delay when backing off.",
"fieldValue": null,
"fieldDefaultValue": 10000000000,
"fieldFlag": "distributor.forwarding.grpc-client.backoff-max-period",
"fieldType": "duration",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "max_retries",
"required": false,
"desc": "Number of times to backoff and retry before failing.",
"fieldValue": null,
"fieldDefaultValue": 10,
"fieldFlag": "distributor.forwarding.grpc-client.backoff-retries",
"fieldType": "int",
"fieldCategory": "advanced"
}
],
"fieldValue": null,
"fieldDefaultValue": null
},
{
"kind": "field",
"name": "tls_enabled",
"required": false,
"desc": "Enable TLS in the GRPC client. This flag needs to be enabled when any other TLS flag is set. If set to false, insecure connection to gRPC server will be used.",
"fieldValue": null,
"fieldDefaultValue": false,
"fieldFlag": "distributor.forwarding.grpc-client.tls-enabled",
"fieldType": "boolean",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_cert_path",
"required": false,
"desc": "Path to the client certificate file, which will be used for authenticating with the server. Also requires the key path to be configured.",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "distributor.forwarding.grpc-client.tls-cert-path",
"fieldType": "string",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_key_path",
"required": false,
"desc": "Path to the key file for the client certificate. Also requires the client certificate to be configured.",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "distributor.forwarding.grpc-client.tls-key-path",
"fieldType": "string",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_ca_path",
"required": false,
"desc": "Path to the CA certificates file to validate server certificate against. If not set, the host's root CA certificates are used.",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "distributor.forwarding.grpc-client.tls-ca-path",
"fieldType": "string",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_server_name",
"required": false,
"desc": "Override the expected name on the server certificate.",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "distributor.forwarding.grpc-client.tls-server-name",
"fieldType": "string",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_insecure_skip_verify",
"required": false,
"desc": "Skip validating server certificate.",
"fieldValue": null,
"fieldDefaultValue": false,
"fieldFlag": "distributor.forwarding.grpc-client.tls-insecure-skip-verify",
"fieldType": "boolean",
"fieldCategory": "advanced"
}
],
"fieldValue": null,
"fieldDefaultValue": null
}
],
"fieldValue": null,
Expand Down
30 changes: 30 additions & 0 deletions cmd/mimir/help-all.txt.tmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -748,6 +748,36 @@ Usage of ./cmd/mimir/mimir:
This flag can be used to specify label names that to drop during sample ingestion within the distributor and can be repeated in order to drop multiple labels.
-distributor.forwarding.enabled
[experimental] Enables the feature to forward certain metrics in remote_write requests, depending on defined rules.
-distributor.forwarding.grpc-client.backoff-max-period duration
Maximum delay when backing off. (default 10s)
-distributor.forwarding.grpc-client.backoff-min-period duration
Minimum delay when backing off. (default 100ms)
-distributor.forwarding.grpc-client.backoff-on-ratelimits
Enable backoff and retry when we hit ratelimits.
-distributor.forwarding.grpc-client.backoff-retries int
Number of times to backoff and retry before failing. (default 10)
-distributor.forwarding.grpc-client.grpc-client-rate-limit float
Rate limit for gRPC client; 0 means disabled.
-distributor.forwarding.grpc-client.grpc-client-rate-limit-burst int
Rate limit burst for gRPC client.
-distributor.forwarding.grpc-client.grpc-compression string
Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)
-distributor.forwarding.grpc-client.grpc-max-recv-msg-size int
gRPC client max receive message size (bytes). (default 104857600)
-distributor.forwarding.grpc-client.grpc-max-send-msg-size int
gRPC client max send message size (bytes). (default 104857600)
-distributor.forwarding.grpc-client.tls-ca-path string
Path to the CA certificates file to validate server certificate against. If not set, the host's root CA certificates are used.
-distributor.forwarding.grpc-client.tls-cert-path string
Path to the client certificate file, which will be used for authenticating with the server. Also requires the key path to be configured.
-distributor.forwarding.grpc-client.tls-enabled
Enable TLS in the GRPC client. This flag needs to be enabled when any other TLS flag is set. If set to false, insecure connection to gRPC server will be used.
-distributor.forwarding.grpc-client.tls-insecure-skip-verify
Skip validating server certificate.
-distributor.forwarding.grpc-client.tls-key-path string
Path to the key file for the client certificate. Also requires the client certificate to be configured.
-distributor.forwarding.grpc-client.tls-server-name string
Override the expected name on the server certificate.
-distributor.forwarding.propagate-errors
[experimental] If disabled then forwarding requests are always considered to be successful, errors are ignored. (default true)
-distributor.forwarding.request-concurrency int
Expand Down
69 changes: 69 additions & 0 deletions docs/sources/operators-guide/configure/reference-configuration-parameters/index.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -633,6 +633,75 @@ forwarding:
# be successful, errors are ignored.
# CLI flag: -distributor.forwarding.propagate-errors
[propagate_errors: <boolean> | default = true]

grpc_client:
# (advanced) gRPC client max receive message size (bytes).
# CLI flag: -distributor.forwarding.grpc-client.grpc-max-recv-msg-size
[max_recv_msg_size: <int> | default = 104857600]

# (advanced) gRPC client max send message size (bytes).
# CLI flag: -distributor.forwarding.grpc-client.grpc-max-send-msg-size
[max_send_msg_size: <int> | default = 104857600]

# (advanced) Use compression when sending messages. Supported values are:
# 'gzip', 'snappy' and '' (disable compression)
# CLI flag: -distributor.forwarding.grpc-client.grpc-compression
[grpc_compression: <string> | default = ""]

# (advanced) Rate limit for gRPC client; 0 means disabled.
# CLI flag: -distributor.forwarding.grpc-client.grpc-client-rate-limit
[rate_limit: <float> | default = 0]

# (advanced) Rate limit burst for gRPC client.
# CLI flag: -distributor.forwarding.grpc-client.grpc-client-rate-limit-burst
[rate_limit_burst: <int> | default = 0]

# (advanced) Enable backoff and retry when we hit ratelimits.
# CLI flag: -distributor.forwarding.grpc-client.backoff-on-ratelimits
[backoff_on_ratelimits: <boolean> | default = false]

backoff_config:
# (advanced) Minimum delay when backing off.
# CLI flag: -distributor.forwarding.grpc-client.backoff-min-period
[min_period: <duration> | default = 100ms]

# (advanced) Maximum delay when backing off.
# CLI flag: -distributor.forwarding.grpc-client.backoff-max-period
[max_period: <duration> | default = 10s]

# (advanced) Number of times to backoff and retry before failing.
# CLI flag: -distributor.forwarding.grpc-client.backoff-retries
[max_retries: <int> | default = 10]

# (advanced) Enable TLS in the GRPC client. This flag needs to be enabled
# when any other TLS flag is set. If set to false, insecure connection to
# gRPC server will be used.
# CLI flag: -distributor.forwarding.grpc-client.tls-enabled
[tls_enabled: <boolean> | default = false]

# (advanced) Path to the client certificate file, which will be used for
# authenticating with the server. Also requires the key path to be
# configured.
# CLI flag: -distributor.forwarding.grpc-client.tls-cert-path
[tls_cert_path: <string> | default = ""]

# (advanced) Path to the key file for the client certificate. Also requires
# the client certificate to be configured.
# CLI flag: -distributor.forwarding.grpc-client.tls-key-path
[tls_key_path: <string> | default = ""]

# (advanced) Path to the CA certificates file to validate server certificate
# against. If not set, the host's root CA certificates are used.
# CLI flag: -distributor.forwarding.grpc-client.tls-ca-path
[tls_ca_path: <string> | default = ""]

# (advanced) Override the expected name on the server certificate.
# CLI flag: -distributor.forwarding.grpc-client.tls-server-name
[tls_server_name: <string> | default = ""]

# (advanced) Skip validating server certificate.
# CLI flag: -distributor.forwarding.grpc-client.tls-insecure-skip-verify
[tls_insecure_skip_verify: <boolean> | default = false]
```

### ingester
Expand Down
5 changes: 5 additions & 0 deletions pkg/distributor/forwarding/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,25 @@ import (
"errors"
"flag"
"time"

"github.com/grafana/dskit/grpcclient"
)

type Config struct {
Enabled bool `yaml:"enabled" category:"experimental"`
RequestConcurrency int `yaml:"request_concurrency" category:"experimental"`
RequestTimeout time.Duration `yaml:"request_timeout" category:"experimental"`
PropagateErrors bool `yaml:"propagate_errors" category:"experimental"`

GRPCClientConfig grpcclient.Config `yaml:"grpc_client"`
}

func (c *Config) RegisterFlags(f *flag.FlagSet) {
f.BoolVar(&c.Enabled, "distributor.forwarding.enabled", false, "Enables the feature to forward certain metrics in remote_write requests, depending on defined rules.")
f.IntVar(&c.RequestConcurrency, "distributor.forwarding.request-concurrency", 10, "Maximum concurrency at which forwarding requests get performed.")
f.DurationVar(&c.RequestTimeout, "distributor.forwarding.request-timeout", 2*time.Second, "Timeout for requests to ingestion endpoints to which we forward metrics.")
f.BoolVar(&c.PropagateErrors, "distributor.forwarding.propagate-errors", true, "If disabled then forwarding requests are always considered to be successful, errors are ignored.")
c.GRPCClientConfig.RegisterFlagsWithPrefix("distributor.forwarding.grpc-client", f)
}

func (c *Config) Validate() error {
Expand Down
Loading

0 comments on commit 12db6c0

Please sign in to comment.