From c5cd79939e9be7659c73f623d1fb7bc9a47fd976 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Sun, 11 Jun 2023 06:23:08 +0200 Subject: [PATCH] Report gRPC connnection errors to the caller (#6428) By default `grpc.DialContext()` is non-blocking so any connection issue will not be surfaced to the user. This change makes it blocking and configures the gRPC dialer to report the underlying error if any happens. Signed-off-by: Simon Pasquier --- CHANGELOG.md | 1 + pkg/query/endpointset.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 882458922e8..e7b9526195b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#6342](https://github.com/thanos-io/thanos/pull/6342) Cache/Redis: Upgrade `rueidis` to v1.0.2 to to improve error handling while shrinking a redis cluster. - [#6325](https://github.com/thanos-io/thanos/pull/6325) Store: return gRPC resource exhausted error for byte limiter. - [#6399](https://github.com/thanos-io/thanos/pull/6399) *: Fix double-counting bug in http_request_duration metric +- [#6428](https://github.com/thanos-io/thanos/pull/6428) Report gRPC connnection errors in the logs. ### Changed - [#6168](https://github.com/thanos-io/thanos/pull/6168) Receiver: Make ketama hashring fail early when configured with number of nodes lower than the replication factor. diff --git a/pkg/query/endpointset.go b/pkg/query/endpointset.go index 8842843b1c7..e48139371c5 100644 --- a/pkg/query/endpointset.go +++ b/pkg/query/endpointset.go @@ -638,6 +638,11 @@ type endpointRef struct { // The call to newEndpointRef will return an error if establishing the channel fails. func (e *EndpointSet) newEndpointRef(ctx context.Context, spec *GRPCEndpointSpec) (*endpointRef, error) { dialOpts := append(e.dialOpts, spec.dialOpts...) + // By default DialContext is non-blocking which means that any connection + // failure won't be reported/logged. Instead block until the connection is + // successfully established and return the details of the connection error + // if any. + dialOpts = append(dialOpts, grpc.WithReturnConnectionError()) conn, err := grpc.DialContext(ctx, spec.Addr(), dialOpts...) if err != nil { return nil, errors.Wrap(err, "dialing connection")