[Ruler][DNS] Don't propagate no such host error if using default resolver

CHANGELOG.md

@@ -11,7 +11,10 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 
 ## Unreleased
 
+
+### Fixed
 - [#3261](https://github.com/thanos-io/thanos/pull/3261) Thanos Store: Use segment files specified in meta.json file, if present. If not present, Store does the LIST operation as before.
+- [#3257](https://github.com/thanos-io/thanos/pull/3257) Ruler: Prevent Ruler from crashing when using default DNS to lookup hosts that results in "No such hosts" errors.
 
 ## [v0.16.0](https://github.com/thanos-io/thanos/releases) - Release in progress
 

cmd/thanos/query.go

@@ -261,10 +261,11 @@ func runQuery(
 	}
 
 	fileSDCache := cache.New()
-	dnsStoreProvider := dns.NewProvider(
+	dnsStoreProvider := dns.NewProviderWithReturnOnErrorIfNotFound(
 		logger,
 		extprom.WrapRegistererWithPrefix("thanos_querier_store_apis_", reg),
 		dns.ResolverType(dnsSDResolver),
+		true,
 	)
 
 	for _, store := range strictStores {
@@ -273,10 +274,11 @@ func runQuery(
 		}
 	}
 
-	dnsRuleProvider := dns.NewProvider(
+	dnsRuleProvider := dns.NewProviderWithReturnOnErrorIfNotFound(
 		logger,
 		extprom.WrapRegistererWithPrefix("thanos_querier_rule_apis_", reg),
 		dns.ResolverType(dnsSDResolver),
+		true,
 	)
 
 	var (

cmd/thanos/rule.go

@@ -334,10 +334,11 @@ func runRule(
 		}
 	}
 
-	queryProvider := dns.NewProvider(
+	queryProvider := dns.NewProviderWithReturnOnErrorIfNotFound(
 		logger,
 		extprom.WrapRegistererWithPrefix("thanos_ruler_query_apis_", reg),
 		dns.ResolverType(dnsSDResolver),
+		true,
 	)
 	var queryClients []*http_util.Client
 	for _, cfg := range queryCfg {
@@ -397,10 +398,11 @@ func runRule(
 		level.Warn(logger).Log("msg", "no alertmanager configured")
 	}
 
-	amProvider := dns.NewProvider(
+	amProvider := dns.NewProviderWithReturnOnErrorIfNotFound(
 		logger,
 		extprom.WrapRegistererWithPrefix("thanos_ruler_alertmanagers_", reg),
 		dns.ResolverType(dnsSDResolver),
+		false,
 	)
 	var alertmgrs []*alert.Alertmanager
 	for _, cfg := range alertingCfg.Alertmanagers {

pkg/cacheutil/memcached_client.go

@@ -215,10 +215,11 @@ func newMemcachedClient(
 	config MemcachedClientConfig,
 	reg prometheus.Registerer,
 ) (*memcachedClient, error) {
-	dnsProvider := dns.NewProvider(
+	dnsProvider := dns.NewProviderWithReturnOnErrorIfNotFound(
 		logger,
 		extprom.WrapRegistererWithPrefix("thanos_memcached_", reg),
 		dns.GolangResolverType,
+		true,
 	)
 
 	c := &memcachedClient{

pkg/discovery/dns/provider.go

@@ -53,11 +53,38 @@ func (t ResolverType) ToResolver(logger log.Logger) ipLookupResolver {
 	return r
 }
 
+// Deprecated. Use NewProviderWithReturnOnErrorIfNotFound instead.
+//
 // NewProvider returns a new empty provider with a given resolver type.
 // If empty resolver type is net.DefaultResolver.
+// TODO(OGKevin): Refactor code to use new method and eventually rename NewProviderWithReturnOnErrorIfNotFound back to NewProvider.
 func NewProvider(logger log.Logger, reg prometheus.Registerer, resolverType ResolverType) *Provider {
 	p := &Provider{
-		resolver: NewResolver(resolverType.ToResolver(logger)),
+		resolver: NewResolver(resolverType.ToResolver(logger), logger, true),
+		resolved: make(map[string][]string),
+		logger:   logger,
+		resolverAddrs: extprom.NewTxGaugeVec(reg, prometheus.GaugeOpts{
+			Name: "dns_provider_results",
+			Help: "The number of resolved endpoints for each configured address",
+		}, []string{"addr"}),
+		resolverLookupsCount: promauto.With(reg).NewCounter(prometheus.CounterOpts{
+			Name: "dns_lookups_total",
+			Help: "The number of DNS lookups resolutions attempts",
+		}),
+		resolverFailuresCount: promauto.With(reg).NewCounter(prometheus.CounterOpts{
+			Name: "dns_failures_total",
+			Help: "The number of DNS lookup failures",
+		}),
+	}
+
+	return p
+}
+
+// NewProviderWithReturnOnErrorIfNotFound returns a new empty provider with a given resolver type.
+// If empty resolver type is net.DefaultResolver.
+func NewProviderWithReturnOnErrorIfNotFound(logger log.Logger, reg prometheus.Registerer, resolverType ResolverType, returnErrOnNotFound bool) *Provider {
+	p := &Provider{
+		resolver: NewResolver(resolverType.ToResolver(logger), logger, returnErrOnNotFound),
 		resolved: make(map[string][]string),
 		logger:   logger,
 		resolverAddrs: extprom.NewTxGaugeVec(reg, prometheus.GaugeOpts{

pkg/discovery/dns/provider_test.go

@@ -22,7 +22,7 @@ func TestProvider(t *testing.T) {
 		"127.0.0.5:19095",
 	}
 
-	prv := NewProvider(log.NewNopLogger(), nil, "")
+	prv := NewProviderWithReturnOnErrorIfNotFound(log.NewNopLogger(), nil, "", true)
 	prv.resolver = &mockResolver{
 		res: map[string][]string{
 			"a": ips[:2],

pkg/discovery/dns/resolver.go

@@ -9,6 +9,9 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/go-kit/kit/log"
+	"github.com/go-kit/kit/log/level"
+
 	"github.com/pkg/errors"
 )
 
@@ -38,11 +41,15 @@ type ipLookupResolver interface {
 
 type dnsSD struct {
 	resolver ipLookupResolver
+	logger   log.Logger
+	// https://github.com/thanos-io/thanos/issues/3186
+	// This flag is used to prevent components from crashing if hosts are not found.
+	returnErrOnNotFound bool
 }
 
 // NewResolver creates a resolver with given underlying resolver.
-func NewResolver(resolver ipLookupResolver) Resolver {
-	return &dnsSD{resolver: resolver}
+func NewResolver(resolver ipLookupResolver, logger log.Logger, returnErrOnNotFound bool) Resolver {
+	return &dnsSD{resolver: resolver, logger: logger, returnErrOnNotFound: returnErrOnNotFound}
 }
 
 func (s *dnsSD) Resolve(ctx context.Context, name string, qtype QType) ([]string, error) {
@@ -71,7 +78,14 @@ func (s *dnsSD) Resolve(ctx context.Context, name string, qtype QType) ([]string
 		}
 		ips, err := s.resolver.LookupIPAddr(ctx, host)
 		if err != nil {
-			return nil, errors.Wrapf(err, "lookup IP addresses %q", host)
+			dnsErr, ok := err.(*net.DNSError)
+			// https://github.com/thanos-io/thanos/issues/3186
+			// Default DNS resolver can make thanos components crash if DSN resolutions results in EAI_NONAME.
+			// the flag returnErrOnNotFound can be used to prevent such crash.
+			if !(!s.returnErrOnNotFound && ok && dnsErr.IsNotFound) {
+				return nil, errors.Wrapf(err, "lookup IP addresses %q", host)
+			}
+			level.Error(s.logger).Log("msg", "failed to lookup IP addresses", "host", host, "err", err)
 		}
 		for _, ip := range ips {
 			res = append(res, appendScheme(scheme, net.JoinHostPort(ip.String(), port)))
@@ -106,6 +120,12 @@ func (s *dnsSD) Resolve(ctx context.Context, name string, qtype QType) ([]string
 		return nil, errors.Errorf("invalid lookup scheme %q", qtype)
 	}
 
+	// https://github.com/thanos-io/thanos/issues/3186
+	// This happens when miekg is used as resolver. When the host cannot be found, nothing is returned.
+	if res == nil && err == nil {
+		level.Warn(s.logger).Log("msg", "IP address lookup yielded no results nor errors", "host", host)
+	}
+
 	return res, nil
 }
 

pkg/discovery/dns/resolver_test.go

@@ -9,6 +9,8 @@ import (
 	"sort"
 	"testing"
 
+	"github.com/go-kit/kit/log"
+
 	"github.com/pkg/errors"
 	"github.com/thanos-io/thanos/pkg/testutil"
 )
@@ -184,7 +186,7 @@ func TestDnsSD_Resolve(t *testing.T) {
 
 func testDnsSd(t *testing.T, tt DNSSDTest) {
 	ctx := context.TODO()
-	dnsSD := dnsSD{tt.resolver}
+	dnsSD := dnsSD{tt.resolver, log.NewNopLogger(), false}
 
 	result, err := dnsSD.Resolve(ctx, tt.addr, tt.qtype)
 	if tt.expectedErr != nil {