rule: prevent rule crash from no such host error

CHANGELOG.md

@@ -22,6 +22,7 @@ We use _breaking :warning:_ to mark changes that are not backward compatible (re
 
 - [#3527](https://github.com/thanos-io/thanos/pull/3527) Query Frontend: Fix query_range behavior when start/end times are the same
 - [#3560](https://github.com/thanos-io/thanos/pull/3560) query-frontend: Allow separate label cache
+- [#3672](https://github.com/thanos-io/thanos/pull/3672) rule: prevent rule crash from no such host error when using `dnssrv+` or `dnssrvnoa+`.
 
 ### Changed
 

pkg/discovery/dns/godns/resolver.go

@@ -0,0 +1,25 @@
+// Copyright (c) The Thanos Authors.
+// Licensed under the Apache License 2.0.
+
+package godns
+
+import (
+	"net"
+
+	"github.com/pkg/errors"
+)
+
+// Resolver is a wrapper for net.Resolver.
+type Resolver struct {
+	*net.Resolver
+}
+
+// IsNotFound checkout if DNS record is not found.
+func (r *Resolver) IsNotFound(err error) bool {
+	if err == nil {
+		return false
+	}
+	err = errors.Cause(err)
+	dnsErr, ok := err.(*net.DNSError)
+	return ok && dnsErr.IsNotFound
+}

pkg/discovery/dns/miekgdns/lookup.go

@@ -11,6 +11,8 @@ import (
 	"github.com/pkg/errors"
 )
 
+var ErrNoSuchHost = errors.New("no such host")
+
 // Copied and slightly adjusted from Prometheus DNS SD:
 // https://github.com/prometheus/prometheus/blob/be3c082539d85908ce03b6d280f83343e7c930eb/discovery/dns/dns.go#L212
 
@@ -68,7 +70,7 @@ func (r *Resolver) lookupWithSearchPath(name string, qtype dns.Type) (*dns.Msg,
 
 	if len(errs) == 0 {
 		// Outcome 2: everyone says NXDOMAIN.
-		return &dns.Msg{}, nil
+		return &dns.Msg{}, ErrNoSuchHost
 	}
 	// Outcome 3: boned.
 	return nil, errors.Errorf("could not resolve %q: all servers responded with errors to at least one search domain. Errs %s", name, fmtErrs(errs))

pkg/discovery/dns/miekgdns/resolver.go

@@ -72,3 +72,7 @@ func (r *Resolver) LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr,
 	}
 	return resp, nil
 }
+
+func (r *Resolver) IsNotFound(err error) bool {
+	return errors.Is(errors.Cause(err), ErrNoSuchHost)
+}

pkg/discovery/dns/provider.go

@@ -14,6 +14,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 
+	"github.com/thanos-io/thanos/pkg/discovery/dns/godns"
 	"github.com/thanos-io/thanos/pkg/discovery/dns/miekgdns"
 	"github.com/thanos-io/thanos/pkg/errutil"
 	"github.com/thanos-io/thanos/pkg/extprom"
@@ -43,12 +44,12 @@ func (t ResolverType) ToResolver(logger log.Logger) ipLookupResolver {
 	var r ipLookupResolver
 	switch t {
 	case GolangResolverType:
-		r = net.DefaultResolver
+		r = &godns.Resolver{Resolver: net.DefaultResolver}
 	case MiekgdnsResolverType:
 		r = &miekgdns.Resolver{ResolvConf: miekgdns.DefaultResolvConfPath}
 	default:
 		level.Warn(logger).Log("msg", "no such resolver type, defaulting to golang", "type", t)
-		r = net.DefaultResolver
+		r = &godns.Resolver{Resolver: net.DefaultResolver}
 	}
 	return r
 }

pkg/discovery/dns/resolver.go

@@ -37,6 +37,7 @@ type Resolver interface {
 type ipLookupResolver interface {
 	LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr, error)
 	LookupSRV(ctx context.Context, service, proto, name string) (cname string, addrs []*net.SRV, err error)
+	IsNotFound(err error) bool
 }
 
 type dnsSD struct {
@@ -78,7 +79,7 @@ func (s *dnsSD) Resolve(ctx context.Context, name string, qtype QType) ([]string
 			// We exclude error from std Golang resolver for the case of the domain (e.g `NXDOMAIN`) not being found by DNS
 			// server. Since `miekg` does not consider this as an error,  when the host cannot be found, empty slice will be
 			// returned.
-			if dnsErr, ok := err.(*net.DNSError); !ok || !dnsErr.IsNotFound {
+			if !s.resolver.IsNotFound(err) {
 				return nil, errors.Wrapf(err, "lookup IP addresses %q", host)
 			}
 			if ips == nil {
@@ -91,7 +92,12 @@ func (s *dnsSD) Resolve(ctx context.Context, name string, qtype QType) ([]string
 	case SRV, SRVNoA:
 		_, recs, err := s.resolver.LookupSRV(ctx, "", "", host)
 		if err != nil {
-			return nil, errors.Wrapf(err, "lookup SRV records %q", host)
+			if !s.resolver.IsNotFound(err) {
+				return nil, errors.Wrapf(err, "lookup SRV records %q", host)
+			}
+			if len(recs) == 0 {
+				level.Error(s.logger).Log("msg", "failed to lookup SRV records", "host", host, "err", err)
+			}
 		}
 
 		for _, rec := range recs {
@@ -108,7 +114,12 @@ func (s *dnsSD) Resolve(ctx context.Context, name string, qtype QType) ([]string
 			// Do A lookup for the domain in SRV answer.
 			resIPs, err := s.resolver.LookupIPAddr(ctx, rec.Target)
 			if err != nil {
-				return nil, errors.Wrapf(err, "look IP addresses %q", rec.Target)
+				if !s.resolver.IsNotFound(err) {
+					return nil, errors.Wrapf(err, "lookup IP addresses %q", host)
+				}
+				if len(resIPs) == 0 {
+					level.Error(s.logger).Log("msg", "failed to lookup IP addresses", "host", host, "err", err)
+				}
 			}
 			for _, resIP := range resIPs {
 				res = append(res, appendScheme(scheme, net.JoinHostPort(resIP.String(), resPort)))

pkg/discovery/dns/resolver_test.go

@@ -35,6 +35,10 @@ func (m mockHostnameResolver) LookupSRV(ctx context.Context, service, proto, nam
 	return "", m.resultSRVs[name], nil
 }
 
+func (m mockHostnameResolver) IsNotFound(err error) bool {
+	return false
+}
+
 type DNSSDTest struct {
 	testName       string
 	addr           string