implement only monitoring dns blocklists, without using them for inco…

…ming deliveries

so you can still know when someone has put you on their blocklist (which may
affect delivery), without using them.

also query dnsbls for our ips more often when we do more outgoing connections
for delivery: once every 100 messages, but at least 5 mins and at most 3 hours
since the previous check.

config/config.go

@@ -111,8 +111,10 @@ type Dynamic struct {
 	WebDomainRedirects map[string]string  `sconf:"optional" sconf-doc:"Redirect all requests from domain (key) to domain (value). Always redirects to HTTPS. For plain HTTP redirects, use a WebHandler with a WebRedirect."`
 	WebHandlers        []WebHandler       `sconf:"optional" sconf-doc:"Handle webserver requests by serving static files, redirecting or reverse-proxying HTTP(s). The first matching WebHandler will handle the request. Built-in handlers, e.g. for account, admin, autoconfig and mta-sts always run first. If no handler matches, the response status code is file not found (404). If functionality you need is missng, simply forward the requests to an application that can provide the needed functionality."`
 	Routes             []Route            `sconf:"optional" sconf-doc:"Routes for delivering outgoing messages through the queue. Each delivery attempt evaluates account routes, domain routes and finally these global routes. The transport of the first matching route is used in the delivery attempt. If no routes match, which is the default with no configured routes, messages are delivered directly from the queue."`
+	MonitorDNSBLs      []string           `sconf:"optional" sconf-doc:"DNS blocklists to periodically check with if IPs we send from are present, without using them for checking incoming deliveries.. Also see DNSBLs in SMTP listeners in mox.conf, which specifies DNSBLs to use both for incoming deliveries and for checking our IPs against. Example DNSBLs: sbl.spamhaus.org, bl.spamcop.net."`
 
 	WebDNSDomainRedirects map[dns.Domain]dns.Domain `sconf:"-" json:"-"`
+	MonitorDNSBLZones     []dns.Domain              `sconf:"-"`
 }
 
 type ACME struct {
@@ -150,7 +152,7 @@ type Listener struct {
 		// Reoriginated messages (such as messages sent to mailing list subscribers) should
 		// keep REQUIRETLS. ../rfc/8689:412
 
-		DNSBLs []string `sconf:"optional" sconf-doc:"Addresses of DNS block lists for incoming messages. Block lists are only consulted for connections/messages without enough reputation to make an accept/reject decision. This prevents sending IPs of all communications to the block list provider. If any of the listed DNSBLs contains a requested IP address, the message is rejected as spam. The DNSBLs are checked for healthiness before use, at most once per 4 hours. Example DNSBLs: sbl.spamhaus.org, bl.spamcop.net. See https://www.spamhaus.org/sbl/ and https://www.spamcop.net/ for more information and terms of use."`
+		DNSBLs []string `sconf:"optional" sconf-doc:"Addresses of DNS block lists for incoming messages. Block lists are only consulted for connections/messages without enough reputation to make an accept/reject decision. This prevents sending IPs of all communications to the block list provider. If any of the listed DNSBLs contains a requested IP address, the message is rejected as spam. The DNSBLs are checked for healthiness before use, at most once per 4 hours. IPs we can send from are periodically checked for being in the configured DNSBLs. See MonitorDNSBLs in domains.conf to only monitor IPs we send from, without using those DNSBLs for incoming messages. Example DNSBLs: sbl.spamhaus.org, bl.spamcop.net. See https://www.spamhaus.org/sbl/ and https://www.spamcop.net/ for more information and terms of use."`
 
 		FirstTimeSenderDelay *time.Duration `sconf:"optional" sconf-doc:"Delay before accepting a message from a first-time sender for the destination account. Default: 15s."`
 

config/doc.go

@@ -239,9 +239,12 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 				# accept/reject decision. This prevents sending IPs of all communications to the
 				# block list provider. If any of the listed DNSBLs contains a requested IP
 				# address, the message is rejected as spam. The DNSBLs are checked for healthiness
-				# before use, at most once per 4 hours. Example DNSBLs: sbl.spamhaus.org,
-				# bl.spamcop.net. See https://www.spamhaus.org/sbl/ and https://www.spamcop.net/
-				# for more information and terms of use. (optional)
+				# before use, at most once per 4 hours. IPs we can send from are periodically
+				# checked for being in the configured DNSBLs. See MonitorDNSBLs in domains.conf to
+				# only monitor IPs we send from, without using those DNSBLs for incoming messages.
+				# Example DNSBLs: sbl.spamhaus.org, bl.spamcop.net. See
+				# https://www.spamhaus.org/sbl/ and https://www.spamcop.net/ for more information
+				# and terms of use. (optional)
 				DNSBLs:
 					-
 
@@ -1198,6 +1201,14 @@ See https://pkg.go.dev/github.com/mjl-/sconf for details.
 			MinimumAttempts: 0
 			Transport:
 
+	# DNS blocklists to periodically check with if IPs we send from are present,
+	# without using them for checking incoming deliveries.. Also see DNSBLs in SMTP
+	# listeners in mox.conf, which specifies DNSBLs to use both for incoming
+	# deliveries and for checking our IPs against. Example DNSBLs: sbl.spamhaus.org,
+	# bl.spamcop.net. (optional)
+	MonitorDNSBLs:
+		-
+
 # Examples
 
 Mox includes configuration files to illustrate common setups. You can see these

mox-/admin.go

@@ -1069,6 +1069,35 @@ func AccountLimitsSave(ctx context.Context, account string, maxOutgoingMessagesP
 	return nil
 }
 
+func MonitorDNSBLsSave(ctx context.Context, zones []dns.Domain) (rerr error) {
+	log := pkglog.WithContext(ctx)
+	defer func() {
+		if rerr != nil {
+			log.Errorx("saving monitor dnsbl zones", rerr)
+		}
+	}()
+
+	Conf.dynamicMutex.Lock()
+	defer Conf.dynamicMutex.Unlock()
+
+	c := Conf.Dynamic
+
+	// Compose new config without modifying existing data structures. If we fail, we
+	// leave no trace.
+	nc := c
+	nc.MonitorDNSBLs = make([]string, len(zones))
+	nc.MonitorDNSBLZones = nil
+	for i, z := range zones {
+		nc.MonitorDNSBLs[i] = z.Name()
+	}
+
+	if err := writeDynamic(ctx, log, nc); err != nil {
+		return fmt.Errorf("writing domains.conf: %v", err)
+	}
+	log.Info("monitor dnsbl zones saved")
+	return nil
+}
+
 type TLSMode uint8
 
 const (

mox-/config.go

@@ -24,6 +24,7 @@ import (
 	"os/user"
 	"path/filepath"
 	"regexp"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -242,6 +243,13 @@ func (c *Config) Routes(accountName string, domain dns.Domain) (accountRoutes, d
 	return
 }
 
+func (c *Config) MonitorDNSBLs() (zones []dns.Domain) {
+	c.withDynamicLock(func() {
+		zones = c.Dynamic.MonitorDNSBLZones
+	})
+	return
+}
+
 func (c *Config) allowACMEHosts(log mlog.Log, checkACMEHosts bool) {
 	for _, l := range c.Static.Listeners {
 		if l.TLS == nil || l.TLS.ACME == "" {
@@ -1609,6 +1617,19 @@ func prepareDynamicConfig(ctx context.Context, log mlog.Log, dynamicPath string,
 		}
 	}
 
+	for _, s := range c.MonitorDNSBLs {
+		d, err := dns.ParseDomain(s)
+		if err != nil {
+			addErrorf("invalid monitor dnsbl zone %s: %v", s, err)
+			continue
+		}
+		if slices.Contains(c.MonitorDNSBLZones, d) {
+			addErrorf("duplicate zone %s in monitor dnsbl zones", d)
+			continue
+		}
+		c.MonitorDNSBLZones = append(c.MonitorDNSBLZones, d)
+	}
+
 	return
 }
 

queue/direct.go

@@ -10,6 +10,7 @@ import (
 	"net"
 	"os"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -30,6 +31,10 @@ import (
 	"github.com/mjl-/mox/tlsrpt"
 )
 
+// Increased each time an outgoing connection is made for direct delivery. Used by
+// dnsbl monitoring to pace querying.
+var connectionCounter atomic.Int64
+
 var (
 	metricDestinations = promauto.NewCounter(
 		prometheus.CounterOpts{
@@ -88,6 +93,10 @@ var (
 	)
 )
 
+func ConnectionCounter() int64 {
+	return connectionCounter.Load()
+}
+
 // todo: rename function, perhaps put some of the params in a delivery struct so we don't pass all the params all the time?
 func fail(ctx context.Context, qlog mlog.Log, m Msg, backoff time.Duration, permanent bool, remoteMTA dsn.NameIP, secodeOpt, errmsg, firstLine string, moreLines []string) {
 	// todo future: when we implement relaying, we should be able to send DSNs to non-local users. and possibly specify a null mailfrom. ../rfc/5321:1503
@@ -534,6 +543,7 @@ func deliverHost(log mlog.Log, resolver dns.Resolver, dialer smtpclient.Dialer,
 		if m.DialedIPs == nil {
 			m.DialedIPs = map[string][]net.IP{}
 		}
+		connectionCounter.Add(1)
 		conn, remoteIP, err = smtpclient.Dial(ctx, log.Logger, dialer, host, ips, 25, m.DialedIPs, mox.Conf.Static.SpecifiedSMTPListenIPs)
 	}
 	cancel()

quickstart.go

@@ -708,6 +708,11 @@ and check the admin page for the needed DNS records.`)
 		public.SMTP.DNSBLs = append(public.SMTP.DNSBLs, zone.Name())
 	}
 
+	// Monitor DNSBLs by default, without using them for incoming deliveries.
+	for _, zone := range zones {
+		dc.MonitorDNSBLs = append(dc.MonitorDNSBLs, zone.Name())
+	}
+
 	internal := config.Listener{
 		IPs:      privateListenerIPs,
 		Hostname: "localhost",

serve_unix.go

@@ -14,8 +14,8 @@ import (
 	"path/filepath"
 	"runtime"
 	"runtime/debug"
+	"slices"
 	"strings"
-	"sync"
 	"syscall"
 	"time"
 
@@ -29,10 +29,22 @@ import (
 	"github.com/mjl-/mox/mlog"
 	"github.com/mjl-/mox/mox-"
 	"github.com/mjl-/mox/moxvar"
+	"github.com/mjl-/mox/queue"
 	"github.com/mjl-/mox/store"
 	"github.com/mjl-/mox/updates"
 )
 
+var metricDNSBL = promauto.NewGaugeVec(
+	prometheus.GaugeOpts{
+		Name: "mox_dnsbl_ips_success",
+		Help: "DNSBL lookups to configured DNSBLs of our IPs.",
+	},
+	[]string{
+		"zone",
+		"ip",
+	},
+)
+
 func monitorDNSBL(log mlog.Log) {
 	defer func() {
 		// On error, don't bring down the entire server.
@@ -44,48 +56,71 @@ func monitorDNSBL(log mlog.Log) {
 		}
 	}()
 
-	l, ok := mox.Conf.Static.Listeners["public"]
-	if !ok {
-		log.Info("no listener named public, not monitoring our ips at dnsbls")
-		return
-	}
-
-	var zones []dns.Domain
-	for _, zone := range l.SMTP.DNSBLs {
-		d, err := dns.ParseDomain(zone)
-		if err != nil {
-			log.Fatalx("parsing dnsbls zone", err, slog.Any("zone", zone))
-		}
-		zones = append(zones, d)
-	}
-	if len(zones) == 0 {
-		return
-	}
+	publicListener := mox.Conf.Static.Listeners["public"]
 
+	// We keep track of the previous metric values, so we can delete those we no longer
+	// monitor.
 	type key struct {
 		zone dns.Domain
 		ip   string
 	}
-	metrics := map[key]prometheus.GaugeFunc{}
-	var statusMutex sync.Mutex
-	statuses := map[key]bool{}
+	prevResults := map[key]struct{}{}
+
+	// Last time we checked, and how many outgoing delivery connections were made at that time.
+	var last time.Time
+	var lastConns int64
 
 	resolver := dns.StrictResolver{Pkg: "dnsblmonitor"}
 	var sleep time.Duration // No sleep on first iteration.
 	for {
 		time.Sleep(sleep)
-		sleep = 3 * time.Hour
-
+		// We check more often when we send more. Every 100 messages, and between 5 mins
+		// and 3 hours.
+		conns := queue.ConnectionCounter()
+		if sleep > 0 && conns < lastConns+100 && time.Since(last) < 3*time.Hour {
+			continue
+		}
+		sleep = 5 * time.Minute
+		lastConns = conns
+		last = time.Now()
+
+		// Gather zones.
+		zones := append([]dns.Domain{}, publicListener.SMTP.DNSBLZones...)
+		for _, zone := range mox.Conf.MonitorDNSBLs() {
+			if !slices.Contains(zones, zone) {
+				zones = append(zones, zone)
+			}
+		}
+		// And gather IPs.
 		ips, err := mox.IPs(mox.Context, false)
 		if err != nil {
 			log.Errorx("listing ips for dnsbl monitor", err)
+			// Mark checks as broken.
+			for k := range prevResults {
+				metricDNSBL.WithLabelValues(k.zone.Name(), k.ip).Set(-1)
+			}
 			continue
 		}
+		var publicIPs []net.IP
+		var publicIPstrs []string
 		for _, ip := range ips {
 			if ip.IsLoopback() || ip.IsPrivate() {
 				continue
 			}
+			publicIPs = append(publicIPs, ip)
+			publicIPstrs = append(publicIPstrs, ip.String())
+		}
 
+		// Remove labels that no longer exist from metric.
+		for k := range prevResults {
+			if !slices.Contains(zones, k.zone) || !slices.Contains(publicIPstrs, k.ip) {
+				metricDNSBL.DeleteLabelValues(k.zone.Name(), k.ip)
+				delete(prevResults, k)
+			}
+		}
+
+		// Do DNSBL checks and update metric.
+		for _, ip := range publicIPs {
 			for _, zone := range zones {
 				status, expl, err := dnsbl.Lookup(mox.Context, log.Logger, resolver, zone, ip)
 				if err != nil {
@@ -95,32 +130,14 @@ func monitorDNSBL(log mlog.Log) {
 						slog.String("expl", expl),
 						slog.Any("status", status))
 				}
+				var v float64
+				if status == dnsbl.StatusPass {
+					v = 1
+				}
+				metricDNSBL.WithLabelValues(zone.Name(), ip.String()).Set(v)
 				k := key{zone, ip.String()}
+				prevResults[k] = struct{}{}
 
-				statusMutex.Lock()
-				statuses[k] = status == dnsbl.StatusPass
-				statusMutex.Unlock()
-
-				if _, ok := metrics[k]; !ok {
-					metrics[k] = promauto.NewGaugeFunc(
-						prometheus.GaugeOpts{
-							Name: "mox_dnsbl_ips_success",
-							Help: "DNSBL lookups to configured DNSBLs of our IPs.",
-							ConstLabels: prometheus.Labels{
-								"zone": zone.LogString(),
-								"ip":   k.ip,
-							},
-						},
-						func() float64 {
-							statusMutex.Lock()
-							defer statusMutex.Unlock()
-							if statuses[k] {
-								return 1
-							}
-							return 0
-						},
-					)
-				}
 				time.Sleep(time.Second)
 			}
 		}