core: Refactor and improve listener logic (#5089)

* core: Refactor, improve listener logic

Deprecate:
- caddy.Listen
- caddy.ListenTimeout
- caddy.ListenPacket

Prefer caddy.NetworkAddress.Listen() instead.

Change:
- caddy.ListenQUIC (hopefully to remove later)
- caddy.ListenerFunc signature (add context and ListenConfig)

- Don't emit Alt-Svc header advertising h3 over HTTP/3

- Use quic.ListenEarly instead of quic.ListenEarlyAddr; this gives us
more flexibility (e.g. possibility of HTTP/3 over UDS) but also
introduces a new issue:
quic-go/quic-go#3560 (comment)

- Unlink unix socket before and after use

* Appease the linter

* Keep ListenAll

admin.go

@@ -382,7 +382,7 @@ func replaceLocalAdminServer(cfg *Config) error {
 
 	handler := cfg.Admin.newAdminHandler(addr, false)
 
-	ln, err := Listen(addr.Network, addr.JoinHostPort(0))
+	ln, err := addr.Listen(context.TODO(), 0, net.ListenConfig{})
 	if err != nil {
 		return err
 	}
@@ -403,7 +403,7 @@ func replaceLocalAdminServer(cfg *Config) error {
 		serverMu.Lock()
 		server := localAdminServer
 		serverMu.Unlock()
-		if err := server.Serve(ln); !errors.Is(err, http.ErrServerClosed) {
+		if err := server.Serve(ln.(net.Listener)); !errors.Is(err, http.ErrServerClosed) {
 			adminLogger.Error("admin server shutdown for unknown reason", zap.Error(err))
 		}
 	}()
@@ -549,10 +549,11 @@ func replaceRemoteAdminServer(ctx Context, cfg *Config) error {
 	serverMu.Unlock()
 
 	// start listener
-	ln, err := Listen(addr.Network, addr.JoinHostPort(0))
+	lnAny, err := addr.Listen(ctx, 0, net.ListenConfig{})
 	if err != nil {
 		return err
 	}
+	ln := lnAny.(net.Listener)
 	ln = tls.NewListener(ln, tlsConfig)
 
 	go func() {

listen.go

@@ -20,7 +20,7 @@
 package caddy
 
 import (
-	"fmt"
+	"context"
 	"net"
 	"sync"
 	"sync/atomic"
@@ -29,30 +29,22 @@ import (
 	"go.uber.org/zap"
 )
 
-func ListenTimeout(network, addr string, keepAlivePeriod time.Duration) (net.Listener, error) {
-	// check to see if plugin provides listener
-	if ln, err := getListenerFromPlugin(network, addr); err != nil || ln != nil {
-		return ln, err
-	}
-
-	lnKey := listenerKey(network, addr)
+func reuseUnixSocket(network, addr string) (any, error) {
+	return nil, nil
+}
 
+func listenTCPOrUnix(ctx context.Context, lnKey string, network, address string, config net.ListenConfig) (net.Listener, error) {
 	sharedLn, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
-		ln, err := net.Listen(network, addr)
+		ln, err := config.Listen(ctx, network, address)
 		if err != nil {
-			// https://github.com/caddyserver/caddy/pull/4534
-			if isUnixNetwork(network) && isListenBindAddressAlreadyInUseError(err) {
-				return nil, fmt.Errorf("%w: this can happen if Caddy was forcefully killed", err)
-			}
 			return nil, err
 		}
 		return &sharedListener{Listener: ln, key: lnKey}, nil
 	})
 	if err != nil {
 		return nil, err
 	}
-
-	return &fakeCloseListener{sharedListener: sharedLn.(*sharedListener), keepAlivePeriod: keepAlivePeriod}, nil
+	return &fakeCloseListener{sharedListener: sharedLn.(*sharedListener), keepAlivePeriod: config.KeepAlive}, nil
 }
 
 // fakeCloseListener is a private wrapper over a listener that
@@ -156,8 +148,6 @@ func (sl *sharedListener) clearDeadline() error {
 		switch ln := sl.Listener.(type) {
 		case *net.TCPListener:
 			err = ln.SetDeadline(time.Time{})
-		case *net.UnixListener:
-			err = ln.SetDeadline(time.Time{})
 		}
 		sl.deadline = false
 	}
@@ -173,8 +163,6 @@ func (sl *sharedListener) setDeadline() error {
 		switch ln := sl.Listener.(type) {
 		case *net.TCPListener:
 			err = ln.SetDeadline(timeInPast)
-		case *net.UnixListener:
-			err = ln.SetDeadline(timeInPast)
 		}
 		sl.deadline = true
 	}

listen_unix.go

@@ -24,78 +24,88 @@ import (
 	"errors"
 	"io/fs"
 	"net"
-	"sync"
+	"sync/atomic"
 	"syscall"
-	"time"
 
 	"go.uber.org/zap"
 	"golang.org/x/sys/unix"
 )
 
-// ListenTimeout is the same as Listen, but with a configurable keep-alive timeout duration.
-func ListenTimeout(network, addr string, keepalivePeriod time.Duration) (net.Listener, error) {
-	// check to see if plugin provides listener
-	if ln, err := getListenerFromPlugin(network, addr); err != nil || ln != nil {
-		return ln, err
+// reuseUnixSocket copies and reuses the unix domain socket (UDS) if we already
+// have it open; if not, unlink it so we can have it. No-op if not a unix network.
+func reuseUnixSocket(network, addr string) (any, error) {
+	if !isUnixNetwork(network) {
+		return nil, nil
 	}
 
 	socketKey := listenerKey(network, addr)
-	if isUnixNetwork(network) {
-		unixSocketsMu.Lock()
-		defer unixSocketsMu.Unlock()
 
-		socket, exists := unixSockets[socketKey]
-		if exists {
-			// make copy of file descriptor
-			socketFile, err := socket.File() // dup() deep down
+	socket, exists := unixSockets[socketKey]
+	if exists {
+		// make copy of file descriptor
+		socketFile, err := socket.File() // does dup() deep down
+		if err != nil {
+			return nil, err
+		}
+
+		// use copied fd to make new Listener or PacketConn, then replace
+		// it in the map so that future copies always come from the most
+		// recent fd (as the previous ones will be closed, and we'd get
+		// "use of closed network connection" errors) -- note that we
+		// preserve the *pointer* to the counter (not just the value) so
+		// that all socket wrappers will refer to the same value
+		switch unixSocket := socket.(type) {
+		case *unixListener:
+			ln, err := net.FileListener(socketFile)
 			if err != nil {
 				return nil, err
 			}
+			atomic.AddInt32(unixSocket.count, 1)
+			unixSockets[socketKey] = &unixListener{ln.(*net.UnixListener), socketKey, unixSocket.count}
 
-			// use copy to make new listener
-			ln, err := net.FileListener(socketFile)
+		case *unixConn:
+			pc, err := net.FilePacketConn(socketFile)
 			if err != nil {
 				return nil, err
 			}
-
-			// the old socket fd will likely be closed soon, so replace it in the map
-			unixSockets[socketKey] = ln.(*net.UnixListener)
-
-			return ln.(*net.UnixListener), nil
+			atomic.AddInt32(unixSocket.count, 1)
+			unixSockets[socketKey] = &unixConn{pc.(*net.UnixConn), addr, socketKey, unixSocket.count}
 		}
 
-		// from what I can tell after some quick research, it's quite common for programs to
-		// leave their socket file behind after they close, so the typical pattern is to
-		// unlink it before you bind to it -- this is often crucial if the last program using
-		// it was killed forcefully without a chance to clean up the socket, but there is a
-		// race, as the comment in net.UnixListener.close() explains... oh well?
-		if err := syscall.Unlink(addr); err != nil && !errors.Is(err, fs.ErrNotExist) {
-			return nil, err
-		}
+		return unixSockets[socketKey], nil
 	}
 
-	config := &net.ListenConfig{Control: reusePort, KeepAlive: keepalivePeriod}
-
-	ln, err := config.Listen(context.Background(), network, addr)
-	if err != nil {
+	// from what I can tell after some quick research, it's quite common for programs to
+	// leave their socket file behind after they close, so the typical pattern is to
+	// unlink it before you bind to it -- this is often crucial if the last program using
+	// it was killed forcefully without a chance to clean up the socket, but there is a
+	// race, as the comment in net.UnixListener.close() explains... oh well, I guess?
+	if err := syscall.Unlink(addr); err != nil && !errors.Is(err, fs.ErrNotExist) {
 		return nil, err
 	}
 
-	if uln, ok := ln.(*net.UnixListener); ok {
-		// TODO: ideally, we should unlink the socket once we know we're done using it
-		// (i.e. either on exit or a new config that doesn't use this socket; in UsagePool
-		// terms, when the reference count reaches 0), but given that we unlink existing
-		// socket before we create the new one anyway (see above), we don't necessarily
-		// need to clean up after ourselves; still, doing so would probably be more tidy
-		uln.SetUnlinkOnClose(false)
-		unixSockets[socketKey] = uln
-	}
+	return nil, nil
+}
 
-	return ln, nil
+func listenTCPOrUnix(ctx context.Context, lnKey string, network, address string, config net.ListenConfig) (net.Listener, error) {
+	// wrap any Control function set by the user so we can also add our reusePort control without clobbering theirs
+	oldControl := config.Control
+	config.Control = func(network, address string, c syscall.RawConn) error {
+		if oldControl != nil {
+			if err := oldControl(network, address, c); err != nil {
+				return err
+			}
+		}
+		return reusePort(network, address, c)
+	}
+	return config.Listen(ctx, network, address)
 }
 
 // reusePort sets SO_REUSEPORT. Ineffective for unix sockets.
 func reusePort(network, address string, conn syscall.RawConn) error {
+	if isUnixNetwork(network) {
+		return nil
+	}
 	return conn.Control(func(descriptor uintptr) {
 		if err := unix.SetsockoptInt(int(descriptor), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil {
 			Log().Error("setting SO_REUSEPORT",
@@ -106,10 +116,3 @@ func reusePort(network, address string, conn syscall.RawConn) error {
 		}
 	})
 }
-
-// unixSockets keeps track of the currently-active unix sockets
-// so we can transfer their FDs gracefully during reloads.
-var (
-	unixSockets   = make(map[string]*net.UnixListener)
-	unixSocketsMu sync.Mutex
-)