[tunnel] bump TUN MTU to 1420 and fix Geneve MTU update on reload

Introduce TunnelMTU=1420 (sized for QUIC over 1500-byte paths) and
DefaultGeneveMTU=1450 constants. Fix SetUp to update Geneve device MTU
when it already exists, preventing MTU mismatch after binary reload that
caused IPv6 fragmentation failures (Ip6FragFails).

Author: dilyevsky

pkg/net/lwtunnel/constants.go

@@ -0,0 +1,9 @@
+package lwtunnel
+
+const (
+	// DefaultGeneveMTU is the MTU for Geneve tunnel interfaces. Must be >=
+	// the tunnel TUN MTU (1420) so that overlay packets are not fragmented
+	// on the Geneve leg. Safe to set high since Geneve runs over VPC
+	// networks that support jumbo frames.
+	DefaultGeneveMTU = 1450
+)

pkg/net/lwtunnel/geneve.go

@@ -49,7 +49,7 @@ func defaultGeneveOptions() *geneveOptions {
 		dev:  "gnv0",
 		vni:  0x61,
 		port: 6081,
-		mtu:  1380,
+		mtu:  DefaultGeneveMTU,
 	}
 }
 
@@ -180,6 +180,20 @@ func (r *Geneve) SetUp(_ context.Context, privAddr netip.Addr) error {
 		link = geneve
 	} else if err != nil {
 		return fmt.Errorf("failed to get Geneve interface: %w", err)
+	} else if link.Attrs().MTU != r.opts.mtu {
+		slog.Info("Updating Geneve interface MTU",
+			slog.String("dev", r.opts.dev),
+			slog.Int("old_mtu", link.Attrs().MTU),
+			slog.Int("new_mtu", r.opts.mtu),
+		)
+		if h != nil {
+			err = h.LinkSetMTU(link, r.opts.mtu)
+		} else {
+			err = netlink.LinkSetMTU(link, r.opts.mtu)
+		}
+		if err != nil {
+			return fmt.Errorf("failed to update Geneve MTU: %w", err)
+		}
 	}
 
 	if h != nil {

pkg/net/lwtunnel/lwtunnel_stub.go

@@ -35,7 +35,7 @@ func WithMTU(mtu int) option       { return func(o *geneveOptions) { o.mtu = mtu
 func WithNetNS(ns string) option   { return func(o *geneveOptions) { o.netns = ns } }
 
 func NewGeneve(opts ...option) *Geneve {
-	o := &geneveOptions{dev: "gnv0", vni: 0x61, port: 6081, mtu: 1380}
+	o := &geneveOptions{dev: "gnv0", vni: 0x61, port: 6081, mtu: DefaultGeneveMTU}
 	for _, opt := range opts {
 		opt(o)
 	}

pkg/netstack/tun_device.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/dpeckett/network"
+	"github.com/prometheus/client_golang/prometheus"
 	"golang.zx2c4.com/wireguard/tun"
 
 	"gvisor.dev/gvisor/pkg/buffer"
@@ -30,6 +31,12 @@ import (
 
 const IPv6MinMTU = 1280 // IPv6 minimum MTU, required for some PPPoE links.
 
+// TunnelMTU is the MTU used for tunnel TUN devices. Sized to fit in a single
+// QUIC datagram after PMTUD on a typical 1500-byte internet path:
+// 1500 (Ethernet) - 20 (IP) - 8 (UDP) - ~26 (QUIC framing) - 1 (contextID) ≈ 1445.
+// We use 1420 to leave headroom for path variance.
+const TunnelMTU = 1420
+
 var _ tun.Device = (*TunDevice)(nil)
 
 type TunDevice struct {
@@ -80,18 +87,18 @@ func NewTunDevice(pcapPath string) (*TunDevice, error) {
 
 	// High-performance TCP buffer settings.
 	tcpRcvBuf := tcpip.TCPReceiveBufferSizeRangeOption{
-		Min:     64 << 10,  // 64 KiB
-		Default: 2 << 20,   // 2 MiB
-		Max:     16 << 20,  // 16 MiB
+		Min:     64 << 10, // 64 KiB
+		Default: 2 << 20,  // 2 MiB
+		Max:     16 << 20, // 16 MiB
 	}
 	tcpipErr = ipstack.SetTransportProtocolOption(tcp.ProtocolNumber, &tcpRcvBuf)
 	if tcpipErr != nil {
 		return nil, fmt.Errorf("could not set TCP receive buffer size: %v", tcpipErr)
 	}
 	tcpSndBuf := tcpip.TCPSendBufferSizeRangeOption{
-		Min:     64 << 10,  // 64 KiB
-		Default: 2 << 20,   // 2 MiB
-		Max:     16 << 20,  // 16 MiB
+		Min:     64 << 10, // 64 KiB
+		Default: 2 << 20,  // 2 MiB
+		Max:     16 << 20, // 16 MiB
 	}
 	tcpipErr = ipstack.SetTransportProtocolOption(tcp.ProtocolNumber, &tcpSndBuf)
 	if tcpipErr != nil {
@@ -129,7 +136,7 @@ func NewTunDevice(pcapPath string) (*TunDevice, error) {
 	}
 
 	nicID := ipstack.NextNICID()
-	linkEP := channel.New(4096, uint32(IPv6MinMTU), "")
+	linkEP := channel.New(4096, uint32(TunnelMTU), "")
 	var nicEP stack.LinkEndpoint = linkEP
 
 	var pcapFile *os.File
@@ -361,6 +368,36 @@ func (tun *TunDevice) ListenPacket(addr netip.AddrPort) (net.PacketConn, error)
 	return gonet.DialUDP(tun.stack, fa, nil, protoNum)
 }
 
+// RegisterTCPStatsMetrics registers netstack TCP stats as Prometheus gauges
+// that are read at push/scrape time. Call once after creating the TunDevice.
+func (tun *TunDevice) RegisterTCPStatsMetrics(reg prometheus.Registerer) {
+	s := tun.stack.Stats().TCP
+	gauges := []struct {
+		name string
+		help string
+		fn   func() float64
+	}{
+		{"tunnel_netstack_tcp_segments_sent_total", "TCP segments sent.", func() float64 { return float64(s.SegmentsSent.Value()) }},
+		{"tunnel_netstack_tcp_segments_received_total", "TCP segments received.", func() float64 { return float64(s.ValidSegmentsReceived.Value()) }},
+		{"tunnel_netstack_tcp_retransmits_total", "TCP segments retransmitted.", func() float64 { return float64(s.Retransmits.Value()) }},
+		{"tunnel_netstack_tcp_fast_retransmit_total", "TCP fast retransmits.", func() float64 { return float64(s.FastRetransmit.Value()) }},
+		{"tunnel_netstack_tcp_slow_start_retransmits_total", "TCP slow start retransmits.", func() float64 { return float64(s.SlowStartRetransmits.Value()) }},
+		{"tunnel_netstack_tcp_timeouts_total", "TCP RTO timeouts.", func() float64 { return float64(s.Timeouts.Value()) }},
+		{"tunnel_netstack_tcp_fast_recovery_total", "TCP fast recovery events.", func() float64 { return float64(s.FastRecovery.Value()) }},
+		{"tunnel_netstack_tcp_sack_recovery_total", "TCP SACK recovery events.", func() float64 { return float64(s.SACKRecovery.Value()) }},
+		{"tunnel_netstack_tcp_checksum_errors_total", "TCP checksum errors.", func() float64 { return float64(s.ChecksumErrors.Value()) }},
+		{"tunnel_netstack_tcp_established", "Current established TCP connections.", func() float64 { return float64(s.CurrentEstablished.Value()) }},
+		{"tunnel_netstack_tcp_resets_sent_total", "TCP resets sent.", func() float64 { return float64(s.ResetsSent.Value()) }},
+		{"tunnel_netstack_tcp_resets_received_total", "TCP resets received.", func() float64 { return float64(s.ResetsReceived.Value()) }},
+	}
+	for _, g := range gauges {
+		reg.MustRegister(prometheus.NewGaugeFunc(
+			prometheus.GaugeOpts{Name: g.name, Help: g.help},
+			g.fn,
+		))
+	}
+}
+
 // ForwardTo forwards all inbound traffic to the upstream network.
 func (tun *TunDevice) ForwardTo(ctx context.Context, upstream network.Network) error {
 	// Allow outgoing packets to have a source address different from the address

pkg/tunnel/connection/device.go

@@ -25,7 +25,7 @@ func (d *Device) Close() error {
 }
 
 func (d *Device) MTU() (int, error) {
-	return netstack.IPv6MinMTU, nil
+	return netstack.TunnelMTU, nil
 }
 
 func (d *Device) Name() string {

pkg/tunnel/connection/splice.go

@@ -98,7 +98,7 @@ func Splice(tunDev tun.Device, conn Connection, opts ...SpliceOption) error {
 		sizes := make([]int, batchSize)
 		pkts := make([][]byte, batchSize)
 		for i := range pkts {
-			pkts[i] = make([]byte, netstack.IPv6MinMTU)
+			pkts[i] = make([]byte, netstack.TunnelMTU)
 		}
 
 		for {
@@ -235,7 +235,7 @@ func Splice(tunDev tun.Device, conn Connection, opts ...SpliceOption) error {
 		// Non-zero-copy fallback: use intermediate channel for batching.
 		pktPool := &sync.Pool{
 			New: func() any {
-				return ptr.To(make([]byte, netstack.IPv6MinMTU+tunOffset))
+				return ptr.To(make([]byte, netstack.TunnelMTU+tunOffset))
 			},
 		}
 

pkg/tunnel/proxy/proxy.go

@@ -24,7 +24,7 @@ const (
 	defaultProxyGeneveDev  = "proxy-gnv0"
 	defaultProxyGenevePort = 6082
 	defaultProxyGeneveVNI  = 200
-	defaultProxyGeneveMTU  = 1400
+	defaultProxyGeneveMTU  = lwtunnel.DefaultGeneveMTU
 )
 
 // ProxyTunnelReconciler reconciles Proxy objects and manages L3 Geneve tunnels

pkg/tunnel/router/client_netlink_linux.go

@@ -76,9 +76,9 @@ func newClientNetlinkRouter(opts ...Option) (*ClientNetlinkRouter, error) {
 		opt(options)
 	}
 
-	slog.Info("Create a TUN device", "name", options.tunIfaceName, "mtu", netstack.IPv6MinMTU)
+	slog.Info("Create a TUN device", "name", options.tunIfaceName, "mtu", netstack.TunnelMTU)
 
-	tunDev, err := tun.CreateTUN(options.tunIfaceName, netstack.IPv6MinMTU)
+	tunDev, err := tun.CreateTUN(options.tunIfaceName, netstack.TunnelMTU)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create TUN interface: %w", err)
 	}

pkg/tunnel/router/server_netlink_linux.go

@@ -65,7 +65,7 @@ func NewNetlinkRouter(opts ...Option) (*NetlinkRouter, error) {
 		return nil, fmt.Errorf("failed to get external interface %s: %w", options.extIfaceName, err)
 	}
 
-	tunDev, err := tun.CreateTUN(options.tunIfaceName, netstack.IPv6MinMTU)
+	tunDev, err := tun.CreateTUN(options.tunIfaceName, netstack.TunnelMTU)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create TUN interface: %w", err)
 	}