[tunnel] reduce hot-path allocations in packet processing pipeline

Remove per-packet slog.Debug calls that eagerly evaluated addr.String()
even when debug logging was disabled, inline Geneve header check to avoid
full deserialization on every packet, pre-allocate batch classification
slices in the bifurcator, and use netip.Addr as sync.Map key in
peerMACForIP to avoid ip.String() allocation on every lookup.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: dilyevsky

pkg/tunnel/bifurcate/bifurcate.go

@@ -6,7 +6,6 @@ import (
 	"net"
 	"sync"
 
-	"github.com/apoxy-dev/icx/geneve"
 	"gvisor.dev/gvisor/pkg/tcpip/header"
 
 	"github.com/apoxy-dev/apoxy/pkg/tunnel/batchpc"
@@ -35,6 +34,9 @@ func Bifurcate(pc batchpc.BatchPacketConn) (batchpc.BatchPacketConn, batchpc.Bat
 		msgs := make([]batchpc.Message, batchpc.MaxBatchSize)
 		// Shadow array of pooled message pointers we own & recycle.
 		pm := make([]*batchpc.Message, batchpc.MaxBatchSize)
+		// Pre-allocated classification slices, reused via [:0] each iteration.
+		gBatch := make([]*batchpc.Message, 0, batchpc.MaxBatchSize)
+		oBatch := make([]*batchpc.Message, 0, batchpc.MaxBatchSize)
 
 		for {
 			// If both sides are gone, stop.
@@ -86,9 +88,9 @@ func Bifurcate(pc batchpc.BatchPacketConn) (batchpc.BatchPacketConn, batchpc.Bat
 				continue
 			}
 
-			// Classify into destination batches (slices referencing pooled messages).
-			gBatch := make([]*batchpc.Message, 0, n)
-			oBatch := make([]*batchpc.Message, 0, n)
+			// Classify into destination batches (reset pre-allocated slices).
+			gBatch = gBatch[:0]
+			oBatch = oBatch[:0]
 
 			for i := 0; i < n; i++ {
 				m := pm[i]
@@ -134,24 +136,24 @@ func Bifurcate(pc batchpc.BatchPacketConn) (batchpc.BatchPacketConn, batchpc.Bat
 	return geneveConn, otherConn
 }
 
+// isGeneve performs a fast inline check of the Geneve header without full
+// deserialization. The fixed Geneve header is 8 bytes:
+//
+//	byte 0: version (2 bits) | opt len (6 bits)
+//	byte 1: flags (8 bits)
+//	byte 2-3: protocol type (big-endian)
+//	byte 4-7: VNI (24 bits) | reserved (8 bits)
 func isGeneve(b []byte) bool {
-	var hdr geneve.Header
-	_, err := hdr.UnmarshalBinary(b)
-	if err != nil {
+	if len(b) < 8 {
 		return false
 	}
-
 	// Only Geneve version 0 is defined.
-	if hdr.Version != 0 {
+	if b[0]>>6 != 0 {
 		return false
 	}
-
 	// Check for valid protocol types (IPv4 or IPv6) or EtherType 0 (mgmt / oob).
-	if hdr.ProtocolType != uint16(header.IPv4ProtocolNumber) &&
-		hdr.ProtocolType != uint16(header.IPv6ProtocolNumber) &&
-		hdr.ProtocolType != 0 {
-		return false
-	}
-
-	return true
+	proto := uint16(b[2])<<8 | uint16(b[3])
+	return proto == uint16(header.IPv4ProtocolNumber) ||
+		proto == uint16(header.IPv6ProtocolNumber) ||
+		proto == 0
 }

pkg/tunnel/bifurcate/bifurcate_test.go

@@ -218,6 +218,58 @@ func TestBifurcate_BubblesTransientErrorAndContinues(t *testing.T) {
 	require.True(t, bytes.Equal(buf[:n], genevePkt), "expected geneve packet after transient error")
 }
 
+func TestBifurcate_IsGeneveEdgeCases(t *testing.T) {
+	remote := &net.UDPAddr{IP: net.IPv4(10, 1, 1, 1), Port: 9999}
+
+	tests := []struct {
+		name     string
+		pkt      []byte
+		isGeneve bool
+	}{
+		{"too short (7 bytes)", []byte{0, 0, 0x08, 0x00, 0, 0, 0}, false},
+		{"valid IPv4 proto", makeGeneveHeader(0, uint16(header.IPv4ProtocolNumber)), true},
+		{"valid IPv6 proto", makeGeneveHeader(0, uint16(header.IPv6ProtocolNumber)), true},
+		{"valid proto 0 (mgmt)", makeGeneveHeader(0, 0), true},
+		{"invalid version 1", makeGeneveHeader(1, uint16(header.IPv4ProtocolNumber)), false},
+		{"invalid version 3", makeGeneveHeader(3, uint16(header.IPv4ProtocolNumber)), false},
+		{"unsupported proto 0xFFFF", makeGeneveHeader(0, 0xFFFF), false},
+		{"empty", []byte{}, false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			mockConn := newMockBatchPacketConn()
+			mockConn.enqueue(tt.pkt, remote)
+
+			geneveConn, otherConn := bifurcate.Bifurcate(mockConn)
+			defer geneveConn.Close()
+			defer otherConn.Close()
+
+			buf := make([]byte, 1024)
+			if tt.isGeneve {
+				n, _, err := geneveConn.ReadFrom(buf)
+				require.NoError(t, err)
+				require.Equal(t, tt.pkt, buf[:n])
+			} else {
+				n, _, err := otherConn.ReadFrom(buf)
+				require.NoError(t, err)
+				require.Equal(t, tt.pkt, buf[:n])
+			}
+		})
+	}
+}
+
+// makeGeneveHeader builds a minimal 8-byte Geneve header with given version and protocol type.
+func makeGeneveHeader(version uint8, proto uint16) []byte {
+	buf := make([]byte, 8)
+	buf[0] = version << 6 // version in top 2 bits, optLen=0
+	buf[1] = 0            // flags
+	buf[2] = byte(proto >> 8)
+	buf[3] = byte(proto)
+	// bytes 4-7: VNI + reserved (zeros)
+	return buf
+}
+
 func createGenevePacket(t *testing.T) []byte {
 	h := geneve.Header{
 		Version:      0,

pkg/tunnel/connection/muxed_conn.go

@@ -79,8 +79,6 @@ func (m *muxedConn) readFromConn(src netip.Prefix, conn Connection) {
 			continue
 		}
 
-		slog.Debug("Read packet from connection", slog.Int("bytes", n))
-
 		*pkt = (*pkt)[:n]
 		select {
 		case m.incomingPackets <- pkt:
@@ -242,8 +240,6 @@ func (m *muxedConn) writeToConn(addr netip.Addr, pkt []byte) []byte {
 		return nil
 	}
 
-	slog.Debug("Writing packet to connection", slog.String("ip", addr.String()), slog.String("conn", conn.String()))
-
 	metrics.TunnelPacketsSent.Inc()
 	metrics.TunnelBytesSent.Add(float64(len(pkt)))
 
@@ -284,8 +280,6 @@ func (m *SrcMuxedConn) WritePacket(pkt []byte) ([]byte, error) {
 		return nil, net.ErrClosed
 	}
 
-	slog.Debug("Write packet to connection", slog.Int("bytes", len(pkt)))
-
 	var srcAddr netip.Addr
 	switch pkt[0] >> 4 {
 	case 6:
@@ -310,8 +304,6 @@ func (m *SrcMuxedConn) WritePacket(pkt []byte) ([]byte, error) {
 		return nil, fmt.Errorf("unknown packet type: %d", pkt[0]>>4)
 	}
 
-	slog.Debug("Packet source", slog.String("ip", srcAddr.String()))
-
 	return m.writeToConn(srcAddr, pkt), nil
 }
 
@@ -339,8 +331,6 @@ func (m *DstMuxedConn) WritePacket(pkt []byte) ([]byte, error) {
 		return nil, net.ErrClosed
 	}
 
-	slog.Debug("Write packet to connection", slog.Int("bytes", len(pkt)))
-
 	var dstAddr netip.Addr
 	switch pkt[0] >> 4 {
 	case 6:
@@ -365,7 +355,5 @@ func (m *DstMuxedConn) WritePacket(pkt []byte) ([]byte, error) {
 		return nil, fmt.Errorf("unknown packet type: %d", pkt[0]>>4)
 	}
 
-	slog.Debug("Packet destination", slog.String("ip", dstAddr.String()))
-
 	return m.writeToConn(dstAddr, pkt), nil
 }

pkg/tunnel/connection/splice.go

@@ -110,7 +110,6 @@ func Splice(tunDev tun.Device, conn Connection, opts ...SpliceOption) error {
 
 			for i := 0; i < n; i++ {
 				packetData := pkts[i][:sizes[i]]
-				slog.Debug("Read packet from TUN", slog.Int("len", sizes[i]))
 
 				// Notify observer if set
 				if config.observer != nil {
@@ -167,8 +166,6 @@ func Splice(tunDev tun.Device, conn Connection, opts ...SpliceOption) error {
 					return fmt.Errorf("failed to read from connection: %w", err)
 				}
 
-				slog.Debug("Read packet from connection", slog.Int("len", n))
-
 				// Notify observer if set
 				if config.observer != nil && n > 0 {
 					config.observer.OnPacket(ExtractPacketInfo((*pkt)[tunOffset:tunOffset+n], DirectionInbound))
@@ -217,8 +214,6 @@ func Splice(tunDev tun.Device, conn Connection, opts ...SpliceOption) error {
 					}
 				}
 
-				slog.Debug("Write packet to TUN", slog.Int("batch_size", batchCount))
-
 				if _, err := tunDev.Write(pkts[:batchCount], tunOffset); err != nil {
 					if strings.Contains(err.Error(), "closed") {
 						slog.Debug("TUN device closed")

pkg/tunnel/l2pc/l2pc.go

@@ -250,14 +250,19 @@ func (c *L2PacketConn) LocalMAC() tcpip.LinkAddress {
 // peerMACForIP returns a cached random, locally-administered unicast MAC
 // for the given remote IP, creating it on first use.
 func (c *L2PacketConn) peerMACForIP(ip net.IP) tcpip.LinkAddress {
-	key := ip.String()
-	if v, ok := c.peerMACCache.Load(key); ok {
+	// Use netip.Addr as map key to avoid ip.String() allocation on every lookup.
+	addr, ok := netip.AddrFromSlice(ip)
+	if !ok {
+		// Fallback for malformed IPs — should not happen in practice.
+		addr = netip.Addr{}
+	}
+	if v, ok := c.peerMACCache.Load(addr); ok {
 		return v.(tcpip.LinkAddress)
 	}
 	// Create and publish a new random MAC.
 	newMAC := tcpip.GetRandMacAddr()
 	// Use LoadOrStore to avoid races/duplication.
-	if v, loaded := c.peerMACCache.LoadOrStore(key, newMAC); loaded {
+	if v, loaded := c.peerMACCache.LoadOrStore(addr, newMAC); loaded {
 		return v.(tcpip.LinkAddress)
 	}
 	return newMAC