[icxtunnel] get the netlink + AF_XDP router working

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/adrg/xdg v0.5.3
 	github.com/alphadose/haxmap v1.4.1
 	github.com/anatol/vmtest v0.0.0-20250318022921-2f32244e2f0f
-	github.com/apoxy-dev/icx v0.10.0
+	github.com/apoxy-dev/icx v0.11.0
 	github.com/avast/retry-go/v4 v4.6.1
 	github.com/bramvdbogaerde/go-scp v1.5.0
 	github.com/buraksezer/olric v0.5.6

go.sum

@@ -123,8 +123,8 @@ github.com/apoxy-dev/apiserver-runtime v0.0.0-20250420214109-979c605051d1 h1:sAS
 github.com/apoxy-dev/apiserver-runtime v0.0.0-20250420214109-979c605051d1/go.mod h1:zOVeivsnCWenmbgr6kiefIExoqlbuv2xyg9SXXfbs5U=
 github.com/apoxy-dev/connect-ip-go v0.0.0-20250530062404-603929a73f45 h1:SwPk1n/oSVX7YwlNpC9KNH9YaYkcL/k6OfqSGVnxyyI=
 github.com/apoxy-dev/connect-ip-go v0.0.0-20250530062404-603929a73f45/go.mod h1:z5rtgIizc+/K27UtB0occwZgqg/mz3IqgyUJW8aubbI=
-github.com/apoxy-dev/icx v0.10.0 h1:Jgb1a+uPtV7a0s+QnvP3Pv2k2oTLpwqmxOqltBGHiIE=
-github.com/apoxy-dev/icx v0.10.0/go.mod h1:Muuk3bRXTp3YB5Xj+xHOGQ/T1xVxIKJuvmMfLBXhIN4=
+github.com/apoxy-dev/icx v0.11.0 h1:/HFcCoPHyjBp2/+wZ09BM3hyQ4zh8mdIe/4CG61JWQk=
+github.com/apoxy-dev/icx v0.11.0/go.mod h1:Muuk3bRXTp3YB5Xj+xHOGQ/T1xVxIKJuvmMfLBXhIN4=
 github.com/apoxy-dev/quic-go v0.0.0-20250530165952-53cca597715e h1:10GIpiVyKoRgCyr0J2TvJtdn17bsFHN+ROWkeVJpcOU=
 github.com/apoxy-dev/quic-go v0.0.0-20250530165952-53cca597715e/go.mod h1:MFlGGpcpJqRAfmYi6NC2cptDPSxRWTOGNuP4wqrWmzQ=
 github.com/apoxy-dev/upgrade-cli v0.0.0-20240213232412-a56c3a52fa0e h1:FBNxMQD93z2ththupB/BYKLEaMWaEr+G+sJWJqU2wC4=

pkg/cmd/alpha/tunnel_relay.go

@@ -30,6 +30,7 @@ var (
 	listenAddress        string // the address to listen on for incoming connections
 	userMode             bool   // whether to use user-mode routing (no special privileges required)
 	relaySocksListenAddr string // when using user-mode routing, the address to listen on for SOCKS5 connections
+	relayPcapPath        string // optional pcap path
 )
 
 var tunnelRelayCmd = &cobra.Command{
@@ -43,6 +44,10 @@ var tunnelRelayCmd = &cobra.Command{
 			router.WithSocksListenAddr(relaySocksListenAddr), // only used in user-mode
 		}
 
+		if relayPcapPath != "" {
+			routerOpts = append(routerOpts, router.WithPcapPath(relayPcapPath))
+		}
+
 		// One UDP socket shared between Geneve (data) and QUIC (control).
 		pc, err := net.ListenPacket("udp", listenAddress)
 		if err != nil {
@@ -128,7 +133,7 @@ var tunnelRelayCmd = &cobra.Command{
 				slog.String("agent", agentName), slog.String("connID", conn.ID()),
 				slog.Int("vni", int(vni)))
 
-			if err := conn.SetVNI(vni); err != nil {
+			if err := conn.SetVNI(cmd.Context(), vni); err != nil {
 				return fmt.Errorf("failed to set VNI on connection: %w", err)
 			}
 
@@ -172,6 +177,7 @@ func init() {
 	tunnelRelayCmd.Flags().StringVar(&listenAddress, "listen-addr", "127.0.0.1:6081", "The address to listen on for incoming connections.")
 	tunnelRelayCmd.Flags().BoolVar(&userMode, "user-mode", runtime.GOOS != "linux", "Use user-mode routing (no special privileges required).")
 	tunnelRelayCmd.Flags().StringVar(&relaySocksListenAddr, "socks-addr", "localhost:1080", "When using user-mode routing, the address to listen on for SOCKS5 connections.")
+	tunnelRelayCmd.Flags().StringVarP(&relayPcapPath, "pcap", "p", "", "Path to an optional packet capture file to write.")
 
 	tunnelCmd.AddCommand(tunnelRelayCmd)
 }

pkg/cmd/alpha/tunnel_run_test.go

@@ -35,7 +35,7 @@ func TestTunnelRun(t *testing.T) {
 	// onConnect assigns VNI and overlay address so handleConnect can proceed.
 	onConnect := func(ctx context.Context, agent string, conn controllers.Connection) error {
 		// Choose a deterministic VNI for the test.
-		conn.SetVNI(101)
+		conn.SetVNI(ctx, 101)
 		conn.SetOverlayAddress("10.0.0.2/32")
 		t.Logf("onConnect called, agent=%s", agent)
 		if agent == "test-agent" {

pkg/netstack/icx_network.go

@@ -31,6 +31,7 @@ import (
 	"github.com/apoxy-dev/apoxy/pkg/tunnel/l2pc"
 )
 
+// TODO (dpeckett): nuke this at some point and merge the logic into the router.
 type ICXNetwork struct {
 	network.Network
 	handler        *icx.Handler

pkg/tunnel/connection.go

@@ -1,10 +1,12 @@
 package tunnel
 
 import (
+	"context"
 	"fmt"
 	"net/netip"
 	"sync"
 	"sync/atomic"
+	"time"
 
 	"github.com/apoxy-dev/apoxy/pkg/netstack"
 	"github.com/apoxy-dev/apoxy/pkg/tunnel/controllers"
@@ -15,6 +17,7 @@ import (
 var _ controllers.Connection = (*connection)(nil)
 
 // connection is a connection like abstraction over an icx virtual network.
+// TODO (dpeckett): nuke this at some point and merge the logic into the router.
 type connection struct {
 	mu          sync.Mutex
 	id          string
@@ -65,7 +68,7 @@ func (c *connection) VNI() *uint {
 }
 
 // Set the VNI assigned to this connection.
-func (c *connection) SetVNI(vni uint) error {
+func (c *connection) SetVNI(ctx context.Context, vni uint) error {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -88,7 +91,22 @@ func (c *connection) SetVNI(vni uint) error {
 		addrs = []netip.Prefix{*c.overlayAddr}
 	}
 
-	if err := c.handler.AddVirtualNetwork(vni, netstack.ToFullAddress(c.remoteAddr), addrs); err != nil {
+	fa := netstack.ToFullAddress(c.remoteAddr)
+
+	// If using the netlink router, we need to resolve the MAC address of the peer.
+	rtr, ok := c.router.(*router.ICXNetlinkRouter)
+	if ok {
+		ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+		defer cancel()
+
+		var err error
+		fa.LinkAddr, err = rtr.ResolveMAC(ctx, c.remoteAddr)
+		if err != nil {
+			return fmt.Errorf("failed to resolve peer MAC address: %w", err)
+		}
+	}
+
+	if err := c.handler.AddVirtualNetwork(vni, fa, addrs); err != nil {
 		return fmt.Errorf("failed to add virtual network %d: %w", vni, err)
 	}
 	c.vni = &vni

pkg/tunnel/controllers/connection.go

@@ -1,6 +1,9 @@
 package controllers
 
-import "io"
+import (
+	"context"
+	"io"
+)
 
 // Connection is a simple abstraction representing a connection from a TunnelAgent to a Relay.
 type Connection interface {
@@ -10,5 +13,5 @@ type Connection interface {
 	// Set the overlay address/prefix assigned to this connection.
 	SetOverlayAddress(addr string) error
 	// Set the VNI assigned to this connection.
-	SetVNI(vni uint) error
+	SetVNI(ctx context.Context, vni uint) error
 }

pkg/tunnel/controllers/tunnel_agent_reconciler.go

@@ -84,7 +84,7 @@ func (r *TunnelAgentReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 			}
 
 			if sc.VNI != nil {
-				if err := conn.SetVNI(uint(*sc.VNI)); err != nil {
+				if err := conn.SetVNI(ctx, uint(*sc.VNI)); err != nil {
 					return ctrl.Result{}, fmt.Errorf("failed to set VNI for connection %q: %w", sc.ID, err)
 				}
 			}

pkg/tunnel/controllers/tunnel_agent_reconciler_test.go

@@ -1,6 +1,7 @@
 package controllers_test
 
 import (
+	"context"
 	"net/netip"
 	"testing"
 
@@ -244,8 +245,8 @@ func (m *mockConn) SetOverlayAddress(addr string) error {
 	return args.Error(0)
 }
 
-func (m *mockConn) SetVNI(v uint) error {
-	args := m.Called(v)
+func (m *mockConn) SetVNI(ctx context.Context, v uint) error {
+	args := m.Called(ctx, v)
 	return args.Error(0)
 }
 

pkg/tunnel/relay_test.go

@@ -34,7 +34,7 @@ func TestRelay_Connect_UpdateKeys_Disconnect(t *testing.T) {
 	// onConnect assigns VNI and overlay address so handleConnect can proceed.
 	onConnect := func(ctx context.Context, agent string, conn controllers.Connection) error {
 		// Choose a deterministic VNI for the test.
-		conn.SetVNI(101)
+		conn.SetVNI(ctx, 101)
 		conn.SetOverlayAddress("10.0.0.2/32")
 		return nil
 	}

pkg/tunnel/router/icx_netlink.go

@@ -15,40 +15,44 @@ import (
 	"github.com/apoxy-dev/apoxy/pkg/tunnel/connection"
 )
 
-func NewICXNetlinkRouter(_ ...Option) (*ICXNotImplementedRouter, error) {
+func NewICXNetlinkRouter(_ ...Option) (*ICXNetlinkRouter, error) {
 	h, err := icx.NewHandler(icx.WithLocalAddr(netstack.ToFullAddress(netip.MustParseAddrPort("127.0.0.1:6081"))),
 		icx.WithVirtMAC(tcpip.GetRandMacAddr()))
 	if err != nil {
 		return nil, fmt.Errorf("failed to create ICX handler: %w", err)
 	}
 
-	return &ICXNotImplementedRouter{Handler: h}, nil
+	return &ICXNetlinkRouter{Handler: h}, nil
 }
 
-type ICXNotImplementedRouter struct {
+type ICXNetlinkRouter struct {
 	Handler *icx.Handler
 }
 
-func (r *ICXNotImplementedRouter) Start(ctx context.Context) error {
+func (r *ICXNetlinkRouter) Start(ctx context.Context) error {
 	return errors.New("not implemented")
 }
 
-func (r *ICXNotImplementedRouter) AddAddr(_ netip.Prefix, _ connection.Connection) error {
+func (r *ICXNetlinkRouter) AddAddr(_ netip.Prefix, _ connection.Connection) error {
 	return errors.New("not implemented")
 }
 
-func (r *ICXNotImplementedRouter) DelAddr(_ netip.Prefix) error {
+func (r *ICXNetlinkRouter) DelAddr(_ netip.Prefix) error {
 	return errors.New("not implemented")
 }
 
-func (r *ICXNotImplementedRouter) AddRoute(dst netip.Prefix) error {
+func (r *ICXNetlinkRouter) AddRoute(dst netip.Prefix) error {
 	return errors.New("not implemented")
 }
 
-func (r *ICXNotImplementedRouter) DelRoute(dst netip.Prefix) error {
+func (r *ICXNetlinkRouter) DelRoute(dst netip.Prefix) error {
 	return errors.New("not implemented")
 }
 
-func (r *ICXNotImplementedRouter) Close() error {
+func (r *ICXNetlinkRouter) Close() error {
 	return nil
 }
+
+func (r *ICXNetlinkRouter) ResolveMAC(ctx context.Context, addr netip.AddrPort) (tcpip.LinkAddress, error) {
+	return "", errors.New("not implemented")
+}

pkg/tunnel/router/icx_netlink_linux.go

@@ -13,7 +13,9 @@ import (
 	"sync"
 
 	"github.com/apoxy-dev/icx"
+	"github.com/apoxy-dev/icx/addrselect"
 	"github.com/apoxy-dev/icx/filter"
+	"github.com/apoxy-dev/icx/mac"
 	"github.com/apoxy-dev/icx/tunnel"
 	"github.com/apoxy-dev/icx/veth"
 	"github.com/google/gopacket/layers"
@@ -47,6 +49,7 @@ type ICXNetlinkRouter struct {
 	pcapFile      *os.File
 	tun           *tunnel.Tunnel
 	iptV4, iptV6  utiliptables.Interface
+	extAddrs      addrselect.AddressList
 	closeOnce     sync.Once
 }
 
@@ -108,12 +111,12 @@ func NewICXNetlinkRouter(opts ...Option) (*ICXNetlinkRouter, error) {
 	}
 
 	for _, addr := range extAddrs {
-		ua, ok := addr.(*net.UDPAddr)
-		if !ok {
-			continue
-		}
+		fa := netstack.ToFullAddress(netip.MustParseAddrPort(addr.String()))
+		fa.LinkAddr = tcpip.LinkAddress(extLink.Attrs().HardwareAddr)
+
 		handlerOpts = append(handlerOpts,
-			icx.WithLocalAddr(netstack.ToFullAddress(netip.MustParseAddrPort(ua.String()))))
+			icx.WithLocalAddr(fa),
+		)
 	}
 
 	if options.sourcePortHashing {
@@ -161,6 +164,11 @@ func NewICXNetlinkRouter(opts ...Option) (*ICXNetlinkRouter, error) {
 		return nil, fmt.Errorf("failed to create tunnel: %w", err)
 	}
 
+	var extAddrsList addrselect.AddressList
+	for _, addr := range extAddrs {
+		extAddrsList = append(extAddrsList, netstack.ToFullAddress(netip.MustParseAddrPort(addr.String())))
+	}
+
 	return &ICXNetlinkRouter{
 		Handler:       handler,
 		extLink:       extLink,
@@ -171,6 +179,7 @@ func NewICXNetlinkRouter(opts ...Option) (*ICXNetlinkRouter, error) {
 		tun:           tun,
 		iptV4:         utiliptables.New(utilexec.New(), utiliptables.ProtocolIPv4),
 		iptV6:         utiliptables.New(utilexec.New(), utiliptables.ProtocolIPv6),
+		extAddrs:      extAddrsList,
 	}, nil
 }
 
@@ -299,6 +308,31 @@ func (r *ICXNetlinkRouter) DelRoute(dst netip.Prefix) error {
 	return nil
 }
 
+// ResolveMAC resolves the MAC address for the given peer address.
+func (r *ICXNetlinkRouter) ResolveMAC(ctx context.Context, peerAddr netip.AddrPort) (tcpip.LinkAddress, error) {
+	peerFullAddr := netstack.ToFullAddress(peerAddr)
+
+	localFullAddr := r.extAddrs.Pick(peerFullAddr)
+
+	slog.Debug("Resolving MAC address",
+		slog.String("local", localFullAddr.Addr.String()),
+		slog.String("peer", peerFullAddr.Addr.String()),
+	)
+
+	linkAddr, err := mac.Resolve(ctx, r.extLink, localFullAddr, peerFullAddr.Addr)
+	if err != nil {
+		return "", fmt.Errorf("failed to resolve peer MAC address: %w", err)
+	}
+
+	slog.Info("Resolved peer MAC address",
+		slog.String("local", localFullAddr.Addr.String()),
+		slog.String("peer", peerFullAddr.Addr.String()),
+		slog.String("mac", linkAddr.String()),
+	)
+
+	return linkAddr, nil
+}
+
 func (r *ICXNetlinkRouter) setupDNAT() error {
 	exists, err := r.iptV6.EnsureChain(utiliptables.TableNAT, ChainA3yTunRules)
 	if err != nil {