From 3f8789e50050b52beaa4f152018894ce49e92711 Mon Sep 17 00:00:00 2001 From: Damian Peckett Date: Sat, 11 Oct 2025 14:04:14 +0200 Subject: [PATCH 1/2] [icxtunnel] get the netlink + AF_XDP router working --- go.mod | 2 +- go.sum | 4 +- pkg/cmd/alpha/tunnel_relay.go | 8 +++- pkg/cmd/alpha/tunnel_run_test.go | 2 +- pkg/netstack/icx_network.go | 1 + pkg/tunnel/connection.go | 22 +++++++++- pkg/tunnel/controllers/connection.go | 7 ++- .../controllers/tunnel_agent_reconciler.go | 2 +- .../tunnel_agent_reconciler_test.go | 5 ++- pkg/tunnel/relay_test.go | 2 +- pkg/tunnel/router/icx_netlink_linux.go | 44 ++++++++++++++++--- 11 files changed, 81 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 6395b8a..3f90f1c 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/adrg/xdg v0.5.3 github.com/alphadose/haxmap v1.4.1 github.com/anatol/vmtest v0.0.0-20250318022921-2f32244e2f0f - github.com/apoxy-dev/icx v0.10.0 + github.com/apoxy-dev/icx v0.11.0 github.com/avast/retry-go/v4 v4.6.1 github.com/bramvdbogaerde/go-scp v1.5.0 github.com/buraksezer/olric v0.5.6 diff --git a/go.sum b/go.sum index 6f7c7dd..2d4e71e 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,8 @@ github.com/apoxy-dev/apiserver-runtime v0.0.0-20250420214109-979c605051d1 h1:sAS github.com/apoxy-dev/apiserver-runtime v0.0.0-20250420214109-979c605051d1/go.mod h1:zOVeivsnCWenmbgr6kiefIExoqlbuv2xyg9SXXfbs5U= github.com/apoxy-dev/connect-ip-go v0.0.0-20250530062404-603929a73f45 h1:SwPk1n/oSVX7YwlNpC9KNH9YaYkcL/k6OfqSGVnxyyI= github.com/apoxy-dev/connect-ip-go v0.0.0-20250530062404-603929a73f45/go.mod h1:z5rtgIizc+/K27UtB0occwZgqg/mz3IqgyUJW8aubbI= -github.com/apoxy-dev/icx v0.10.0 h1:Jgb1a+uPtV7a0s+QnvP3Pv2k2oTLpwqmxOqltBGHiIE= -github.com/apoxy-dev/icx v0.10.0/go.mod h1:Muuk3bRXTp3YB5Xj+xHOGQ/T1xVxIKJuvmMfLBXhIN4= +github.com/apoxy-dev/icx v0.11.0 h1:/HFcCoPHyjBp2/+wZ09BM3hyQ4zh8mdIe/4CG61JWQk= +github.com/apoxy-dev/icx v0.11.0/go.mod h1:Muuk3bRXTp3YB5Xj+xHOGQ/T1xVxIKJuvmMfLBXhIN4= github.com/apoxy-dev/quic-go v0.0.0-20250530165952-53cca597715e h1:10GIpiVyKoRgCyr0J2TvJtdn17bsFHN+ROWkeVJpcOU= github.com/apoxy-dev/quic-go v0.0.0-20250530165952-53cca597715e/go.mod h1:MFlGGpcpJqRAfmYi6NC2cptDPSxRWTOGNuP4wqrWmzQ= github.com/apoxy-dev/upgrade-cli v0.0.0-20240213232412-a56c3a52fa0e h1:FBNxMQD93z2ththupB/BYKLEaMWaEr+G+sJWJqU2wC4= diff --git a/pkg/cmd/alpha/tunnel_relay.go b/pkg/cmd/alpha/tunnel_relay.go index d8073e6..b51209f 100644 --- a/pkg/cmd/alpha/tunnel_relay.go +++ b/pkg/cmd/alpha/tunnel_relay.go @@ -30,6 +30,7 @@ var ( listenAddress string // the address to listen on for incoming connections userMode bool // whether to use user-mode routing (no special privileges required) relaySocksListenAddr string // when using user-mode routing, the address to listen on for SOCKS5 connections + relayPcapPath string // optional pcap path ) var tunnelRelayCmd = &cobra.Command{ @@ -43,6 +44,10 @@ var tunnelRelayCmd = &cobra.Command{ router.WithSocksListenAddr(relaySocksListenAddr), // only used in user-mode } + if relayPcapPath != "" { + routerOpts = append(routerOpts, router.WithPcapPath(relayPcapPath)) + } + // One UDP socket shared between Geneve (data) and QUIC (control). pc, err := net.ListenPacket("udp", listenAddress) if err != nil { @@ -128,7 +133,7 @@ var tunnelRelayCmd = &cobra.Command{ slog.String("agent", agentName), slog.String("connID", conn.ID()), slog.Int("vni", int(vni))) - if err := conn.SetVNI(vni); err != nil { + if err := conn.SetVNI(cmd.Context(), vni); err != nil { return fmt.Errorf("failed to set VNI on connection: %w", err) } @@ -172,6 +177,7 @@ func init() { tunnelRelayCmd.Flags().StringVar(&listenAddress, "listen-addr", "127.0.0.1:6081", "The address to listen on for incoming connections.") tunnelRelayCmd.Flags().BoolVar(&userMode, "user-mode", runtime.GOOS != "linux", "Use user-mode routing (no special privileges required).") tunnelRelayCmd.Flags().StringVar(&relaySocksListenAddr, "socks-addr", "localhost:1080", "When using user-mode routing, the address to listen on for SOCKS5 connections.") + tunnelRelayCmd.Flags().StringVarP(&relayPcapPath, "pcap", "p", "", "Path to an optional packet capture file to write.") tunnelCmd.AddCommand(tunnelRelayCmd) } diff --git a/pkg/cmd/alpha/tunnel_run_test.go b/pkg/cmd/alpha/tunnel_run_test.go index 96e7fec..9e8165c 100644 --- a/pkg/cmd/alpha/tunnel_run_test.go +++ b/pkg/cmd/alpha/tunnel_run_test.go @@ -35,7 +35,7 @@ func TestTunnelRun(t *testing.T) { // onConnect assigns VNI and overlay address so handleConnect can proceed. onConnect := func(ctx context.Context, agent string, conn controllers.Connection) error { // Choose a deterministic VNI for the test. - conn.SetVNI(101) + conn.SetVNI(ctx, 101) conn.SetOverlayAddress("10.0.0.2/32") t.Logf("onConnect called, agent=%s", agent) if agent == "test-agent" { diff --git a/pkg/netstack/icx_network.go b/pkg/netstack/icx_network.go index ead2f91..2880360 100644 --- a/pkg/netstack/icx_network.go +++ b/pkg/netstack/icx_network.go @@ -31,6 +31,7 @@ import ( "github.com/apoxy-dev/apoxy/pkg/tunnel/l2pc" ) +// TODO (dpeckett): nuke this at some point and merge the logic into the router. type ICXNetwork struct { network.Network handler *icx.Handler diff --git a/pkg/tunnel/connection.go b/pkg/tunnel/connection.go index bc3544c..85eb196 100644 --- a/pkg/tunnel/connection.go +++ b/pkg/tunnel/connection.go @@ -1,10 +1,12 @@ package tunnel import ( + "context" "fmt" "net/netip" "sync" "sync/atomic" + "time" "github.com/apoxy-dev/apoxy/pkg/netstack" "github.com/apoxy-dev/apoxy/pkg/tunnel/controllers" @@ -15,6 +17,7 @@ import ( var _ controllers.Connection = (*connection)(nil) // connection is a connection like abstraction over an icx virtual network. +// TODO (dpeckett): nuke this at some point and merge the logic into the router. type connection struct { mu sync.Mutex id string @@ -65,7 +68,7 @@ func (c *connection) VNI() *uint { } // Set the VNI assigned to this connection. -func (c *connection) SetVNI(vni uint) error { +func (c *connection) SetVNI(ctx context.Context, vni uint) error { c.mu.Lock() defer c.mu.Unlock() @@ -88,7 +91,22 @@ func (c *connection) SetVNI(vni uint) error { addrs = []netip.Prefix{*c.overlayAddr} } - if err := c.handler.AddVirtualNetwork(vni, netstack.ToFullAddress(c.remoteAddr), addrs); err != nil { + fa := netstack.ToFullAddress(c.remoteAddr) + + // If using the netlink router, we need to resolve the MAC address of the peer. + rtr, ok := c.router.(*router.ICXNetlinkRouter) + if ok { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + var err error + fa.LinkAddr, err = rtr.ResolveMAC(ctx, c.remoteAddr) + if err != nil { + return fmt.Errorf("failed to resolve peer MAC address: %w", err) + } + } + + if err := c.handler.AddVirtualNetwork(vni, fa, addrs); err != nil { return fmt.Errorf("failed to add virtual network %d: %w", vni, err) } c.vni = &vni diff --git a/pkg/tunnel/controllers/connection.go b/pkg/tunnel/controllers/connection.go index 37044d2..35743a0 100644 --- a/pkg/tunnel/controllers/connection.go +++ b/pkg/tunnel/controllers/connection.go @@ -1,6 +1,9 @@ package controllers -import "io" +import ( + "context" + "io" +) // Connection is a simple abstraction representing a connection from a TunnelAgent to a Relay. type Connection interface { @@ -10,5 +13,5 @@ type Connection interface { // Set the overlay address/prefix assigned to this connection. SetOverlayAddress(addr string) error // Set the VNI assigned to this connection. - SetVNI(vni uint) error + SetVNI(ctx context.Context, vni uint) error } diff --git a/pkg/tunnel/controllers/tunnel_agent_reconciler.go b/pkg/tunnel/controllers/tunnel_agent_reconciler.go index 543fc81..7390bac 100644 --- a/pkg/tunnel/controllers/tunnel_agent_reconciler.go +++ b/pkg/tunnel/controllers/tunnel_agent_reconciler.go @@ -84,7 +84,7 @@ func (r *TunnelAgentReconciler) Reconcile(ctx context.Context, req ctrl.Request) } if sc.VNI != nil { - if err := conn.SetVNI(uint(*sc.VNI)); err != nil { + if err := conn.SetVNI(ctx, uint(*sc.VNI)); err != nil { return ctrl.Result{}, fmt.Errorf("failed to set VNI for connection %q: %w", sc.ID, err) } } diff --git a/pkg/tunnel/controllers/tunnel_agent_reconciler_test.go b/pkg/tunnel/controllers/tunnel_agent_reconciler_test.go index 9241f29..c091e24 100644 --- a/pkg/tunnel/controllers/tunnel_agent_reconciler_test.go +++ b/pkg/tunnel/controllers/tunnel_agent_reconciler_test.go @@ -1,6 +1,7 @@ package controllers_test import ( + "context" "net/netip" "testing" @@ -244,8 +245,8 @@ func (m *mockConn) SetOverlayAddress(addr string) error { return args.Error(0) } -func (m *mockConn) SetVNI(v uint) error { - args := m.Called(v) +func (m *mockConn) SetVNI(ctx context.Context, v uint) error { + args := m.Called(ctx, v) return args.Error(0) } diff --git a/pkg/tunnel/relay_test.go b/pkg/tunnel/relay_test.go index 5786717..0713ec0 100644 --- a/pkg/tunnel/relay_test.go +++ b/pkg/tunnel/relay_test.go @@ -34,7 +34,7 @@ func TestRelay_Connect_UpdateKeys_Disconnect(t *testing.T) { // onConnect assigns VNI and overlay address so handleConnect can proceed. onConnect := func(ctx context.Context, agent string, conn controllers.Connection) error { // Choose a deterministic VNI for the test. - conn.SetVNI(101) + conn.SetVNI(ctx, 101) conn.SetOverlayAddress("10.0.0.2/32") return nil } diff --git a/pkg/tunnel/router/icx_netlink_linux.go b/pkg/tunnel/router/icx_netlink_linux.go index cdf5d57..7cf7059 100644 --- a/pkg/tunnel/router/icx_netlink_linux.go +++ b/pkg/tunnel/router/icx_netlink_linux.go @@ -13,7 +13,9 @@ import ( "sync" "github.com/apoxy-dev/icx" + "github.com/apoxy-dev/icx/addrselect" "github.com/apoxy-dev/icx/filter" + "github.com/apoxy-dev/icx/mac" "github.com/apoxy-dev/icx/tunnel" "github.com/apoxy-dev/icx/veth" "github.com/google/gopacket/layers" @@ -47,6 +49,7 @@ type ICXNetlinkRouter struct { pcapFile *os.File tun *tunnel.Tunnel iptV4, iptV6 utiliptables.Interface + extAddrs addrselect.AddressList closeOnce sync.Once } @@ -108,12 +111,12 @@ func NewICXNetlinkRouter(opts ...Option) (*ICXNetlinkRouter, error) { } for _, addr := range extAddrs { - ua, ok := addr.(*net.UDPAddr) - if !ok { - continue - } + fa := netstack.ToFullAddress(netip.MustParseAddrPort(addr.String())) + fa.LinkAddr = tcpip.LinkAddress(extLink.Attrs().HardwareAddr) + handlerOpts = append(handlerOpts, - icx.WithLocalAddr(netstack.ToFullAddress(netip.MustParseAddrPort(ua.String())))) + icx.WithLocalAddr(fa), + ) } if options.sourcePortHashing { @@ -161,6 +164,11 @@ func NewICXNetlinkRouter(opts ...Option) (*ICXNetlinkRouter, error) { return nil, fmt.Errorf("failed to create tunnel: %w", err) } + var extAddrsList addrselect.AddressList + for _, addr := range extAddrs { + extAddrsList = append(extAddrsList, netstack.ToFullAddress(netip.MustParseAddrPort(addr.String()))) + } + return &ICXNetlinkRouter{ Handler: handler, extLink: extLink, @@ -171,6 +179,7 @@ func NewICXNetlinkRouter(opts ...Option) (*ICXNetlinkRouter, error) { tun: tun, iptV4: utiliptables.New(utilexec.New(), utiliptables.ProtocolIPv4), iptV6: utiliptables.New(utilexec.New(), utiliptables.ProtocolIPv6), + extAddrs: extAddrsList, }, nil } @@ -299,6 +308,31 @@ func (r *ICXNetlinkRouter) DelRoute(dst netip.Prefix) error { return nil } +// ResolveMAC resolves the MAC address for the given peer address. +func (r *ICXNetlinkRouter) ResolveMAC(ctx context.Context, peerAddr netip.AddrPort) (tcpip.LinkAddress, error) { + peerFullAddr := netstack.ToFullAddress(peerAddr) + + localFullAddr := r.extAddrs.Pick(peerFullAddr) + + slog.Debug("Resolving MAC address", + slog.String("local", localFullAddr.Addr.String()), + slog.String("peer", peerFullAddr.Addr.String()), + ) + + linkAddr, err := mac.Resolve(ctx, r.extLink, localFullAddr, peerFullAddr.Addr) + if err != nil { + return "", fmt.Errorf("failed to resolve peer MAC address: %w", err) + } + + slog.Info("Resolved peer MAC address", + slog.String("local", localFullAddr.Addr.String()), + slog.String("peer", peerFullAddr.Addr.String()), + slog.String("mac", linkAddr.String()), + ) + + return linkAddr, nil +} + func (r *ICXNetlinkRouter) setupDNAT() error { exists, err := r.iptV6.EnsureChain(utiliptables.TableNAT, ChainA3yTunRules) if err != nil { From b80af3ca54285182004f19939df169133c65aae9 Mon Sep 17 00:00:00 2001 From: Damian Peckett Date: Sat, 11 Oct 2025 14:08:53 +0200 Subject: [PATCH 2/2] [icxtunnel] fix build on darwin --- pkg/tunnel/router/icx_netlink.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/pkg/tunnel/router/icx_netlink.go b/pkg/tunnel/router/icx_netlink.go index 81719fb..34e926f 100644 --- a/pkg/tunnel/router/icx_netlink.go +++ b/pkg/tunnel/router/icx_netlink.go @@ -15,40 +15,44 @@ import ( "github.com/apoxy-dev/apoxy/pkg/tunnel/connection" ) -func NewICXNetlinkRouter(_ ...Option) (*ICXNotImplementedRouter, error) { +func NewICXNetlinkRouter(_ ...Option) (*ICXNetlinkRouter, error) { h, err := icx.NewHandler(icx.WithLocalAddr(netstack.ToFullAddress(netip.MustParseAddrPort("127.0.0.1:6081"))), icx.WithVirtMAC(tcpip.GetRandMacAddr())) if err != nil { return nil, fmt.Errorf("failed to create ICX handler: %w", err) } - return &ICXNotImplementedRouter{Handler: h}, nil + return &ICXNetlinkRouter{Handler: h}, nil } -type ICXNotImplementedRouter struct { +type ICXNetlinkRouter struct { Handler *icx.Handler } -func (r *ICXNotImplementedRouter) Start(ctx context.Context) error { +func (r *ICXNetlinkRouter) Start(ctx context.Context) error { return errors.New("not implemented") } -func (r *ICXNotImplementedRouter) AddAddr(_ netip.Prefix, _ connection.Connection) error { +func (r *ICXNetlinkRouter) AddAddr(_ netip.Prefix, _ connection.Connection) error { return errors.New("not implemented") } -func (r *ICXNotImplementedRouter) DelAddr(_ netip.Prefix) error { +func (r *ICXNetlinkRouter) DelAddr(_ netip.Prefix) error { return errors.New("not implemented") } -func (r *ICXNotImplementedRouter) AddRoute(dst netip.Prefix) error { +func (r *ICXNetlinkRouter) AddRoute(dst netip.Prefix) error { return errors.New("not implemented") } -func (r *ICXNotImplementedRouter) DelRoute(dst netip.Prefix) error { +func (r *ICXNetlinkRouter) DelRoute(dst netip.Prefix) error { return errors.New("not implemented") } -func (r *ICXNotImplementedRouter) Close() error { +func (r *ICXNetlinkRouter) Close() error { return nil } + +func (r *ICXNetlinkRouter) ResolveMAC(ctx context.Context, addr netip.AddrPort) (tcpip.LinkAddress, error) { + return "", errors.New("not implemented") +}