[icxtunnel] add usermode router for relays and get end-to-end working

cmd/backplane/main.go

@@ -263,7 +263,7 @@ func main() {
 	var (
 		extIPv4Prefix netip.Prefix
 	)
-	extAddrs, err := tunnet.GetGlobalUnicastAddresses(*extIface)
+	extAddrs, err := tunnet.GetGlobalUnicastAddresses(*extIface, false)
 	if err != nil {
 		log.Warnf("Failed to get local IPv6 address: %v", err)
 	} else {

cmd/tunnelproxy/main.go

@@ -119,7 +119,7 @@ func main() {
 		extIPv4Prefix netip.Prefix
 		extIPv6Prefix netip.Prefix
 	)
-	extAddrs, err := tunnet.GetGlobalUnicastAddresses(*extIface)
+	extAddrs, err := tunnet.GetGlobalUnicastAddresses(*extIface, false)
 	if err != nil {
 		log.Warnf("Failed to get local IPv6 address: %v", err)
 	} else {

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/adrg/xdg v0.5.3
 	github.com/alphadose/haxmap v1.4.1
 	github.com/anatol/vmtest v0.0.0-20250318022921-2f32244e2f0f
-	github.com/apoxy-dev/icx v0.8.0
+	github.com/apoxy-dev/icx v0.9.0
 	github.com/avast/retry-go/v4 v4.6.1
 	github.com/bramvdbogaerde/go-scp v1.5.0
 	github.com/buraksezer/olric v0.5.6

go.sum

@@ -123,6 +123,8 @@ github.com/apoxy-dev/connect-ip-go v0.0.0-20250530062404-603929a73f45 h1:SwPk1n/
 github.com/apoxy-dev/connect-ip-go v0.0.0-20250530062404-603929a73f45/go.mod h1:z5rtgIizc+/K27UtB0occwZgqg/mz3IqgyUJW8aubbI=
 github.com/apoxy-dev/icx v0.8.0 h1:Aj/LWtFokyBYYFuISFqgbiWBQJpMdIN6vCMa21MIROc=
 github.com/apoxy-dev/icx v0.8.0/go.mod h1:Muuk3bRXTp3YB5Xj+xHOGQ/T1xVxIKJuvmMfLBXhIN4=
+github.com/apoxy-dev/icx v0.9.0 h1:kdRnjfaksrhtLhEt7caVGlpNPJiClvvQ23KJbb5MsY4=
+github.com/apoxy-dev/icx v0.9.0/go.mod h1:Muuk3bRXTp3YB5Xj+xHOGQ/T1xVxIKJuvmMfLBXhIN4=
 github.com/apoxy-dev/quic-go v0.0.0-20250530165952-53cca597715e h1:10GIpiVyKoRgCyr0J2TvJtdn17bsFHN+ROWkeVJpcOU=
 github.com/apoxy-dev/quic-go v0.0.0-20250530165952-53cca597715e/go.mod h1:MFlGGpcpJqRAfmYi6NC2cptDPSxRWTOGNuP4wqrWmzQ=
 github.com/apoxy-dev/upgrade-cli v0.0.0-20240213232412-a56c3a52fa0e h1:FBNxMQD93z2ththupB/BYKLEaMWaEr+G+sJWJqU2wC4=

pkg/cmd/alpha/tunnel_relay.go

@@ -1,5 +1,3 @@
-//go:build linux
-
 package alpha
 
 import (
@@ -9,24 +7,29 @@ import (
 	"log/slog"
 	"net"
 	"net/netip"
+	"runtime"
 
 	"github.com/alphadose/haxmap"
 	"github.com/spf13/cobra"
 
 	"github.com/apoxy-dev/apoxy/pkg/cryptoutils"
 	"github.com/apoxy-dev/apoxy/pkg/tunnel"
+	"github.com/apoxy-dev/apoxy/pkg/tunnel/bifurcate"
 	"github.com/apoxy-dev/apoxy/pkg/tunnel/controllers"
 	"github.com/apoxy-dev/apoxy/pkg/tunnel/hasher"
 	tunnet "github.com/apoxy-dev/apoxy/pkg/tunnel/net"
 	"github.com/apoxy-dev/apoxy/pkg/tunnel/router"
 	"github.com/apoxy-dev/apoxy/pkg/tunnel/vni"
+	"github.com/apoxy-dev/icx"
 )
 
 var (
-	relayName     string // the name for the relay
-	relayTunnel   string // the name of the tunnel to serve
-	extIfaceName  string // the external interface name
-	listenAddress string // the address to listen on for incoming connections
+	relayName            string // the name for the relay
+	relayTunnel          string // the name of the tunnel to serve
+	extIfaceName         string // the external interface name
+	listenAddress        string // the address to listen on for incoming connections
+	userMode             bool   // whether to use user-mode routing (no special privileges required)
+	relaySocksListenAddr string // when using user-mode routing, the address to listen on for SOCKS5 connections
 )
 
 var tunnelRelayCmd = &cobra.Command{
@@ -36,31 +39,54 @@ var tunnelRelayCmd = &cobra.Command{
 	RunE: func(cmd *cobra.Command, args []string) error {
 		routerOpts := []router.Option{
 			router.WithExternalInterface(extIfaceName),
+			router.WithEgressGateway(true),
+			router.WithSocksListenAddr(relaySocksListenAddr), // only used in user-mode
 		}
 
-		rtr, err := router.NewICXNetlinkRouter(routerOpts...)
-		if err != nil {
-			return fmt.Errorf("failed to create router: %w", err)
-		}
-
+		// One UDP socket shared between Geneve (data) and QUIC (control).
 		pc, err := net.ListenPacket("udp", listenAddress)
 		if err != nil {
 			return fmt.Errorf("failed to create UDP listener: %w", err)
 		}
 
+		pcGeneve, pcQuic := bifurcate.Bifurcate(pc)
+		defer pcGeneve.Close()
+		defer pcQuic.Close()
+
+		var rtr router.Router
+		var handler *icx.Handler
+		if userMode {
+			routerOpts = append(routerOpts, router.WithPacketConn(pcGeneve))
+
+			r, err := router.NewICXNetstackRouter(routerOpts...)
+			if err != nil {
+				return fmt.Errorf("failed to create router: %w", err)
+			}
+			rtr = r
+			handler = r.Handler
+		} else {
+			r, err := router.NewICXNetlinkRouter(routerOpts...)
+			if err != nil {
+				return fmt.Errorf("failed to create router: %w", err)
+			}
+			rtr = r
+			handler = r.Handler
+		}
+
 		idHasher := hasher.NewHasher([]byte("C0rr3ct-Horse-Battery-Staple_But_Salty_1x9Q7p3Z"))
 
 		_, cert, err := cryptoutils.GenerateSelfSignedTLSCert(relayName)
 		if err != nil {
 			return fmt.Errorf("failed to generate self-signed TLS cert: %w", err)
 		}
 
-		relay := tunnel.NewRelay(relayName, pc, cert, rtr.Handler, idHasher, rtr)
+		relay := tunnel.NewRelay(relayName, pcQuic, cert, handler, idHasher, rtr)
 
 		slog.Info("Configuring relay", slog.String("tunnelName", relayTunnel), slog.String("listenAddress", listenAddress), slog.String("externalInterface", extIfaceName))
 
 		relay.SetCredentials(relayTunnel, "letmein")
-		relay.SetRelayAddresses(relayTunnel, []string{pc.LocalAddr().String()})
+		relay.SetRelayAddresses(relayTunnel, []string{pcQuic.LocalAddr().String()})
+		relay.SetEgressGateway(true)
 
 		systemULA := tunnet.NewULA(cmd.Context(), tunnet.SystemNetworkID)
 		agentIPAM, err := systemULA.IPAM(cmd.Context(), 96)
@@ -88,7 +114,10 @@ var tunnelRelayCmd = &cobra.Command{
 				slog.String("agent", agentName), slog.String("connID", conn.ID()),
 				slog.String("prefix", pfx.String()))
 
-			conn.SetOverlayAddress(pfx.String())
+			if err := conn.SetOverlayAddress(pfx.String()); err != nil {
+				agentIPAM.Release(pfx)
+				return fmt.Errorf("failed to set overlay address on connection: %w", err)
+			}
 
 			vni, err := vpool.Allocate()
 			if err != nil {
@@ -140,7 +169,9 @@ func init() {
 	tunnelRelayCmd.Flags().StringVarP(&relayName, "name", "n", "dev", "The name of the relay.")
 	tunnelRelayCmd.Flags().StringVarP(&relayTunnel, "tunnel-name", "t", "dev", "The name of the tunnel to serve.")
 	tunnelRelayCmd.Flags().StringVar(&extIfaceName, "ext-iface", "eth0", "External interface name.")
-	tunnelRelayCmd.Flags().StringVar(&listenAddress, "listen-addr", ":6081", "The address to listen on for incoming connections.")
+	tunnelRelayCmd.Flags().StringVar(&listenAddress, "listen-addr", "127.0.0.1:6081", "The address to listen on for incoming connections.")
+	tunnelRelayCmd.Flags().BoolVar(&userMode, "user-mode", runtime.GOOS != "linux", "Use user-mode routing (no special privileges required).")
+	tunnelRelayCmd.Flags().StringVar(&relaySocksListenAddr, "socks-addr", "localhost:1080", "When using user-mode routing, the address to listen on for SOCKS5 connections.")
 
 	tunnelCmd.AddCommand(tunnelRelayCmd)
 }

pkg/cmd/alpha/tunnel_run.go

@@ -73,7 +73,11 @@ var tunnelRunCmd = &cobra.Command{
 		// Lazily create router/handler on first successful Connect.
 		getHandler := func(connectResp *api.ConnectResponse) (*icx.Handler, error) {
 			routerOnce.Do(func() {
-				var routerOpts []router.Option
+				routerOpts := []router.Option{
+					router.WithPacketConn(pcGeneve),
+					router.WithTunnelMTU(connectResp.MTU),
+				}
+
 				if socksListenAddr != "" {
 					routerOpts = append(routerOpts, router.WithSocksListenAddr(socksListenAddr))
 				}
@@ -89,13 +93,29 @@ var tunnelRunCmd = &cobra.Command{
 					routerOpts = append(routerOpts, router.WithResolveConfig(resolveConf))
 				}
 
-				r, routerErr = router.NewICXNetstackRouter(pcGeneve, connectResp.MTU, routerOpts...)
+				r, routerErr = router.NewICXNetstackRouter(routerOpts...)
 				if routerErr != nil {
 					return
 				}
 				handler = r.Handler
 
+				for _, addrStr := range connectResp.Addresses {
+					slog.Info("Adding address", slog.String("address", addrStr))
+
+					addr, err := netip.ParsePrefix(addrStr)
+					if err != nil {
+						slog.Warn("Failed to parse address", slog.String("address", addrStr), slog.Any("error", err))
+						continue
+					}
+
+					if err := r.AddAddr(addr, nil); err != nil {
+						slog.Warn("Failed to add address", slog.String("address", addrStr), slog.Any("error", err))
+					}
+				}
+
 				for _, route := range connectResp.Routes {
+					slog.Info("Adding route", slog.String("destination", route.Destination))
+
 					dst, err := netip.ParsePrefix(route.Destination)
 					if err != nil {
 						slog.Warn("Failed to parse route prefix", slog.String("prefix", route.Destination), slog.Any("error", err))
@@ -336,7 +356,16 @@ func manageRelayConnection(
 					return cleanupOnErr(fmt.Errorf("parse assigned addresses: %w", err))
 				}
 
-				// Multi-home: same VNI across different relays.
+				for _, route := range connectResp.Routes {
+					dst, err := netip.ParsePrefix(route.Destination)
+					if err != nil {
+						slog.Warn("Failed to parse route prefix", slog.String("prefix", route.Destination), slog.Any("error", err))
+						continue
+					}
+
+					overlayAddrs = append(overlayAddrs, dst)
+				}
+
 				if err := handler.AddVirtualNetwork(connectResp.VNI, netstack.ToFullAddress(remoteAddr), overlayAddrs); err != nil {
 					return cleanupOnErr(fmt.Errorf("add virtual network: %w", err))
 				}

pkg/netstack/icx_network.go

@@ -291,21 +291,6 @@ func (net *ICXNetwork) DelAddr(addr netip.Prefix) error {
 	return nil
 }
 
-// LocalAddresses returns the list of local addresses assigned to the network.
-func (net *ICXNetwork) LocalAddresses() ([]netip.Prefix, error) {
-	nic := net.stack.NICInfo()[net.nicID]
-
-	var addrs []netip.Prefix
-	for _, assignedAddr := range nic.ProtocolAddresses {
-		addrs = append(addrs, netip.PrefixFrom(
-			addrFromNetstackIP(assignedAddr.AddressWithPrefix.Address),
-			assignedAddr.AddressWithPrefix.PrefixLen,
-		))
-	}
-
-	return addrs, nil
-}
-
 // ForwardTo forwards all inbound TCP traffic to the upstream network.
 func (net *ICXNetwork) ForwardTo(ctx context.Context, upstream network.Network) error {
 	// Allow outgoing packets to have a source address different from the NIC.
@@ -326,3 +311,24 @@ func (net *ICXNetwork) ForwardTo(ctx context.Context, upstream network.Network)
 
 	return nil
 }
+
+func prefixToSubnet(p netip.Prefix) (tcpip.Subnet, error) {
+	addr := tcpip.AddrFromSlice(p.Addr().AsSlice())
+
+	totalBits := 128
+	if p.Addr().Is4() {
+		totalBits = 32
+	}
+	ones := p.Bits()
+	if ones < 0 || ones > totalBits {
+		return tcpip.Subnet{}, fmt.Errorf("invalid prefix length %d", ones)
+	}
+
+	maskBytes := make([]byte, totalBits/8)
+	for i := 0; i < ones; i++ {
+		maskBytes[i/8] |= 1 << (7 - uint(i%8))
+	}
+	mask := tcpip.MaskFromBytes(maskBytes)
+
+	return tcpip.NewSubnet(addr, mask)
+}

pkg/netstack/icx_network_test.go

@@ -15,7 +15,6 @@ import (
 
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sync/errgroup"
-	"gvisor.dev/gvisor/pkg/tcpip"
 
 	"github.com/apoxy-dev/icx"
 
@@ -58,12 +57,10 @@ func TestICXNetwork_Speed(t *testing.T) {
 	aB := netip.AddrPortFrom(netip.MustParseAddr(uaB.IP.String()), uint16(uaB.Port))
 
 	// Build ICX handlers in L3 mode and link them together
-	hA, err := icx.NewHandler(icx.WithLocalAddr(netstack.ToFullAddress(aA)),
-		icx.WithVirtMAC(tcpip.GetRandMacAddr()), icx.WithLayer3VirtFrames())
+	hA, err := icx.NewHandler(icx.WithLocalAddr(netstack.ToFullAddress(aA)), icx.WithLayer3VirtFrames())
 	require.NoError(t, err)
 
-	hB, err := icx.NewHandler(icx.WithLocalAddr(netstack.ToFullAddress(aB)),
-		icx.WithVirtMAC(tcpip.GetRandMacAddr()), icx.WithLayer3VirtFrames())
+	hB, err := icx.NewHandler(icx.WithLocalAddr(netstack.ToFullAddress(aB)), icx.WithLayer3VirtFrames())
 	require.NoError(t, err)
 
 	const vni = 0x424242

pkg/netstack/tcp_forwarder.go

@@ -30,27 +30,6 @@ func TCPForwarder(ctx context.Context, ipstack *stack.Stack, upstream network.Ne
 	return tcpForwarder.HandlePacket
 }
 
-// Unmap4in6 converts an IPv6 address to an IPv4 address if it is an IPv4-mapped IPv6 address.
-// If the address is not an IPv4-mapped IPv6 address, it is returned unchanged.
-// If the IPv4 address is zero, returns 127.0.0.1.
-// It is following /96 embedding scheme from RFC 6052 (https://datatracker.ietf.org/doc/html/rfc6052#section-2.2).
-func Unmap4in6(addr netip.Addr) netip.Addr {
-	if !addr.Is6() {
-		return addr
-	}
-	b16 := addr.As16()
-	v4addr := netip.AddrFrom4([4]byte{
-		b16[12],
-		b16[13],
-		b16[14],
-		b16[15],
-	})
-	if !v4addr.IsValid() {
-		return netip.AddrFrom4([4]byte{127, 0, 0, 1})
-	}
-	return v4addr
-}
-
 func tcpHandler(ctx context.Context, upstream network.Network) func(req *tcp.ForwarderRequest) {
 	return func(req *tcp.ForwarderRequest) {
 		reqDetails := req.ID()
@@ -60,7 +39,7 @@ func tcpHandler(ctx context.Context, upstream network.Network) func(req *tcp.For
 		// - IPv4-mapped IPv6 addresses (::ffff:192.168.1.1) are converted to IPv4
 		// - Regular IPv6 addresses left as is (::1)
 		dstAddrPort := netip.AddrPortFrom(
-			Unmap4in6(addrFromNetstackIP(reqDetails.LocalAddress)),
+			addrFromNetstackIP(reqDetails.LocalAddress).Unmap(),
 			reqDetails.LocalPort,
 		)
 

pkg/netstack/udp_forwarder.go

@@ -86,7 +86,7 @@ func udpHandler(ctx context.Context, upstream network.Network) func(req *udp.For
 		// - IPv4-mapped IPv6 addresses (::ffff:192.168.1.1) are converted to IPv4
 		// - Regular IPv6 addresses left as is (::1)
 		dstAddrPort := netip.AddrPortFrom(
-			Unmap4in6(addrFromNetstackIP(reqDetails.LocalAddress)),
+			addrFromNetstackIP(reqDetails.LocalAddress).Unmap(),
 			reqDetails.LocalPort,
 		)
 

pkg/tunnel/bifurcate/bifurcate.go

@@ -1,7 +1,6 @@
 package bifurcate
 
 import (
-	"log/slog"
 	"net"
 	"sync"
 
@@ -36,8 +35,6 @@ func Bifurcate(pc net.PacketConn) (net.PacketConn, net.PacketConn) {
 	otherClosed := otherConn.closed
 
 	go func() {
-		defer func() { slog.Info("Bifurcate exiting") }()
-
 		for {
 			// If both sides are gone, stop.
 			if geneveCh == nil && otherCh == nil {