Netstack (#1403)

Add netstack support for the agent to run it without privileges. - use interface for tun device - use common IPC for userspace WireGuard integration - move udpmux creation and sharedsock to tun layer
netbirdio · Jan 3, 2024 · 1de3bb5 · 1de3bb5
1 parent 163933d
commit 1de3bb5
Show file tree

Hide file tree

Showing 40 changed files with 1,731 additions and 988 deletions.
diff --git a/client/internal/dns/server_test.go b/client/internal/dns/server_test.go
@@ -12,6 +12,7 @@ import (
 
 	"github.com/golang/mock/gomock"
 	log "github.com/sirupsen/logrus"
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 
 	"github.com/netbirdio/netbird/client/firewall/uspfilter"
 	"github.com/netbirdio/netbird/client/internal/stdnet"
@@ -250,11 +251,12 @@ func TestUpdateDNSServer(t *testing.T) {
 
 	for n, testCase := range testCases {
 		t.Run(testCase.name, func(t *testing.T) {
+			privKey, _ := wgtypes.GenerateKey()
 			newNet, err := stdnet.NewNet(nil)
 			if err != nil {
 				t.Fatal(err)
 			}
-			wgIface, err := iface.NewWGIFace(fmt.Sprintf("utun230%d", n), fmt.Sprintf("100.66.100.%d/32", n+1), iface.DefaultMTU, nil, newNet)
+			wgIface, err := iface.NewWGIFace(fmt.Sprintf("utun230%d", n), fmt.Sprintf("100.66.100.%d/32", n+1), 33100, privKey.String(), iface.DefaultMTU, newNet, nil)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -331,7 +333,8 @@ func TestDNSFakeResolverHandleUpdates(t *testing.T) {
 		return
 	}
 
-	wgIface, err := iface.NewWGIFace("utun2301", "100.66.100.1/32", iface.DefaultMTU, nil, newNet)
+	privKey, _ := wgtypes.GeneratePrivateKey()
+	wgIface, err := iface.NewWGIFace("utun2301", "100.66.100.1/32", 33100, privKey.String(), iface.DefaultMTU, newNet, nil)
 	if err != nil {
 		t.Errorf("build interface wireguard: %v", err)
 		return
@@ -782,7 +785,8 @@ func createWgInterfaceWithBind(t *testing.T) (*iface.WGIface, error) {
 		return nil, err
 	}
 
-	wgIface, err := iface.NewWGIFace("utun2301", "100.66.100.2/24", iface.DefaultMTU, nil, newNet)
+	privKey, _ := wgtypes.GeneratePrivateKey()
+	wgIface, err := iface.NewWGIFace("utun2301", "100.66.100.2/24", 33100, privKey.String(), iface.DefaultMTU, newNet, nil)
 	if err != nil {
 		t.Fatalf("build interface wireguard: %v", err)
 		return nil, err

diff --git a/client/internal/engine.go b/client/internal/engine.go
@@ -3,7 +3,6 @@ package internal
 import (
 	"context"
 	"fmt"
-	"io"
 	"math/rand"
 	"net"
 	"net/netip"
@@ -32,7 +31,6 @@ import (
 	mgm "github.com/netbirdio/netbird/management/client"
 	mgmProto "github.com/netbirdio/netbird/management/proto"
 	"github.com/netbirdio/netbird/route"
-	"github.com/netbirdio/netbird/sharedsock"
 	signal "github.com/netbirdio/netbird/signal/client"
 	sProto "github.com/netbirdio/netbird/signal/proto"
 	"github.com/netbirdio/netbird/util"
@@ -107,8 +105,7 @@ type Engine struct {
 	wgInterface    *iface.WGIface
 	wgProxyFactory *wgproxy.Factory
 
-	udpMux     *bind.UniversalUDPMuxDefault
-	udpMuxConn io.Closer
+	udpMux *bind.UniversalUDPMuxDefault
 
 	// networkSerial is the latest CurrentSerial (state ID) of the network sent by the Management service
 	networkSerial uint64
@@ -181,66 +178,26 @@ func (e *Engine) Start() error {
 	e.syncMsgMux.Lock()
 	defer e.syncMsgMux.Unlock()
 
-	wgIFaceName := e.config.WgIfaceName
-	wgAddr := e.config.WgAddr
-	myPrivateKey := e.config.WgPrivateKey
-	var err error
-	transportNet, err := e.newStdNet()
+	wgIface, err := e.newWgIface()
 	if err != nil {
-		log.Errorf("failed to create pion's stdnet: %s", err)
+		log.Errorf("failed creating wireguard interface instance %s: [%s]", e.config.WgIfaceName, err.Error())
+		return err
 	}
+	e.wgInterface = wgIface
 
-	e.wgInterface, err = iface.NewWGIFace(wgIFaceName, wgAddr, iface.DefaultMTU, e.mobileDep.TunAdapter, transportNet)
+	initialRoutes, dnsServer, err := e.newDnsServer()
 	if err != nil {
-		log.Errorf("failed creating wireguard interface instance %s: [%s]", wgIFaceName, err.Error())
+		e.close()
 		return err
 	}
+	e.dnsServer = dnsServer
 
-	var routes []*route.Route
-
-	switch runtime.GOOS {
-	case "android":
-		var dnsConfig *nbdns.Config
-		routes, dnsConfig, err = e.readInitialSettings()
-		if err != nil {
-			return err
-		}
-		if e.dnsServer == nil {
-			e.dnsServer = dns.NewDefaultServerPermanentUpstream(e.ctx, e.wgInterface, e.mobileDep.HostDNSAddresses, *dnsConfig, e.mobileDep.NetworkChangeListener)
-			go e.mobileDep.DnsReadyListener.OnReady()
-		}
-	case "ios":
-		if e.dnsServer == nil {
-			e.dnsServer = dns.NewDefaultServerIos(e.ctx, e.wgInterface, e.mobileDep.DnsManager)
-		}
-	default:
-		if e.dnsServer == nil {
-			e.dnsServer, err = dns.NewDefaultServer(e.ctx, e.wgInterface, e.config.CustomDNSAddress)
-			if err != nil {
-				e.close()
-				return err
-			}
-		}
-	}
-
-	e.routeManager = routemanager.NewManager(e.ctx, e.config.WgPrivateKey.PublicKey().String(), e.wgInterface, e.statusRecorder, routes)
+	e.routeManager = routemanager.NewManager(e.ctx, e.config.WgPrivateKey.PublicKey().String(), e.wgInterface, e.statusRecorder, initialRoutes)
 	e.routeManager.SetRouteChangeListener(e.mobileDep.NetworkChangeListener)
 
-	switch runtime.GOOS {
-	case "android":
-		err = e.wgInterface.CreateOnAndroid(iface.MobileIFaceArguments{
-			Routes:        e.routeManager.InitialRouteRange(),
-			Dns:           e.dnsServer.DnsIP(),
-			SearchDomains: e.dnsServer.SearchDomains(),
-		})
-	case "ios":
-		e.mobileDep.NetworkChangeListener.SetInterfaceIP(wgAddr)
-		err = e.wgInterface.CreateOniOS(e.mobileDep.FileDescriptor)
-	default:
-		err = e.wgInterface.Create()
-	}
+	err = e.wgInterfaceCreate()
 	if err != nil {
-		log.Errorf("failed creating tunnel interface %s: [%s]", wgIFaceName, err.Error())
+		log.Errorf("failed creating tunnel interface %s: [%s]", e.config.WgIfaceName, err.Error())
 		e.close()
 		return err
 	}
@@ -258,33 +215,13 @@ func (e *Engine) Start() error {
 		}
 	}
 
-	err = e.wgInterface.Configure(myPrivateKey.String(), e.config.WgPort)
+	e.udpMux, err = e.wgInterface.Up()
 	if err != nil {
-		log.Errorf("failed configuring Wireguard interface [%s]: %s", wgIFaceName, err.Error())
+		log.Errorf("failed to pull up wgInterface [%s]: %s", e.wgInterface.Name(), err.Error())
 		e.close()
 		return err
 	}
 
-	if e.wgInterface.IsUserspaceBind() {
-		iceBind := e.wgInterface.GetBind()
-		udpMux, err := iceBind.GetICEMux()
-		if err != nil {
-			e.close()
-			return err
-		}
-		e.udpMux = udpMux
-		log.Infof("using userspace bind mode %s", udpMux.LocalAddr().String())
-	} else {
-		rawSock, err := sharedsock.Listen(e.config.WgPort, sharedsock.NewIncomingSTUNFilter())
-		if err != nil {
-			return err
-		}
-		mux := bind.NewUniversalUDPMuxDefault(bind.UniversalUDPMuxParams{UDPConn: rawSock, Net: transportNet})
-		go mux.ReadFromConn(e.ctx)
-		e.udpMuxConn = rawSock
-		e.udpMux = mux
-	}
-
 	if e.firewall != nil {
 		e.acl = acl.NewDefaultManager(e.firewall)
 	}
@@ -1042,18 +979,6 @@ func (e *Engine) close() {
 		}
 	}
 
-	if e.udpMux != nil {
-		if err := e.udpMux.Close(); err != nil {
-			log.Debugf("close udp mux: %v", err)
-		}
-	}
-
-	if e.udpMuxConn != nil {
-		if err := e.udpMuxConn.Close(); err != nil {
-			log.Debugf("close udp mux connection: %v", err)
-		}
-	}
-
 	if !isNil(e.sshServer) {
 		err := e.sshServer.Stop()
 		if err != nil {
@@ -1087,6 +1012,68 @@ func (e *Engine) readInitialSettings() ([]*route.Route, *nbdns.Config, error) {
 	return routes, &dnsCfg, nil
 }
 
+func (e *Engine) newWgIface() (*iface.WGIface, error) {
+	transportNet, err := e.newStdNet()
+	if err != nil {
+		log.Errorf("failed to create pion's stdnet: %s", err)
+	}
+
+	var mArgs *iface.MobileIFaceArguments
+	switch runtime.GOOS {
+	case "android":
+		mArgs = &iface.MobileIFaceArguments{
+			TunAdapter: e.mobileDep.TunAdapter,
+			TunFd:      int(e.mobileDep.FileDescriptor),
+		}
+	case "ios":
+		mArgs = &iface.MobileIFaceArguments{
+			TunFd: int(e.mobileDep.FileDescriptor),
+		}
+	default:
+	}
+
+	return iface.NewWGIFace(e.config.WgIfaceName, e.config.WgAddr, e.config.WgPort, e.config.WgPrivateKey.String(), iface.DefaultMTU, transportNet, mArgs)
+}
+
+func (e *Engine) wgInterfaceCreate() (err error) {
+	switch runtime.GOOS {
+	case "android":
+		err = e.wgInterface.CreateOnAndroid(e.routeManager.InitialRouteRange(), e.dnsServer.DnsIP(), e.dnsServer.SearchDomains())
+	case "ios":
+		e.mobileDep.NetworkChangeListener.SetInterfaceIP(e.config.WgAddr)
+		err = e.wgInterface.Create()
+	default:
+		err = e.wgInterface.Create()
+	}
+	return err
+}
+
+func (e *Engine) newDnsServer() ([]*route.Route, dns.Server, error) {
+	// due to tests where we are using a mocked version of the DNS server
+	if e.dnsServer != nil {
+		return nil, e.dnsServer, nil
+	}
+	switch runtime.GOOS {
+	case "android":
+		routes, dnsConfig, err := e.readInitialSettings()
+		if err != nil {
+			return nil, nil, err
+		}
+		dnsServer := dns.NewDefaultServerPermanentUpstream(e.ctx, e.wgInterface, e.mobileDep.HostDNSAddresses, *dnsConfig, e.mobileDep.NetworkChangeListener)
+		go e.mobileDep.DnsReadyListener.OnReady()
+		return routes, dnsServer, nil
+	case "ios":
+		dnsServer := dns.NewDefaultServerIos(e.ctx, e.wgInterface, e.mobileDep.DnsManager)
+		return nil, dnsServer, nil
+	default:
+		dnsServer, err := dns.NewDefaultServer(e.ctx, e.wgInterface, e.config.CustomDNSAddress)
+		if err != nil {
+			return nil, nil, err
+		}
+		return nil, dnsServer, nil
+	}
+}
+
 func findIPFromInterfaceName(ifaceName string) (net.IP, error) {
 	iface, err := net.InterfaceByName(ifaceName)
 	if err != nil {

diff --git a/client/internal/engine_test.go b/client/internal/engine_test.go
@@ -213,7 +213,7 @@ func TestEngine_UpdateNetworkMap(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	engine.wgInterface, err = iface.NewWGIFace("utun102", "100.64.0.1/24", iface.DefaultMTU, nil, newNet)
+	engine.wgInterface, err = iface.NewWGIFace("utun102", "100.64.0.1/24", engine.config.WgPort, key.String(), iface.DefaultMTU, newNet, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -567,7 +567,7 @@ func TestEngine_UpdateNetworkMapWithRoutes(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			engine.wgInterface, err = iface.NewWGIFace(wgIfaceName, wgAddr, iface.DefaultMTU, nil, newNet)
+			engine.wgInterface, err = iface.NewWGIFace(wgIfaceName, wgAddr, engine.config.WgPort, key.String(), iface.DefaultMTU, newNet, nil)
 			assert.NoError(t, err, "shouldn't return error")
 			input := struct {
 				inputSerial uint64
@@ -736,7 +736,7 @@ func TestEngine_UpdateNetworkMapWithDNSUpdate(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			engine.wgInterface, err = iface.NewWGIFace(wgIfaceName, wgAddr, iface.DefaultMTU, nil, newNet)
+			engine.wgInterface, err = iface.NewWGIFace(wgIfaceName, wgAddr, 33100, key.String(), iface.DefaultMTU, newNet, nil)
 			assert.NoError(t, err, "shouldn't return error")
 
 			mockRouteManager := &routemanager.MockManager{

diff --git a/client/internal/mobile_dependency.go b/client/internal/mobile_dependency.go
@@ -9,11 +9,14 @@ import (
 
 // MobileDependency collect all dependencies for mobile platform
 type MobileDependency struct {
+	// Android only
 	TunAdapter            iface.TunAdapter
 	IFaceDiscover         stdnet.ExternalIFaceDiscover
 	NetworkChangeListener listener.NetworkChangeListener
 	HostDNSAddresses      []string
 	DnsReadyListener      dns.ReadyListener
-	DnsManager            dns.IosDnsManager
-	FileDescriptor        int32
+
+	//	iOS only
+	DnsManager     dns.IosDnsManager
+	FileDescriptor int32
 }
diff --git a/client/internal/routemanager/manager_test.go b/client/internal/routemanager/manager_test.go
@@ -8,6 +8,7 @@ import (
 	"testing"
 
 	"github.com/pion/transport/v3/stdnet"
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 
 	"github.com/stretchr/testify/require"
 
@@ -399,12 +400,12 @@ func TestManagerUpdateRoutes(t *testing.T) {
 
 	for n, testCase := range testCases {
 		t.Run(testCase.name, func(t *testing.T) {
-
+			peerPrivateKey, _ := wgtypes.GeneratePrivateKey()
 			newNet, err := stdnet.NewNet()
 			if err != nil {
 				t.Fatal(err)
 			}
-			wgInterface, err := iface.NewWGIFace(fmt.Sprintf("utun43%d", n), "100.65.65.2/24", iface.DefaultMTU, nil, newNet)
+			wgInterface, err := iface.NewWGIFace(fmt.Sprintf("utun43%d", n), "100.65.65.2/24", 33100, peerPrivateKey.String(), iface.DefaultMTU, newNet, nil)
 			require.NoError(t, err, "should create testing WGIface interface")
 			defer wgInterface.Close()
 

diff --git a/client/internal/routemanager/systemops_nonandroid_test.go b/client/internal/routemanager/systemops_nonandroid_test.go
@@ -14,6 +14,7 @@ import (
 	"github.com/pion/transport/v3/stdnet"
 	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 
 	"github.com/netbirdio/netbird/iface"
 )
@@ -41,11 +42,12 @@ func TestAddRemoveRoutes(t *testing.T) {
 
 	for n, testCase := range testCases {
 		t.Run(testCase.name, func(t *testing.T) {
+			peerPrivateKey, _ := wgtypes.GeneratePrivateKey()
 			newNet, err := stdnet.NewNet()
 			if err != nil {
 				t.Fatal(err)
 			}
-			wgInterface, err := iface.NewWGIFace(fmt.Sprintf("utun53%d", n), "100.65.75.2/24", iface.DefaultMTU, nil, newNet)
+			wgInterface, err := iface.NewWGIFace(fmt.Sprintf("utun53%d", n), "100.65.75.2/24", 33100, peerPrivateKey.String(), iface.DefaultMTU, newNet, nil)
 			require.NoError(t, err, "should create testing WGIface interface")
 			defer wgInterface.Close()
 
@@ -175,11 +177,12 @@ func TestAddExistAndRemoveRouteNonAndroid(t *testing.T) {
 			log.SetOutput(os.Stderr)
 		}()
 		t.Run(testCase.name, func(t *testing.T) {
+			peerPrivateKey, _ := wgtypes.GeneratePrivateKey()
 			newNet, err := stdnet.NewNet()
 			if err != nil {
 				t.Fatal(err)
 			}
-			wgInterface, err := iface.NewWGIFace(fmt.Sprintf("utun53%d", n), "100.65.75.2/24", iface.DefaultMTU, nil, newNet)
+			wgInterface, err := iface.NewWGIFace(fmt.Sprintf("utun53%d", n), "100.65.75.2/24", 33100, peerPrivateKey.String(), iface.DefaultMTU, newNet, nil)
 			require.NoError(t, err, "should create testing WGIface interface")
 			defer wgInterface.Close()