Merge pull request #211 from dcbw/e2e-testing

Fix namespace switch issues and add ipvlan, macvlan, and bridge e2e testing

pkg/ip/link.go

@@ -81,7 +81,7 @@ func RandomVethName() (string, error) {
 // SetupVeth sets up a virtual ethernet link.
 // Should be in container netns, and will switch back to hostNS to set the host
 // veth end up.
-func SetupVeth(contVethName string, mtu int, hostNS *os.File) (hostVeth, contVeth netlink.Link, err error) {
+func SetupVeth(contVethName string, mtu int, hostNS ns.NetNS) (hostVeth, contVeth netlink.Link, err error) {
 	var hostVethName string
 	hostVethName, contVeth, err = makeVeth(contVethName, mtu)
 	if err != nil {
@@ -104,10 +104,10 @@ func SetupVeth(contVethName string, mtu int, hostNS *os.File) (hostVeth, contVet
 		return
 	}
 
-	err = ns.WithNetNS(hostNS, false, func(_ *os.File) error {
+	err = hostNS.Do(func(_ ns.NetNS) error {
 		hostVeth, err := netlink.LinkByName(hostVethName)
 		if err != nil {
-			return fmt.Errorf("failed to lookup %q in %q: %v", hostVethName, hostNS.Name(), err)
+			return fmt.Errorf("failed to lookup %q in %q: %v", hostVethName, hostNS.Path(), err)
 		}
 
 		if err = netlink.LinkSetUp(hostVeth); err != nil {

pkg/ns/README.md

@@ -0,0 +1,31 @@
+### Namespaces, Threads, and Go
+On Linux each OS thread can have a different network namespace.  Go's thread scheduling model switches goroutines between OS threads based on OS thread load and whether the goroutine would block other goroutines.  This can result in a goroutine switching network namespaces without notice and lead to errors in your code.
+
+### Namespace Switching
+Switching namespaces with the `ns.Set()` method is not recommended without additional strategies to prevent unexpected namespace changes when your goroutines switch OS threads.
+
+Go provides the `runtime.LockOSThread()` function to ensure a specific goroutine executes on its current OS thread and prevents any other goroutine from running in that thread until the locked one exits.  Careful usage of `LockOSThread()` and goroutines can provide good control over which network namespace a given goroutine executes in.
+
+For example, you cannot rely on the `ns.Set()` namespace being the current namespace after the `Set()` call unless you do two things.  First, the goroutine calling `Set()` must have previously called `LockOSThread()`.  Second, you must ensure `runtime.UnlockOSThread()` is not called somewhere in-between.  You also cannot rely on the initial network namespace remaining the current network namespace if any other code in your program switches namespaces, unless you have already called `LockOSThread()` in that goroutine.  Note that `LockOSThread()` prevents the Go scheduler from optimally scheduling goroutines for best performance, so `LockOSThread()` should only be used in small, isolated goroutines that release the lock quickly.
+
+### Do() The Recommended Thing
+The `ns.Do()` method provides control over network namespaces for you by implementing these strategies. All code dependent on a particular network namespace should be wrapped in the `ns.Do()` method to ensure the correct namespace is selected for the duration of your code.  For example:
+
+```go
+targetNs, err := ns.NewNS()
+if err != nil {
+    return err
+}
+err = targetNs.Do(func(hostNs ns.NetNS) error {
+	dummy := &netlink.Dummy{
+		LinkAttrs: netlink.LinkAttrs{
+			Name: "dummy0",
+		},
+	}
+	return netlink.LinkAdd(dummy)
+})
+```
+
+### Further Reading
+ - https://github.com/golang/go/wiki/LockOSThread
+ - http://morsmachine.dk/go-scheduler

pkg/ns/ns.go

@@ -15,64 +15,223 @@
 package ns
 
 import (
+	"crypto/rand"
 	"fmt"
 	"os"
+	"path"
 	"runtime"
-	"syscall"
+	"sync"
+
+	"golang.org/x/sys/unix"
 )
 
-// SetNS sets the network namespace on a target file.
-func SetNS(f *os.File, flags uintptr) error {
-	_, _, err := syscall.RawSyscall(setNsNr, f.Fd(), flags, 0)
-	if err != 0 {
-		return err
-	}
+type NetNS interface {
+	// Executes the passed closure in this object's network namespace,
+	// attemtping to restore the original namespace before returning.
+	// However, since each OS thread can have a different network namespace,
+	// and Go's thread scheduling is highly variable, callers cannot
+	// guarantee any specific namespace is set unless operations that
+	// require that namespace are wrapped with Do().  Also, no code called
+	// from Do() should call runtime.UnlockOSThread(), or the risk
+	// of executing code in an incorrect namespace will be greater.  See
+	// https://github.com/golang/go/wiki/LockOSThread for further details.
+	Do(toRun func(NetNS) error) error
 
-	return nil
+	// Sets the current network namespace to this object's network namespace.
+	// Note that since Go's thread scheduling is highly variable, callers
+	// cannot guarantee the requested namespace will be the current namespace
+	// after this function is called; to ensure this wrap operations that
+	// require the namespace with Do() instead.
+	Set() error
+
+	// Returns the filesystem path representing this object's network namespace
+	Path() string
+
+	// Returns a file descriptor representing this object's network namespace
+	Fd() uintptr
+
+	// Cleans up this instance of the network namespace; if this instance
+	// is the last user the namespace will be destroyed
+	Close() error
 }
 
-// WithNetNSPath executes the passed closure under the given network
-// namespace, restoring the original namespace afterwards.
-// Changing namespaces must be done on a goroutine that has been
-// locked to an OS thread. If lockThread arg is true, this function
-// locks the goroutine prior to change namespace and unlocks before
-// returning
-func WithNetNSPath(nspath string, lockThread bool, f func(*os.File) error) error {
-	ns, err := os.Open(nspath)
+type netNS struct {
+	file    *os.File
+	mounted bool
+}
+
+func getCurrentThreadNetNSPath() string {
+	// /proc/self/ns/net returns the namespace of the main thread, not
+	// of whatever thread this goroutine is running on.  Make sure we
+	// use the thread's net namespace since the thread is switching around
+	return fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), unix.Gettid())
+}
+
+// Returns an object representing the current OS thread's network namespace
+func GetCurrentNS() (NetNS, error) {
+	return GetNS(getCurrentThreadNetNSPath())
+}
+
+// Returns an object representing the namespace referred to by @path
+func GetNS(nspath string) (NetNS, error) {
+	fd, err := os.Open(nspath)
 	if err != nil {
-		return fmt.Errorf("Failed to open %v: %v", nspath, err)
+		return nil, err
 	}
-	defer ns.Close()
-	return WithNetNS(ns, lockThread, f)
+	return &netNS{file: fd}, nil
 }
 
-// WithNetNS executes the passed closure under the given network
-// namespace, restoring the original namespace afterwards.
-// Changing namespaces must be done on a goroutine that has been
-// locked to an OS thread. If lockThread arg is true, this function
-// locks the goroutine prior to change namespace and unlocks before
-// returning.  If the closure returns an error, WithNetNS attempts to
-// restore the original namespace before returning.
-func WithNetNS(ns *os.File, lockThread bool, f func(*os.File) error) error {
-	if lockThread {
-		runtime.LockOSThread()
-		defer runtime.UnlockOSThread()
+// Creates a new persistent network namespace and returns an object
+// representing that namespace, without switching to it
+func NewNS() (NetNS, error) {
+	const nsRunDir = "/var/run/netns"
+
+	b := make([]byte, 16)
+	_, err := rand.Reader.Read(b)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate random netns name: %v", err)
+	}
+
+	err = os.MkdirAll(nsRunDir, 0755)
+	if err != nil {
+		return nil, err
+	}
+
+	// create an empty file at the mount point
+	nsName := fmt.Sprintf("cni-%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
+	nsPath := path.Join(nsRunDir, nsName)
+	mountPointFd, err := os.Create(nsPath)
+	if err != nil {
+		return nil, err
 	}
-	// save a handle to current (host) network namespace
-	thisNS, err := os.Open("/proc/self/ns/net")
+	mountPointFd.Close()
+
+	// Ensure the mount point is cleaned up on errors; if the namespace
+	// was successfully mounted this will have no effect because the file
+	// is in-use
+	defer os.RemoveAll(nsPath)
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	// do namespace work in a dedicated goroutine, so that we can safely
+	// Lock/Unlock OSThread without upsetting the lock/unlock state of
+	// the caller of this function
+	var fd *os.File
+	go (func() {
+		defer wg.Done()
+		runtime.LockOSThread()
+
+		var origNS NetNS
+		origNS, err = GetNS(getCurrentThreadNetNSPath())
+		if err != nil {
+			return
+		}
+		defer origNS.Close()
+
+		// create a new netns on the current thread
+		err = unix.Unshare(unix.CLONE_NEWNET)
+		if err != nil {
+			return
+		}
+		defer origNS.Set()
+
+		// bind mount the new netns from the current thread onto the mount point
+		err = unix.Mount(getCurrentThreadNetNSPath(), nsPath, "none", unix.MS_BIND, "")
+		if err != nil {
+			return
+		}
+
+		fd, err = os.Open(nsPath)
+		if err != nil {
+			return
+		}
+	})()
+	wg.Wait()
+
 	if err != nil {
-		return fmt.Errorf("Failed to open /proc/self/ns/net: %v", err)
+		unix.Unmount(nsPath, unix.MNT_DETACH)
+		return nil, fmt.Errorf("failed to create namespace: %v", err)
+	}
+
+	return &netNS{file: fd, mounted: true}, nil
+}
+
+func (ns *netNS) Path() string {
+	return ns.file.Name()
+}
+
+func (ns *netNS) Fd() uintptr {
+	return ns.file.Fd()
+}
+
+func (ns *netNS) Close() error {
+	ns.file.Close()
+
+	if ns.mounted {
+		if err := unix.Unmount(ns.file.Name(), unix.MNT_DETACH); err != nil {
+			return fmt.Errorf("Failed to unmount namespace %s: %v", ns.file.Name(), err)
+		}
+		if err := os.RemoveAll(ns.file.Name()); err != nil {
+			return fmt.Errorf("Failed to clean up namespace %s: %v", ns.file.Name(), err)
+		}
+	}
+	return nil
+}
+
+func (ns *netNS) Do(toRun func(NetNS) error) error {
+	containedCall := func(hostNS NetNS) error {
+		threadNS, err := GetNS(getCurrentThreadNetNSPath())
+		if err != nil {
+			return fmt.Errorf("failed to open current netns: %v", err)
+		}
+		defer threadNS.Close()
+
+		// switch to target namespace
+		if err = ns.Set(); err != nil {
+			return fmt.Errorf("error switching to ns %v: %v", ns.file.Name(), err)
+		}
+		defer threadNS.Set() // switch back
+
+		return toRun(hostNS)
 	}
-	defer thisNS.Close()
 
-	if err = SetNS(ns, syscall.CLONE_NEWNET); err != nil {
-		return fmt.Errorf("Error switching to ns %v: %v", ns.Name(), err)
+	// save a handle to current network namespace
+	hostNS, err := GetNS(getCurrentThreadNetNSPath())
+	if err != nil {
+		return fmt.Errorf("Failed to open current namespace: %v", err)
 	}
-	defer SetNS(thisNS, syscall.CLONE_NEWNET) // switch back
+	defer hostNS.Close()
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	var innerError error
+	go func() {
+		defer wg.Done()
+		runtime.LockOSThread()
+		innerError = containedCall(hostNS)
+	}()
+	wg.Wait()
+
+	return innerError
+}
 
-	if err = f(thisNS); err != nil {
-		return err
+func (ns *netNS) Set() error {
+	if _, _, err := unix.Syscall(unix.SYS_SETNS, ns.Fd(), uintptr(unix.CLONE_NEWNET), 0); err != 0 {
+		return fmt.Errorf("Error switching to ns %v: %v", ns.file.Name(), err)
 	}
 
 	return nil
 }
+
+// WithNetNSPath executes the passed closure under the given network
+// namespace, restoring the original namespace afterwards.
+func WithNetNSPath(nspath string, toRun func(NetNS) error) error {
+	ns, err := GetNS(nspath)
+	if err != nil {
+		return fmt.Errorf("Failed to open %v: %v", nspath, err)
+	}
+	defer ns.Close()
+	return ns.Do(toRun)
+}