Replaced the pipes for communcation with a socketpair instead

I was not aware of socketpair existing until a few days ago. This
basically functions exactly the same as the pipe implementation. Except
that I only have 1 file descriptor for both sides, that are both read
and write. Making book-keeping for it easier. However just like with the
pipe implementation default performance is not great. That's why we're
using the unix.SOCK_SEQPACKET setting. Which seems very similar to the
O_DIRECT with the pipe2 call. Below is a iperf output with and without
this setting.

Without unix.SOCK_SEQPACKET:
Accepted connection from 192.168.100.123, port 49020
[  5] local 192.168.100.123 port 5201 connected to 192.168.100.123 port 49024
[ ID] Interval           Transfer     Bitrate
[  5]   0.00-1.00   sec  12.4 MBytes   104 Mbits/sec
[  5]   1.00-2.00   sec  9.26 MBytes  77.7 Mbits/sec
[  5]   2.00-3.00   sec  5.47 MBytes  45.9 Mbits/sec
[  5]   3.00-4.00   sec  8.44 MBytes  70.8 Mbits/sec
[  5]   4.00-5.00   sec  2.99 MBytes  25.1 Mbits/sec
[  5]   5.00-6.00   sec  12.6 MBytes   106 Mbits/sec
[  5]   6.00-7.00   sec  5.61 MBytes  47.0 Mbits/sec
[  5]   7.00-8.00   sec  5.39 MBytes  45.2 Mbits/sec
[  5]   8.00-9.00   sec  5.99 MBytes  50.3 Mbits/sec
[  5]   9.00-10.00  sec  4.88 MBytes  41.0 Mbits/sec
- - - - - - - - - - - - - - - - - - - - - - - - -
[ ID] Interval           Transfer     Bitrate
[  5]   0.00-10.00  sec  73.1 MBytes  61.3 Mbits/sec                  receiver

With unix.SOCK_SEQPACKET
ccepted connection from 192.168.100.123, port 49678
[  5] local 192.168.100.123 port 5201 connected to 192.168.100.123 port 49682
[ ID] Interval           Transfer     Bitrate
[  5]   0.00-1.00   sec   208 MBytes  1.75 Gbits/sec
[  5]   1.00-2.00   sec   213 MBytes  1.78 Gbits/sec
[  5]   2.00-3.00   sec   203 MBytes  1.70 Gbits/sec
[  5]   3.00-4.00   sec   207 MBytes  1.73 Gbits/sec
[  5]   4.00-5.00   sec   204 MBytes  1.71 Gbits/sec
[  5]   5.00-6.00   sec   205 MBytes  1.72 Gbits/sec
[  5]   6.00-7.00   sec   206 MBytes  1.73 Gbits/sec
[  5]   7.00-8.00   sec   212 MBytes  1.78 Gbits/sec
[  5]   8.00-9.00   sec   208 MBytes  1.74 Gbits/sec
[  5]   9.00-10.00  sec   213 MBytes  1.78 Gbits/sec
[  5]  10.00-10.00  sec  64.0 KBytes   300 Mbits/sec
- - - - - - - - - - - - - - - - - - - - - - - - -
[ ID] Interval           Transfer     Bitrate
[  5]   0.00-10.00  sec  2.03 GBytes  1.74 Gbits/sec                  receiver

pkg/container/tun.go

@@ -7,13 +7,13 @@ import (
 
 	"github.com/songgao/water"
 	"github.com/vishvananda/netlink"
+	"go.uber.org/multierr"
 )
 
 type TunDevice struct {
 	iface *water.Interface
 
-	writer io.Writer
-	reader io.Reader
+	bridge io.ReadWriteCloser
 }
 
 func New() (*TunDevice, error) {
@@ -29,13 +29,14 @@ func New() (*TunDevice, error) {
 
 	return &TunDevice{
 		iface:  ifce,
-		writer: os.NewFile(3, "writer"),
-		reader: os.NewFile(4, "reader"),
+		bridge: os.NewFile(3, "bridge"),
 	}, nil
 }
 
 func (t *TunDevice) Close() error {
-	return t.iface.Close()
+	return multierr.Combine(t.iface.Close(),
+		t.bridge.Close(),
+	)
 }
 
 func (t *TunDevice) SetupNetwork() error {
@@ -69,9 +70,9 @@ func (t *TunDevice) SetupNetwork() error {
 }
 
 func (t *TunDevice) ReadLoop() {
-	_, _ = io.Copy(t.writer, t.iface)
+	_, _ = io.Copy(t.bridge, t.iface)
 }
 
 func (t *TunDevice) WriteLoop() {
-	_, _ = io.Copy(t.iface, t.reader)
+	_, _ = io.Copy(t.iface, t.bridge)
 }

pkg/host/nic.go

@@ -84,7 +84,7 @@ func (t *TunDevice) AddHeader(local, remote tcpip.LinkAddress, protocol tcpip.Ne
 // r.LocalLinkAddress if it is provided.
 func (t *TunDevice) WritePacket(_ stack.RouteInfo, _ tcpip.NetworkProtocolNumber, pkt *stack.PacketBuffer) tcpip.Error {
 	view := buffer.NewVectorisedView(pkt.Size(), pkt.Views())
-	if _, err := t.writePipe.Write(view.ToView()); err != nil {
+	if _, err := t.bridge.Write(view.ToView()); err != nil {
 		return &tcpip.ErrInvalidEndpointState{}
 	}
 	return nil

pkg/host/reading.go

@@ -11,7 +11,7 @@ import (
 func (t *TunDevice) dispatchLoop() {
 	buf := make([]byte, MTU)
 	for {
-		n, err := t.readPipe.Read(buf)
+		n, err := t.bridge.Read(buf)
 		if err != nil {
 			return
 		}

pkg/host/setup.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"go.uber.org/multierr"
+	"golang.org/x/sys/unix"
 	"gvisor.dev/gvisor/pkg/tcpip"
 	"gvisor.dev/gvisor/pkg/tcpip/header"
 	"gvisor.dev/gvisor/pkg/tcpip/network/ipv4"
@@ -43,10 +44,9 @@ func DefaultOptions() Options {
 }
 
 type TunDevice struct {
-	readPipe   io.ReadCloser  // host
-	wReadPipe  *os.File       // child
-	writePipe  io.WriteCloser // host
-	rWritePipe *os.File       // child
+	bridge io.ReadWriteCloser
+
+	containerFd *os.File
 
 	stack      *stack.Stack
 	dispatcher stack.NetworkDispatcher
@@ -63,15 +63,14 @@ func New(opts Options) (out *TunDevice, err error) {
 		}),
 	}
 
-	out.readPipe, out.wReadPipe, err = Pipe()
-	if err != nil {
-		return nil, err
-	}
-	out.rWritePipe, out.writePipe, err = Pipe()
+	fds, err := unix.Socketpair(unix.AF_LOCAL, unix.SOCK_STREAM|unix.SOCK_SEQPACKET, 0)
 	if err != nil {
 		return nil, err
 	}
 
+	out.bridge = os.NewFile(uintptr(fds[0]), "bridge")
+	out.containerFd = os.NewFile(uintptr(fds[1]), "bridge-container")
+
 	out.stack.AddRoute(tcpip.Route{
 		Destination: header.IPv4EmptySubnet,
 		NIC:         nicID,
@@ -116,15 +115,12 @@ func New(opts Options) (out *TunDevice, err error) {
 }
 
 func (t *TunDevice) Close() error {
-	return multierr.Combine(t.readPipe.Close(),
-		t.wReadPipe.Close(),
-		t.writePipe.Close(),
-		t.rWritePipe.Close(),
+	return multierr.Combine(t.bridge.Close(),
 		t.udpHandler.Close(),
 		t.tcpHandler.Close(),
 	)
 }
 
 func (t *TunDevice) AttachToCmd(cmd *exec.Cmd) {
-	cmd.ExtraFiles = []*os.File{t.wReadPipe, t.rWritePipe}
+	cmd.ExtraFiles = []*os.File{t.containerFd}
 }