impl_unix.go

// +build darwin freebsd dragonfly netbsd openbsd linux

package reuseport

import (
	"context"
	"golang.org/x/sys/unix"
	"net"
	"os"
	"strconv"
	"syscall"
	"time"

	"github.com/libp2p/go-reuseport/singlepoll"
	sockaddrnet "github.com/libp2p/go-sockaddr/net"
)

const (
	filePrefix = "port."
)

// Wrapper around the socket system call that marks the returned file
// descriptor as nonblocking and close-on-exec.
func socket(family, socktype, protocol int) (fd int, err error) {
	syscall.ForkLock.RLock()
	fd, err = unix.Socket(family, socktype, protocol)
	if err == nil {
		unix.CloseOnExec(fd)
	}
	syscall.ForkLock.RUnlock()

	if err != nil {
		return -1, err
	}

	// cant set it until after connect
	// if err = unix.SetNonblock(fd, true); err != nil {
	// 	unix.Close(fd)
	// 	return -1, err
	// }

	if err = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_REUSEADDR, 1); err != nil {
		unix.Close(fd)
		return -1, err
	}

	if err = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil {
		unix.Close(fd)
		return -1, err
	}

	// set setLinger to 5 as reusing exact same (srcip:srcport, dstip:dstport)
	// will otherwise fail on connect.
	if err = setLinger(fd, 5); err != nil {
		unix.Close(fd)
		return -1, err
	}

	return fd, nil
}

func dial(ctx context.Context, dialer net.Dialer, netw, addr string) (c net.Conn, err error) {
	var (
		fd             int
		lfamily        int
		rfamily        int
		socktype       int
		lprotocol      int
		rprotocol      int
		file           *os.File
		deadline       time.Time
		remoteSockaddr unix.Sockaddr
		localSockaddr  unix.Sockaddr
	)

	netAddr, err := ResolveAddr(netw, addr)
	if err != nil {
		return nil, err
	}

	switch netAddr.(type) {
	case *net.TCPAddr, *net.UDPAddr:
	default:
		return nil, ErrUnsupportedProtocol
	}

	switch {
	case !dialer.Deadline.IsZero():
		deadline = dialer.Deadline
	case dialer.Timeout != 0:
		deadline = time.Now().Add(dialer.Timeout)
	}

	ctxdeadline, ok := ctx.Deadline()
	if ok && ctxdeadline.Before(deadline) {
		deadline = ctxdeadline
	}

	localSockaddr = sockaddrnet.NetAddrToSockaddr(dialer.LocalAddr)
	remoteSockaddr = sockaddrnet.NetAddrToSockaddr(netAddr)

	rfamily = sockaddrnet.NetAddrAF(netAddr)
	rprotocol = sockaddrnet.NetAddrIPPROTO(netAddr)
	socktype = sockaddrnet.NetAddrSOCK(netAddr)

	if dialer.LocalAddr != nil {
		switch dialer.LocalAddr.(type) {
		case *net.TCPAddr, *net.UDPAddr:
		default:
			return nil, ErrUnsupportedProtocol
		}

		// check family and protocols match.
		lfamily = sockaddrnet.NetAddrAF(dialer.LocalAddr)
		lprotocol = sockaddrnet.NetAddrIPPROTO(dialer.LocalAddr)
		if lfamily != rfamily || lprotocol != rprotocol {
			return nil, &net.AddrError{Err: "unexpected address type", Addr: netAddr.String()}
		}
	}

	// look at dialTCP in http://golang.org/src/net/tcpsock_posix.go  .... !
	// here we just try again 3 times.
	for i := 0; i < 3; i++ {
		if !deadline.IsZero() && deadline.Before(time.Now()) {
			err = errTimeout
			break
		}

		if fd, err = socket(rfamily, socktype, rprotocol); err != nil {
			return nil, err
		}

		if localSockaddr != nil {
			if err = unix.Bind(fd, localSockaddr); err != nil {
				unix.Close(fd)
				return nil, err
			}
		}

		if err = unix.SetNonblock(fd, true); err != nil {
			unix.Close(fd)
			return nil, err
		}

		if err = connect(ctx, fd, remoteSockaddr, deadline); err != nil {
			unix.Close(fd)
			if ctx.Err() != nil {
				return nil, ctx.Err()
			}
			continue // try again.
		}

		break
	}
	if err != nil {
		return nil, err
	}

	if rprotocol == unix.IPPROTO_TCP {
		sa, err := unix.Getsockname(fd)
		if err != nil {
			unix.Close(fd)
			return nil, err
		}
		ra, err := unix.Getpeername(fd)
		if err != nil {
			unix.Close(fd)
			return nil, err
		}

		// Need to call setLinger 0. Otherwise, the close will block for the linger period. Linux bug?
		switch s := sa.(type) {
		case *unix.SockaddrInet4:
			if r, ok := ra.(*unix.SockaddrInet4); ok && r.Addr == s.Addr && r.Port == s.Port {
				setLinger(fd, 0)
				unix.Close(fd)
				return nil, ErrDialSelf
			}
		case *unix.SockaddrInet6:
			if r, ok := ra.(*unix.SockaddrInet6); ok && r.Addr == s.Addr && r.Port == s.Port && r.ZoneId == s.ZoneId {
				setLinger(fd, 0)
				unix.Close(fd)
				return nil, ErrDialSelf
			}
		}

		//  by default golang/net sets TCP no delay to true.
		if err = setNoDelay(fd, true); err != nil {
			unix.Close(fd)
			return nil, err
		}
	}

	// NOTE:XXX: never call unix.Close on fd after os.NewFile
	file = os.NewFile(uintptr(fd), filePrefix+strconv.Itoa(os.Getpid()))
	fd = -1 // so we don't touch it, we handled the control to Golang with NewFile
	if c, err = net.FileConn(file); err != nil {
		_ = file.Close() // shouldn't error either way
		return nil, err
	}

	if err = file.Close(); err != nil {
		c.Close()
		return nil, err
	}

	// c = wrapConnWithRemoteAddr(c, netAddr)
	return c, err
}

func listen(netw, addr string) (fd int, err error) {
	var (
		family   int
		socktype int
		protocol int
		sockaddr unix.Sockaddr
	)

	netAddr, err := ResolveAddr(netw, addr)
	if err != nil {
		return -1, err
	}

	switch netAddr.(type) {
	case *net.TCPAddr, *net.UDPAddr:
	default:
		return -1, ErrUnsupportedProtocol
	}

	family = sockaddrnet.NetAddrAF(netAddr)
	protocol = sockaddrnet.NetAddrIPPROTO(netAddr)
	sockaddr = sockaddrnet.NetAddrToSockaddr(netAddr)
	socktype = sockaddrnet.NetAddrSOCK(netAddr)

	if fd, err = socket(family, socktype, protocol); err != nil {
		return -1, err
	}

	if err = unix.Bind(fd, sockaddr); err != nil {
		unix.Close(fd)
		return -1, err
	}

	if protocol == unix.IPPROTO_TCP {
		//  by default golang/net sets TCP no delay to true.
		if err = setNoDelay(fd, true); err != nil {
			unix.Close(fd)
			return -1, err
		}
	}

	if err = unix.SetNonblock(fd, true); err != nil {
		unix.Close(fd)
		return -1, err
	}

	return fd, nil
}

func listenStream(netw, addr string) (l net.Listener, err error) {
	var (
		file *os.File
	)

	fd, err := listen(netw, addr)
	if err != nil {
		return nil, err
	}

	// Set backlog size to the maximum
	if err = unix.Listen(fd, unix.SOMAXCONN); err != nil {
		unix.Close(fd)
		return nil, err
	}

	file = os.NewFile(uintptr(fd), filePrefix+strconv.Itoa(os.Getpid()))
	if l, err = net.FileListener(file); err != nil {
		unix.Close(fd)
		return nil, err
	}

	if err = file.Close(); err != nil {
		unix.Close(fd)
		l.Close()
		return nil, err
	}

	return l, err
}

func listenPacket(netw, addr string) (p net.PacketConn, err error) {
	var (
		file *os.File
	)

	fd, err := listen(netw, addr)
	if err != nil {
		return nil, err
	}

	file = os.NewFile(uintptr(fd), filePrefix+strconv.Itoa(os.Getpid()))
	if p, err = net.FilePacketConn(file); err != nil {
		unix.Close(fd)
		return nil, err
	}

	if err = file.Close(); err != nil {
		unix.Close(fd)
		p.Close()
		return nil, err
	}

	return p, err
}

func listenUDP(netw, addr string) (c net.Conn, err error) {
	var (
		file *os.File
	)

	fd, err := listen(netw, addr)
	if err != nil {
		return nil, err
	}

	file = os.NewFile(uintptr(fd), filePrefix+strconv.Itoa(os.Getpid()))
	if c, err = net.FileConn(file); err != nil {
		unix.Close(fd)
		return nil, err
	}

	if err = file.Close(); err != nil {
		unix.Close(fd)
		return nil, err
	}

	return c, err
}

// this is close to the connect() function inside stdlib/net
func connect(ctx context.Context, fd int, ra unix.Sockaddr, deadline time.Time) error {
	if !deadline.IsZero() {
		var cancel context.CancelFunc
		ctx, cancel = context.WithDeadline(ctx, deadline)
		defer cancel()
	}

	switch err := unix.Connect(fd, ra); err {
	case unix.EINPROGRESS, unix.EALREADY, unix.EINTR:
	case nil, unix.EISCONN:
		if !deadline.IsZero() && deadline.Before(time.Now()) {
			return errTimeout
		}
		if ctx.Err() != nil {
			return ctx.Err()
		}
		return nil
	default:
		return err
	}

	for {
		if err := singlepoll.PollPark(ctx, fd, "w"); err != nil {
			return err
		}

		// if err := fd.pd.WaitWrite(); err != nil {
		// 	return err
		// }
		// i'd use the above fd.pd.WaitWrite to poll io correctly, just like net sockets...
		// but of course, it uses the damn runtime_* functions that _cannot_ be used by
		// non-go-stdlib source... seriously guys, this is not nice.
		// we're relegated to using unix.Select (what nightmare that is) or using
		// a simple but totally bogus time-based wait. such garbage.
		nerr, err := unix.GetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_ERROR)
		if err != nil {
			return err
		}
		switch err = syscall.Errno(nerr); err {
		case unix.EINPROGRESS, unix.EALREADY, unix.EINTR:
			continue
		case syscall.Errno(0), unix.EISCONN:
			if !deadline.IsZero() && deadline.Before(time.Now()) {
				return errTimeout
			}
			return nil
		default:
			return err
		}
	}
}

var errTimeout = &timeoutError{}

type timeoutError struct{}

func (e *timeoutError) Error() string   { return "i/o timeout" }
func (e *timeoutError) Timeout() bool   { return true }
func (e *timeoutError) Temporary() bool { return true }