os: use poller for file I/O

This changes the os package to use the runtime poller for file I/O
where possible. When a system call blocks on a pollable descriptor,
the goroutine will be blocked on the poller but the thread will be
released to run other goroutines. When using a non-pollable
descriptor, the os package will continue to use thread-blocking system
calls as before.

For example, on GNU/Linux, the runtime poller uses epoll. epoll does
not support ordinary disk files, so they will continue to use blocking
I/O as before. The poller will be used for pipes.

Since this means that the poller is used for many more programs, this
modifies the runtime to only block waiting for the poller if there is
some goroutine that is waiting on the poller. Otherwise, there is no
point, as the poller will never make any goroutine ready. This
preserves the runtime's current simple deadlock detection.

This seems to crash FreeBSD systems, so it is disabled on FreeBSD.
This is issue 19093.

Using the poller on Windows requires opening the file with
FILE_FLAG_OVERLAPPED. We should only do that if we can remove that
flag if the program calls the Fd method. This is issue 19098.

Update #6817.
Update #7903.
Update #15021.
Update #18507.
Update #19093.
Update #19098.

Change-Id: Ia5197dcefa7c6fbcca97d19a6f8621b2abcbb1fe
Reviewed-on: https://go-review.googlesource.com/36800
Run-TryBot: Ian Lance Taylor <iant@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Russ Cox <rsc@golang.org>

src/internal/poll/fd_poll_nacl.go

@@ -85,3 +85,7 @@ func setDeadlineImpl(fd *FD, t time.Time, mode int) error {
 	fd.decref()
 	return nil
 }
+
+func PollDescriptor() uintptr {
+	return ^uintptr(0)
+}

src/internal/poll/fd_poll_runtime.go

@@ -17,6 +17,7 @@ import (
 func runtimeNano() int64
 
 func runtime_pollServerInit()
+func runtime_pollServerDescriptor() uintptr
 func runtime_pollOpen(fd uintptr) (uintptr, int)
 func runtime_pollClose(ctx uintptr)
 func runtime_pollWait(ctx uintptr, mode int) int
@@ -146,3 +147,9 @@ func setDeadlineImpl(fd *FD, t time.Time, mode int) error {
 	fd.decref()
 	return nil
 }
+
+// PollDescriptor returns the descriptor being used by the poller,
+// or ^uintptr(0) if there isn't one. This is only used for testing.
+func PollDescriptor() uintptr {
+	return runtime_pollServerDescriptor()
+}

src/internal/poll/fd_unix.go

@@ -365,7 +365,19 @@ func (fd *FD) ReadDirent(buf []byte) (int, error) {
 		return 0, err
 	}
 	defer fd.decref()
-	return syscall.ReadDirent(fd.Sysfd, buf)
+	for {
+		n, err := syscall.ReadDirent(fd.Sysfd, buf)
+		if err != nil {
+			n = 0
+			if err == syscall.EAGAIN {
+				if err = fd.pd.waitRead(); err == nil {
+					continue
+				}
+			}
+		}
+		// Do not call eofError; caller does not expect to see io.EOF.
+		return n, err
+	}
 }
 
 // Fchdir wraps syscall.Fchdir.

src/internal/poll/fd_windows.go

@@ -523,13 +523,15 @@ func (fd *FD) Pread(b []byte, off int64) (int, error) {
 	var done uint32
 	e = syscall.ReadFile(fd.Sysfd, b, &done, &o)
 	if e != nil {
+		done = 0
 		if e == syscall.ERROR_HANDLE_EOF {
-			// end of file
-			return 0, nil
+			e = io.EOF
 		}
-		return 0, e
 	}
-	return int(done), nil
+	if len(b) != 0 {
+		e = fd.eofError(int(done), e)
+	}
+	return int(done), e
 }
 
 func (fd *FD) RecvFrom(buf []byte) (int, syscall.Sockaddr, error) {

src/os/dir_unix.go

@@ -8,6 +8,7 @@ package os
 
 import (
 	"io"
+	"runtime"
 	"syscall"
 )
 
@@ -63,9 +64,10 @@ func (f *File) readdirnames(n int) (names []string, err error) {
 		if d.bufp >= d.nbuf {
 			d.bufp = 0
 			var errno error
-			d.nbuf, errno = fixCount(syscall.ReadDirent(f.fd, d.buf))
+			d.nbuf, errno = f.pfd.ReadDirent(d.buf)
+			runtime.KeepAlive(f)
 			if errno != nil {
-				return names, NewSyscallError("readdirent", errno)
+				return names, wrapSyscallError("readdirent", errno)
 			}
 			if d.nbuf <= 0 {
 				break // EOF

src/os/dir_windows.go

@@ -6,6 +6,7 @@ package os
 
 import (
 	"io"
+	"runtime"
 	"syscall"
 )
 
@@ -16,7 +17,7 @@ func (file *File) readdir(n int) (fi []FileInfo, err error) {
 	if !file.isdir() {
 		return nil, &PathError{"Readdir", file.name, syscall.ENOTDIR}
 	}
-	if !file.dirinfo.isempty && file.fd == syscall.InvalidHandle {
+	if !file.dirinfo.isempty && file.pfd.Sysfd == syscall.InvalidHandle {
 		return nil, syscall.EINVAL
 	}
 	wantAll := n <= 0
@@ -29,7 +30,8 @@ func (file *File) readdir(n int) (fi []FileInfo, err error) {
 	d := &file.dirinfo.data
 	for n != 0 && !file.dirinfo.isempty {
 		if file.dirinfo.needdata {
-			e := syscall.FindNextFile(file.fd, d)
+			e := file.pfd.FindNextFile(d)
+			runtime.KeepAlive(file)
 			if e != nil {
 				if e == syscall.ERROR_NO_MORE_FILES {
 					break

src/os/error_posix.go

@@ -0,0 +1,18 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows
+
+package os
+
+import "syscall"
+
+// wrapSyscallError takes an error and a syscall name. If the error is
+// a syscall.Errno, it wraps it in a os.SyscallError using the syscall name.
+func wrapSyscallError(name string, err error) error {
+	if _, ok := err.(syscall.Errno); ok {
+		err = NewSyscallError(name, err)
+	}
+	return err
+}

src/os/exec/exec_test.go

@@ -12,6 +12,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"internal/poll"
 	"internal/testenv"
 	"io"
 	"io/ioutil"
@@ -369,12 +370,16 @@ var testedAlreadyLeaked = false
 
 // basefds returns the number of expected file descriptors
 // to be present in a process at start.
+// stdin, stdout, stderr, epoll/kqueue
 func basefds() uintptr {
 	return os.Stderr.Fd() + 1
 }
 
 func closeUnexpectedFds(t *testing.T, m string) {
 	for fd := basefds(); fd <= 101; fd++ {
+		if fd == poll.PollDescriptor() {
+			continue
+		}
 		err := os.NewFile(fd, "").Close()
 		if err == nil {
 			t.Logf("%s: Something already leaked - closed fd %d", m, fd)
@@ -732,6 +737,9 @@ func TestHelperProcess(*testing.T) {
 			// Now verify that there are no other open fds.
 			var files []*os.File
 			for wantfd := basefds() + 1; wantfd <= 100; wantfd++ {
+				if wantfd == poll.PollDescriptor() {
+					continue
+				}
 				f, err := os.Open(os.Args[0])
 				if err != nil {
 					fmt.Printf("error opening file with expected fd %d: %v", wantfd, err)

src/os/export_windows_test.go

@@ -7,7 +7,6 @@ package os
 // Export for testing.
 
 var (
-	FixLongPath     = fixLongPath
-	NewConsoleFile  = newConsoleFile
-	ReadConsoleFunc = &readConsole
+	FixLongPath    = fixLongPath
+	NewConsoleFile = newConsoleFile
 )

src/os/file.go

@@ -99,11 +99,12 @@ func (f *File) Read(b []byte) (n int, err error) {
 		return 0, err
 	}
 	n, e := f.read(b)
-	if n == 0 && len(b) > 0 && e == nil {
-		return 0, io.EOF
-	}
 	if e != nil {
-		err = &PathError{"read", f.name, e}
+		if e == io.EOF {
+			err = e
+		} else {
+			err = &PathError{"read", f.name, e}
+		}
 	}
 	return n, err
 }
@@ -118,11 +119,12 @@ func (f *File) ReadAt(b []byte, off int64) (n int, err error) {
 	}
 	for len(b) > 0 {
 		m, e := f.pread(b, off)
-		if m == 0 && e == nil {
-			return n, io.EOF
-		}
 		if e != nil {
-			err = &PathError{"read", f.name, e}
+			if e == io.EOF {
+				err = e
+			} else {
+				err = &PathError{"read", f.name, e}
+			}
 			break
 		}
 		n += m
@@ -226,19 +228,6 @@ func Chdir(dir string) error {
 	return nil
 }
 
-// Chdir changes the current working directory to the file,
-// which must be a directory.
-// If there is an error, it will be of type *PathError.
-func (f *File) Chdir() error {
-	if err := f.checkValid("chdir"); err != nil {
-		return err
-	}
-	if e := syscall.Fchdir(f.fd); e != nil {
-		return &PathError{"chdir", f.name, e}
-	}
-	return nil
-}
-
 // Open opens the named file for reading. If successful, methods on
 // the returned file can be used for reading; the associated file
 // descriptor has mode O_RDONLY.
@@ -275,15 +264,3 @@ func fixCount(n int, err error) (int, error) {
 	}
 	return n, err
 }
-
-// checkValid checks whether f is valid for use.
-// If not, it returns an appropriate error, perhaps incorporating the operation name op.
-func (f *File) checkValid(op string) error {
-	if f == nil {
-		return ErrInvalid
-	}
-	if f.fd == badFd {
-		return &PathError{op, f.name, ErrClosed}
-	}
-	return nil
-}

src/os/file_plan9.go

@@ -244,14 +244,22 @@ func (f *File) Sync() error {
 // read reads up to len(b) bytes from the File.
 // It returns the number of bytes read and an error, if any.
 func (f *File) read(b []byte) (n int, err error) {
-	return fixCount(syscall.Read(f.fd, b))
+	n, e := fixCount(syscall.Read(f.fd, b))
+	if n == 0 && len(b) > 0 && e == nil {
+		return 0, io.EOF
+	}
+	return n, e
 }
 
 // pread reads len(b) bytes from the File starting at byte offset off.
 // It returns the number of bytes read and the error, if any.
 // EOF is signaled by a zero count with err set to nil.
 func (f *File) pread(b []byte, off int64) (n int, err error) {
-	return fixCount(syscall.Pread(f.fd, b, off))
+	n, e := fixCount(syscall.Pread(f.fd, b, off))
+	if n == 0 && len(b) > 0 && e == nil {
+		return 0, io.EOF
+	}
+	return n, e
 }
 
 // write writes len(b) bytes to the File.
@@ -472,3 +480,28 @@ func (f *File) Chown(uid, gid int) error {
 func TempDir() string {
 	return "/tmp"
 }
+
+// Chdir changes the current working directory to the file,
+// which must be a directory.
+// If there is an error, it will be of type *PathError.
+func (f *File) Chdir() error {
+	if err := f.checkValid("chdir"); err != nil {
+		return err
+	}
+	if e := syscall.Fchdir(f.fd); e != nil {
+		return &PathError{"chdir", f.name, e}
+	}
+	return nil
+}
+
+// checkValid checks whether f is valid for use.
+// If not, it returns an appropriate error, perhaps incorporating the operation name op.
+func (f *File) checkValid(op string) error {
+	if f == nil {
+		return ErrInvalid
+	}
+	if f.fd == badFd {
+		return &PathError{op, f.name, ErrClosed}
+	}
+	return nil
+}

src/os/file_posix.go

@@ -7,6 +7,7 @@
 package os
 
 import (
+	"runtime"
 	"syscall"
 	"time"
 )
@@ -60,9 +61,10 @@ func (f *File) Chmod(mode FileMode) error {
 	if err := f.checkValid("chmod"); err != nil {
 		return err
 	}
-	if e := syscall.Fchmod(f.fd, syscallMode(mode)); e != nil {
+	if e := f.pfd.Fchmod(syscallMode(mode)); e != nil {
 		return &PathError{"chmod", f.name, e}
 	}
+	runtime.KeepAlive(f)
 	return nil
 }
 
@@ -92,9 +94,10 @@ func (f *File) Chown(uid, gid int) error {
 	if err := f.checkValid("chown"); err != nil {
 		return err
 	}
-	if e := syscall.Fchown(f.fd, uid, gid); e != nil {
+	if e := f.pfd.Fchown(uid, gid); e != nil {
 		return &PathError{"chown", f.name, e}
 	}
+	runtime.KeepAlive(f)
 	return nil
 }
 
@@ -105,9 +108,10 @@ func (f *File) Truncate(size int64) error {
 	if err := f.checkValid("truncate"); err != nil {
 		return err
 	}
-	if e := syscall.Ftruncate(f.fd, size); e != nil {
+	if e := f.pfd.Ftruncate(size); e != nil {
 		return &PathError{"truncate", f.name, e}
 	}
+	runtime.KeepAlive(f)
 	return nil
 }
 
@@ -118,9 +122,10 @@ func (f *File) Sync() error {
 	if err := f.checkValid("sync"); err != nil {
 		return err
 	}
-	if e := syscall.Fsync(f.fd); e != nil {
+	if e := f.pfd.Fsync(); e != nil {
 		return &PathError{"sync", f.name, e}
 	}
+	runtime.KeepAlive(f)
 	return nil
 }
 
@@ -139,3 +144,29 @@ func Chtimes(name string, atime time.Time, mtime time.Time) error {
 	}
 	return nil
 }
+
+// Chdir changes the current working directory to the file,
+// which must be a directory.
+// If there is an error, it will be of type *PathError.
+func (f *File) Chdir() error {
+	if err := f.checkValid("chdir"); err != nil {
+		return err
+	}
+	if e := f.pfd.Fchdir(); e != nil {
+		return &PathError{"chdir", f.name, e}
+	}
+	runtime.KeepAlive(f)
+	return nil
+}
+
+// checkValid checks whether f is valid for use.
+// If not, it returns an appropriate error, perhaps incorporating the operation name op.
+func (f *File) checkValid(op string) error {
+	if f == nil {
+		return ErrInvalid
+	}
+	if f.pfd.Sysfd == badFd {
+		return &PathError{op, f.name, ErrClosed}
+	}
+	return nil
+}