syscall: restore original NOFILE rlimit in child process

If we increased the NOFILE rlimit when starting the program,
restore the original rlimit when forking a child process.

For #46279

Change-Id: Ia5d2af9ef435e5932965c15eec2e428d2130d230
Reviewed-on: https://go-review.googlesource.com/c/go/+/476097
Reviewed-by: Bryan Mills <bcmills@google.com>
Reviewed-by: Ian Lance Taylor <iant@google.com>
TryBot-Bypass: Ian Lance Taylor <iant@google.com>

src/runtime/syscall2_solaris.go

@@ -17,6 +17,7 @@ import _ "unsafe" // for go:linkname
 //go:cgo_import_dynamic libc_ioctl ioctl "libc.so"
 //go:cgo_import_dynamic libc_setgid setgid "libc.so"
 //go:cgo_import_dynamic libc_setgroups setgroups "libc.so"
+//go:cgo_import_dynamic libc_setrlimit setrlimit "libc.so"
 //go:cgo_import_dynamic libc_setsid setsid "libc.so"
 //go:cgo_import_dynamic libc_setuid setuid "libc.so"
 //go:cgo_import_dynamic libc_setpgid setpgid "libc.so"
@@ -34,6 +35,7 @@ import _ "unsafe" // for go:linkname
 //go:linkname libc_ioctl libc_ioctl
 //go:linkname libc_setgid libc_setgid
 //go:linkname libc_setgroups libc_setgroups
+//go:linkname libc_setrlimit libc_setrlimit
 //go:linkname libc_setsid libc_setsid
 //go:linkname libc_setuid libc_setuid
 //go:linkname libc_setpgid libc_setpgid

src/runtime/syscall_aix.go

@@ -18,6 +18,7 @@ import "unsafe"
 //go:cgo_import_dynamic libc_ioctl ioctl "libc.a/shr_64.o"
 //go:cgo_import_dynamic libc_setgid setgid "libc.a/shr_64.o"
 //go:cgo_import_dynamic libc_setgroups setgroups "libc.a/shr_64.o"
+//go:cgo_import_dynamic libc_setrlimit setrlimit "libc.a/shr_64.o"
 //go:cgo_import_dynamic libc_setsid setsid "libc.a/shr_64.o"
 //go:cgo_import_dynamic libc_setuid setuid "libc.a/shr_64.o"
 //go:cgo_import_dynamic libc_setpgid setpgid "libc.a/shr_64.o"
@@ -31,6 +32,7 @@ import "unsafe"
 //go:linkname libc_ioctl libc_ioctl
 //go:linkname libc_setgid libc_setgid
 //go:linkname libc_setgroups libc_setgroups
+//go:linkname libc_setrlimit libc_setrlimit
 //go:linkname libc_setsid libc_setsid
 //go:linkname libc_setuid libc_setuid
 //go:linkname libc_setpgid libc_setpgid
@@ -45,6 +47,7 @@ var (
 	libc_ioctl,
 	libc_setgid,
 	libc_setgroups,
+	libc_setrlimit,
 	libc_setsid,
 	libc_setuid,
 	libc_setpgid libFunc
@@ -199,6 +202,13 @@ func syscall_setgroups1(ngid, gid uintptr) (err uintptr) {
 	return
 }
 
+//go:linkname syscall_setrlimit1 syscall.setrlimit1
+//go:nosplit
+func syscall_setrlimit1(which uintptr, lim unsafe.Pointer) (err uintptr) {
+	_, err = syscall2(&libc_setrlimit, which, uintptr(lim))
+	return
+}
+
 //go:linkname syscall_setsid syscall.setsid
 //go:nosplit
 func syscall_setsid() (pid, err uintptr) {

src/runtime/syscall_solaris.go

@@ -18,6 +18,7 @@ var (
 	libc_ioctl,
 	libc_setgid,
 	libc_setgroups,
+	libc_setrlimit,
 	libc_setsid,
 	libc_setuid,
 	libc_setpgid,
@@ -234,6 +235,19 @@ func syscall_setgroups(ngid, gid uintptr) (err uintptr) {
 	return call.err
 }
 
+//go:nosplit
+//go:linkname syscall_setrlimit
+//go:cgo_unsafe_args
+func syscall_setrlimit(which uintptr, lim unsafe.Pointer) (err uintptr) {
+	call := libcall{
+		fn:   uintptr(unsafe.Pointer(&libc_setrlimit)),
+		n:    2,
+		args: uintptr(unsafe.Pointer(&which)),
+	}
+	asmcgocall(unsafe.Pointer(&asmsysvicall6x), unsafe.Pointer(&call))
+	return call.err
+}
+
 //go:nosplit
 //go:linkname syscall_setsid
 func syscall_setsid() (pid, err uintptr) {

src/syscall/asm_solaris_amd64.s

@@ -60,6 +60,9 @@ TEXT ·setgid(SB),NOSPLIT,$0
 TEXT ·setgroups1(SB),NOSPLIT,$0
 	JMP	runtime·syscall_setgroups(SB)
 
+TEXT ·setrlimit1(SB),NOSPLIT,$0
+	JMP	runtime·syscall_setrlimit(SB)
+
 TEXT ·setsid(SB),NOSPLIT,$0
 	JMP	runtime·syscall_setsid(SB)
 

src/syscall/exec_bsd.go

@@ -64,6 +64,8 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr
 		ngroups, groups uintptr
 	)
 
+	rlim, rlimOK := origRlimitNofile.Load().(Rlimit)
+
 	// guard against side effects of shuffling fds below.
 	// Make sure that nextfd is beyond any currently open files so
 	// that we can't run the risk of overwriting any of them.
@@ -273,6 +275,11 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr
 		}
 	}
 
+	// Restore original rlimit.
+	if rlimOK && rlim.Cur != 0 {
+		RawSyscall(SYS_SETRLIMIT, uintptr(RLIMIT_NOFILE), uintptr(unsafe.Pointer(&rlim)), 0)
+	}
+
 	// Time to exec.
 	_, _, err1 = RawSyscall(SYS_EXECVE,
 		uintptr(unsafe.Pointer(argv0)),

src/syscall/exec_freebsd.go

@@ -71,6 +71,8 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr
 		upid            uintptr
 	)
 
+	rlim, rlimOK := origRlimitNofile.Load().(Rlimit)
+
 	// Record parent PID so child can test if it has died.
 	ppid, _, _ := RawSyscall(SYS_GETPID, 0, 0, 0)
 
@@ -297,6 +299,11 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr
 		}
 	}
 
+	// Restore original rlimit.
+	if rlimOK && rlim.Cur != 0 {
+		RawSyscall(SYS_SETRLIMIT, uintptr(RLIMIT_NOFILE), uintptr(unsafe.Pointer(&rlim)), 0)
+	}
+
 	// Time to exec.
 	_, _, err1 = RawSyscall(SYS_EXECVE,
 		uintptr(unsafe.Pointer(argv0)),

src/syscall/exec_libc.go

@@ -53,6 +53,7 @@ func getpid() (pid uintptr, err Errno)
 func ioctl(fd uintptr, req uintptr, arg uintptr) (err Errno)
 func setgid(gid uintptr) (err Errno)
 func setgroups1(ngid uintptr, gid uintptr) (err Errno)
+func setrlimit1(which uintptr, lim unsafe.Pointer) (err Errno)
 func setsid() (pid uintptr, err Errno)
 func setuid(uid uintptr) (err Errno)
 func setpgid(pid uintptr, pgid uintptr) (err Errno)
@@ -90,6 +91,8 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr
 		ngroups, groups uintptr
 	)
 
+	rlim, rlimOK := origRlimitNofile.Load().(Rlimit)
+
 	// guard against side effects of shuffling fds below.
 	// Make sure that nextfd is beyond any currently open files so
 	// that we can't run the risk of overwriting any of them.
@@ -292,6 +295,11 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr
 		}
 	}
 
+	// Restore original rlimit.
+	if rlimOK && rlim.Cur != 0 {
+		setrlimit1(RLIMIT_NOFILE, unsafe.Pointer(&rlim))
+	}
+
 	// Time to exec.
 	err1 = execve(
 		uintptr(unsafe.Pointer(argv0)),

src/syscall/exec_libc2.go

@@ -65,6 +65,8 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr
 		ngroups, groups uintptr
 	)
 
+	rlim, rlimOK := origRlimitNofile.Load().(Rlimit)
+
 	// guard against side effects of shuffling fds below.
 	// Make sure that nextfd is beyond any currently open files so
 	// that we can't run the risk of overwriting any of them.
@@ -269,6 +271,11 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr
 		}
 	}
 
+	// Restore original rlimit.
+	if rlimOK && rlim.Cur != 0 {
+		rawSyscall(abi.FuncPCABI0(libc_setrlimit_trampoline), uintptr(RLIMIT_NOFILE), uintptr(unsafe.Pointer(&rlim)), 0)
+	}
+
 	// Time to exec.
 	_, _, err1 = rawSyscall(abi.FuncPCABI0(libc_execve_trampoline),
 		uintptr(unsafe.Pointer(argv0)),

src/syscall/exec_linux.go

@@ -240,6 +240,8 @@ func forkAndExecInChild1(argv0 *byte, argv, envv []*byte, chroot, dir *byte, att
 		c                         uintptr
 	)
 
+	rlim, rlimOK := origRlimitNofile.Load().(Rlimit)
+
 	if sys.UidMappings != nil {
 		puid = []byte("/proc/self/uid_map\000")
 		uidmap = formatIDMappings(sys.UidMappings)
@@ -609,6 +611,11 @@ func forkAndExecInChild1(argv0 *byte, argv, envv []*byte, chroot, dir *byte, att
 		}
 	}
 
+	// Restore original rlimit.
+	if rlimOK && rlim.Cur != 0 {
+		rawSetrlimit(RLIMIT_NOFILE, &rlim)
+	}
+
 	// Enable tracing if requested.
 	// Do this right before exec so that we don't unnecessarily trace the runtime
 	// setting up after the fork. See issue #21428.

src/syscall/exec_unix.go

@@ -282,6 +282,11 @@ func Exec(argv0 string, argv []string, envv []string) (err error) {
 	}
 	runtime_BeforeExec()
 
+	rlim, rlimOK := origRlimitNofile.Load().(Rlimit)
+	if rlimOK && rlim.Cur != 0 {
+		Setrlimit(RLIMIT_NOFILE, &rlim)
+	}
+
 	var err1 error
 	if runtime.GOOS == "solaris" || runtime.GOOS == "illumos" || runtime.GOOS == "aix" {
 		// RawSyscall should never be used on Solaris, illumos, or AIX.

src/syscall/exec_unix_test.go

@@ -7,12 +7,15 @@
 package syscall_test
 
 import (
+	"bytes"
+	"fmt"
 	"internal/testenv"
 	"io"
 	"math/rand"
 	"os"
 	"os/exec"
 	"os/signal"
+	"strconv"
 	"syscall"
 	"testing"
 	"time"
@@ -345,3 +348,42 @@ func TestExecHelper(t *testing.T) {
 
 	t.Error("syscall.Exec returned")
 }
+
+// Test that rlimit values are restored by exec.
+func TestRlimitRestored(t *testing.T) {
+	if os.Getenv("GO_WANT_HELPER_PROCESS") != "" {
+		fmt.Println(syscall.OrigRlimitNofile().Cur)
+		os.Exit(0)
+	}
+
+	orig := syscall.OrigRlimitNofile()
+	if orig.Cur == 0 {
+		t.Skip("skipping test because rlimit not adjusted at startup")
+	}
+
+	executable, err := os.Executable()
+	if err != nil {
+		executable = os.Args[0]
+	}
+
+	cmd := testenv.Command(t, executable, "-test.run=TestRlimitRestored")
+	cmd = testenv.CleanCmdEnv(cmd)
+	cmd.Env = append(cmd.Env, "GO_WANT_HELPER_PROCESS=1")
+
+	out, err := cmd.CombinedOutput()
+	if len(out) > 0 {
+		t.Logf("%s", out)
+	}
+	if err != nil {
+		t.Fatalf("subprocess failed: %v", err)
+	}
+	s := string(bytes.TrimSpace(out))
+	v, err := strconv.ParseUint(s, 10, 64)
+	if err != nil {
+		t.Fatalf("could not parse %q as number: %v", s, v)
+	}
+
+	if v != uint64(orig.Cur) {
+		t.Errorf("exec rlimit = %d, want %d", v, orig)
+	}
+}

src/syscall/export_rlimit_test.go

@@ -0,0 +1,14 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build unix
+
+package syscall
+
+func OrigRlimitNofile() Rlimit {
+	if rlim, ok := origRlimitNofile.Load().(Rlimit); ok {
+		return rlim
+	}
+	return Rlimit{0, 0}
+}

src/syscall/rlimit.go

@@ -6,6 +6,15 @@
 
 package syscall
 
+import (
+	"sync/atomic"
+)
+
+// origRlimitNofile, if not {0, 0}, is the original soft RLIMIT_NOFILE.
+// When we can assume that we are bootstrapping with Go 1.19,
+// this can be atomic.Pointer[Rlimit].
+var origRlimitNofile atomic.Value // of Rlimit
+
 // Some systems set an artificially low soft limit on open file count, for compatibility
 // with code that uses select and its hard-coded maximum file descriptor
 // (limited by the size of fd_set).
@@ -23,8 +32,19 @@ package syscall
 func init() {
 	var lim Rlimit
 	if err := Getrlimit(RLIMIT_NOFILE, &lim); err == nil && lim.Cur != lim.Max {
+		origRlimitNofile.Store(lim)
 		lim.Cur = lim.Max
 		adjustFileLimit(&lim)
-		Setrlimit(RLIMIT_NOFILE, &lim)
+		setrlimit(RLIMIT_NOFILE, &lim)
+	}
+}
+
+func Setrlimit(resource int, rlim *Rlimit) error {
+	err := setrlimit(resource, rlim)
+	if err == nil && resource == RLIMIT_NOFILE {
+		// Store zeroes in origRlimitNofile to tell StartProcess
+		// to not adjust the rlimit in the child process.
+		origRlimitNofile.Store(Rlimit{0, 0})
 	}
+	return err
 }

src/syscall/syscall_aix.go

@@ -633,7 +633,7 @@ func PtraceDetach(pid int) (err error) { return ptrace64(PT_DETACH, int64(pid),
 //sys	Setpriority(which int, who int, prio int) (err error)
 //sysnb	Setregid(rgid int, egid int) (err error)
 //sysnb	Setreuid(ruid int, euid int) (err error)
-//sysnb	Setrlimit(which int, lim *Rlimit) (err error)
+//sysnb	setrlimit(which int, lim *Rlimit) (err error)
 //sys	Stat(path string, stat *Stat_t) (err error)
 //sys	Statfs(path string, buf *Statfs_t) (err error)
 //sys	Symlink(path string, link string) (err error)

src/syscall/syscall_darwin.go

@@ -195,7 +195,7 @@ func Kill(pid int, signum Signal) (err error) { return kill(pid, int(signum), 1)
 //sys	Setprivexec(flag int) (err error)
 //sysnb	Setregid(rgid int, egid int) (err error)
 //sysnb	Setreuid(ruid int, euid int) (err error)
-//sysnb	Setrlimit(which int, lim *Rlimit) (err error)
+//sysnb	setrlimit(which int, lim *Rlimit) (err error)
 //sysnb	Setsid() (pid int, err error)
 //sysnb	Settimeofday(tp *Timeval) (err error)
 //sysnb	Setuid(uid int) (err error)

src/syscall/syscall_dragonfly.go

@@ -240,7 +240,7 @@ func Getfsstat(buf []Statfs_t, flags int) (n int, err error) {
 //sys	Setpriority(which int, who int, prio int) (err error)
 //sysnb	Setregid(rgid int, egid int) (err error)
 //sysnb	Setreuid(ruid int, euid int) (err error)
-//sysnb	Setrlimit(which int, lim *Rlimit) (err error)
+//sysnb	setrlimit(which int, lim *Rlimit) (err error)
 //sysnb	Setsid() (pid int, err error)
 //sysnb	Settimeofday(tp *Timeval) (err error)
 //sysnb	Setuid(uid int) (err error)

src/syscall/syscall_freebsd.go

@@ -234,7 +234,7 @@ func Mknod(path string, mode uint32, dev uint64) (err error) {
 //sys	Setpriority(which int, who int, prio int) (err error)
 //sysnb	Setregid(rgid int, egid int) (err error)
 //sysnb	Setreuid(ruid int, euid int) (err error)
-//sysnb	Setrlimit(which int, lim *Rlimit) (err error)
+//sysnb	setrlimit(which int, lim *Rlimit) (err error)
 //sysnb	Setsid() (pid int, err error)
 //sysnb	Settimeofday(tp *Timeval) (err error)
 //sysnb	Setuid(uid int) (err error)

src/syscall/syscall_linux.go

@@ -1058,7 +1058,7 @@ func Getpgrp() (pid int) {
 //sys	Mknodat(dirfd int, path string, mode uint32, dev int) (err error)
 //sys	Nanosleep(time *Timespec, leftover *Timespec) (err error)
 //sys	PivotRoot(newroot string, putold string) (err error) = SYS_PIVOT_ROOT
-//sysnb prlimit(pid int, resource int, newlimit *Rlimit, old *Rlimit) (err error) = SYS_PRLIMIT64
+//sysnb prlimit1(pid int, resource int, newlimit *Rlimit, old *Rlimit) (err error) = SYS_PRLIMIT64
 //sys	read(fd int, p []byte) (n int, err error)
 //sys	Removexattr(path string, attr string) (err error)
 //sys	Setdomainname(p []byte) (err error)
@@ -1261,3 +1261,14 @@ func Munmap(b []byte) (err error) {
 //sys	Munlock(b []byte) (err error)
 //sys	Mlockall(flags int) (err error)
 //sys	Munlockall() (err error)
+
+// prlimit changes a resource limit. We use a single definition so that
+// we can tell StartProcess to not restore the original NOFILE limit.
+// This is unexported but can be called from x/sys/unix.
+func prlimit(pid int, resource int, newlimit *Rlimit, old *Rlimit) (err error) {
+	err = prlimit1(pid, resource, newlimit, old)
+	if err == nil && newlimit != nil && resource == RLIMIT_NOFILE {
+		origRlimitNofile.Store(Rlimit{0, 0})
+	}
+	return err
+}