Skip to content

Commit

Permalink
Merge pull request #4277 from lifubang/backport-4265-nofilerlimit
Browse files Browse the repository at this point in the history
[1.1] Fix set nofile rlimit error
  • Loading branch information
kolyshkin authored May 20, 2024
2 parents 51dc972 + c918058 commit 9244703
Show file tree
Hide file tree
Showing 11 changed files with 162 additions and 54 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/opencontainers/runc

go 1.17
go 1.18

require (
github.com/checkpoint-restore/go-criu/v5 v5.3.0
Expand Down
41 changes: 0 additions & 41 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -57,57 +57,16 @@ github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJ
github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k=
github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
Expand Down
16 changes: 16 additions & 0 deletions libcontainer/init_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,13 @@ func newContainerInit(t initType, pipe *os.File, consoleSocket *os.File, fifoFd,
if err := populateProcessEnvironment(config.Env); err != nil {
return nil, err
}

// Clean the RLIMIT_NOFILE cache in go runtime.
// Issue: https://github.com/opencontainers/runc/issues/4195
if containsRlimit(config.Rlimits, unix.RLIMIT_NOFILE) {
system.ClearRlimitNofileCache()
}

switch t {
case initSetns:
// mountFds must be nil in this case. We don't mount while doing runc exec.
Expand Down Expand Up @@ -518,6 +525,15 @@ func setupRoute(config *configs.Config) error {
return nil
}

func containsRlimit(limits []configs.Rlimit, resource int) bool {
for _, rlimit := range limits {
if rlimit.Type == resource {
return true
}
}
return false
}

func setupRlimits(limits []configs.Rlimit, pid int) error {
for _, rlimit := range limits {
if err := unix.Prlimit(pid, rlimit.Type, &unix.Rlimit{Max: rlimit.Hard, Cur: rlimit.Soft}, nil); err != nil {
Expand Down
6 changes: 4 additions & 2 deletions libcontainer/integration/exec_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -135,11 +135,13 @@ func testRlimit(t *testing.T, userns bool) {

config := newTemplateConfig(t, &tParam{userns: userns})

// ensure limit is lower than what the config requests to test that in a user namespace
// Ensure limit is lower than what the config requests to test that in a user namespace
// the Setrlimit call happens early enough that we still have permissions to raise the limit.
// Do not change the Cur value to be equal to the Max value, please see:
// https://github.com/opencontainers/runc/pull/4265#discussion_r1589666444
ok(t, unix.Setrlimit(unix.RLIMIT_NOFILE, &unix.Rlimit{
Max: 1024,
Cur: 1024,
Cur: 512,
}))

out := runContainerOk(t, config, "/bin/sh", "-c", "ulimit -n")
Expand Down
18 changes: 10 additions & 8 deletions libcontainer/process_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -152,20 +152,22 @@ func (p *setnsProcess) start() (retErr error) {
}
}
}
// set rlimits, this has to be done here because we lose permissions
// to raise the limits once we enter a user-namespace
if err := setupRlimits(p.config.Rlimits, p.pid()); err != nil {
return fmt.Errorf("error setting rlimits for process: %w", err)
}

if err := utils.WriteJSON(p.messageSockPair.parent, p.config); err != nil {
return fmt.Errorf("error writing config to pipe: %w", err)
}

ierr := parseSync(p.messageSockPair.parent, func(sync *syncT) error {
switch sync.Type {
case procReady:
// This shouldn't happen.
panic("unexpected procReady in setns")
// Set rlimits, this has to be done here because we lose permissions
// to raise the limits once we enter a user-namespace
if err := setupRlimits(p.config.Rlimits, p.pid()); err != nil {
return fmt.Errorf("error setting rlimits for ready process: %w", err)
}

// Sync with child.
return writeSync(p.messageSockPair.parent, procRun)
case procHooks:
// This shouldn't happen.
panic("unexpected procHooks in setns")
Expand Down Expand Up @@ -495,7 +497,7 @@ func (p *initProcess) start() (retErr error) {
return err
}
case procReady:
// set rlimits, this has to be done here because we lose permissions
// Set rlimits, this has to be done here because we lose permissions
// to raise the limits once we enter a user-namespace
if err := setupRlimits(p.config.Rlimits, p.pid()); err != nil {
return fmt.Errorf("error setting rlimits for ready process: %w", err)
Expand Down
9 changes: 9 additions & 0 deletions libcontainer/setns_init_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ func (l *linuxSetnsInit) Init() error {
}
}
}

if l.config.CreateConsole {
if err := setupConsole(l.consoleSocket, l.config, false); err != nil {
return err
Expand All @@ -61,6 +62,14 @@ func (l *linuxSetnsInit) Init() error {
return err
}
}

// Tell our parent that we're ready to exec. This must be done before the
// Seccomp rules have been applied, because we need to be able to read and
// write to a socket.
if err := syncParentReady(l.pipe); err != nil {
return fmt.Errorf("sync ready: %w", err)
}

if err := selinux.SetExecLabel(l.config.ProcessLabel); err != nil {
return err
}
Expand Down
3 changes: 2 additions & 1 deletion libcontainer/standard_init_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,8 @@ func (l *linuxStandardInit) Init() error {
return &os.SyscallError{Syscall: "prctl(SET_NO_NEW_PRIVS)", Err: err}
}
}
// Tell our parent that we're ready to Execv. This must be done before the

// Tell our parent that we're ready to exec. This must be done before the
// Seccomp rules have been applied, because we need to be able to read and
// write to a socket.
if err := syncParentReady(l.pipe); err != nil {
Expand Down
2 changes: 1 addition & 1 deletion libcontainer/system/linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,12 @@ func (p ParentDeathSignal) Set() error {
return SetParentDeathSignal(uintptr(p))
}

// Deprecated: Execv is not used in runc anymore, it will be removed in v1.2.0.
func Execv(cmd string, args []string, env []string) error {
name, err := exec.LookPath(cmd)
if err != nil {
return err
}

return Exec(name, args, env)
}

Expand Down
25 changes: 25 additions & 0 deletions libcontainer/system/rlimit_go119.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
//go:build go1.19

package system

import (
"sync/atomic"
"syscall"

_ "unsafe" // for go:linkname
)

//go:linkname syscallOrigRlimitNofile syscall.origRlimitNofile
var syscallOrigRlimitNofile atomic.Pointer[syscall.Rlimit]

// ClearRlimitNofileCache is to clear go runtime's nofile rlimit cache.
func ClearRlimitNofileCache() {
// As reported in issue #4195, the new version of go runtime(since 1.19)
// will cache rlimit-nofile. Before executing execve, the rlimit-nofile
// of the process will be restored with the cache. In runc, this will
// cause the rlimit-nofile setting by the parent process for the container
// to become invalid. It can be solved by clearing this cache. But
// unfortunately, go stdlib doesn't provide such function, so we need to
// link to the private var `origRlimitNofile` in package syscall to hack.
syscallOrigRlimitNofile.Store(nil)
}
6 changes: 6 additions & 0 deletions libcontainer/system/rlimit_stub.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
//go:build !go1.19

package system

func ClearRlimitNofileCache() {
}
88 changes: 88 additions & 0 deletions tests/integration/rlimits.bats
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
#!/usr/bin/env bats

load helpers

function setup() {
# Do not change the Cur value to be equal to the Max value
# Because in some environments, the soft and hard nofile limit have the same value.
[ $EUID -eq 0 ] && prlimit --nofile=1024:65536 -p $$
setup_busybox
}

function teardown() {
teardown_bundle
}

# Set and check rlimit_nofile for runc run. Arguments are:
# $1: soft limit;
# $2: hard limit.
function run_check_nofile() {
soft="$1"
hard="$2"
update_config ".process.rlimits = [{\"type\": \"RLIMIT_NOFILE\", \"soft\": ${soft}, \"hard\": ${hard}}]"
update_config '.process.args = ["/bin/sh", "-c", "ulimit -n; ulimit -H -n"]'

runc run test_rlimit
[ "$status" -eq 0 ]
[[ "${lines[0]}" == "${soft}" ]]
[[ "${lines[1]}" == "${hard}" ]]
}

# Set and check rlimit_nofile for runc exec. Arguments are:
# $1: soft limit;
# $2: hard limit.
function exec_check_nofile() {
soft="$1"
hard="$2"
update_config ".process.rlimits = [{\"type\": \"RLIMIT_NOFILE\", \"soft\": ${soft}, \"hard\": ${hard}}]"

runc run -d --console-socket "$CONSOLE_SOCKET" test_rlimit
[ "$status" -eq 0 ]

runc exec test_rlimit /bin/sh -c "ulimit -n; ulimit -H -n"
[ "$status" -eq 0 ]
[[ "${lines[0]}" == "${soft}" ]]
[[ "${lines[1]}" == "${hard}" ]]
}

@test "runc run with RLIMIT_NOFILE(The same as system's hard value)" {
hard=$(ulimit -n -H)
soft="$hard"
run_check_nofile "$soft" "$hard"
}

@test "runc run with RLIMIT_NOFILE(Bigger than system's hard value)" {
requires root
limit=$(ulimit -n -H)
soft=$((limit + 1))
hard=$soft
run_check_nofile "$soft" "$hard"
}

@test "runc run with RLIMIT_NOFILE(Smaller than system's hard value)" {
limit=$(ulimit -n -H)
soft=$((limit - 1))
hard=$soft
run_check_nofile "$soft" "$hard"
}

@test "runc exec with RLIMIT_NOFILE(The same as system's hard value)" {
hard=$(ulimit -n -H)
soft="$hard"
exec_check_nofile "$soft" "$hard"
}

@test "runc exec with RLIMIT_NOFILE(Bigger than system's hard value)" {
requires root
limit=$(ulimit -n -H)
soft=$((limit + 1))
hard=$soft
exec_check_nofile "$soft" "$hard"
}

@test "runc exec with RLIMIT_NOFILE(Smaller than system's hard value)" {
limit=$(ulimit -n -H)
soft=$((limit - 1))
hard=$soft
exec_check_nofile "$soft" "$hard"
}

0 comments on commit 9244703

Please sign in to comment.