[release-1.12] *: implement default ulimits for containers

cmd/crio/config.go

@@ -75,11 +75,17 @@ stream_tls_key = "{{ .StreamTLSKey }}"
 # automatically pick up the changes within 5 minutes.
 stream_tls_ca = "{{ .StreamTLSCA }}"
 
-
 # The crio.runtime table contains settings pertaining to the OCI runtime used
 # and options for how to set up and manage the OCI runtime.
 [crio.runtime]
 
+# A list of ulimits to be set in containers by default, specified as
+# "<ulimit name>=<soft limit>:<hard limit>", for example:
+# "nofile=1024:2048"
+# If nothing is set here, settings will be inherited from the CRI-O daemon
+#default_ulimits = [
+{{ range $ulimit := .DefaultUlimits }}{{ printf "#\t%q,\n" $ulimit }}{{ end }}#]
+
 # Path to the OCI compatible runtime used for trusted container workloads. This
 # is a mandatory setting as this runtime will be the default and will also be
 # used for untrusted container workloads if runtime_untrusted_workload is not

cmd/crio/main.go

@@ -17,6 +17,7 @@ import (
 	"github.com/containers/libpod/pkg/hooks"
 	_ "github.com/containers/libpod/pkg/hooks/0.1.0"
 	"github.com/containers/storage/pkg/reexec"
+	units "github.com/docker/go-units"
 	"github.com/kubernetes-sigs/cri-o/lib"
 	"github.com/kubernetes-sigs/cri-o/oci"
 	"github.com/kubernetes-sigs/cri-o/pkg/signals"
@@ -40,7 +41,23 @@ func validateConfig(config *server.Config) error {
 	case lib.ImageVolumesBind:
 	default:
 		return fmt.Errorf("Unrecognized image volume type specified")
+	}
 
+	// This is somehow duplicated with server.getUlimitsFromConfig under server/utils.go
+	// but I don't want to export that function for the sake of validation here
+	// so, keep it in mind if things start to blow up.
+	// Reason for having this here is that I don't want people to start crio
+	// with invalid ulimits but realize that only after starting a couple of
+	// containers and watching them fail.
+	for _, u := range config.RuntimeConfig.DefaultUlimits {
+		ul, err := units.ParseUlimit(u)
+		if err != nil {
+			return fmt.Errorf("unrecognized ulimit %s: %v", u, err)
+		}
+		_, err = ul.GetRlimit()
+		if err != nil {
+			return err
+		}
 	}
 
 	if config.UIDMappings != "" && config.ManageNetworkNSLifecycle {
@@ -147,6 +164,9 @@ func mergeConfig(config *server.Config, ctx *cli.Context) error {
 	if ctx.GlobalIsSet("default-sysctls") {
 		config.DefaultSysctls = strings.Split(ctx.GlobalString("default-sysctls"), ",")
 	}
+	if ctx.GlobalIsSet("default-ulimits") {
+		config.DefaultUlimits = ctx.GlobalStringSlice("default-ulimits")
+	}
 	if ctx.GlobalIsSet("pids-limit") {
 		config.PidsLimit = ctx.GlobalInt64("pids-limit")
 	}
@@ -375,6 +395,10 @@ func main() {
 			Name:  "default-sysctls",
 			Usage: "sysctls to add to the containers",
 		},
+		cli.StringSliceFlag{
+			Name:  "default-ulimits",
+			Usage: "ulimits to apply to conatainers by default (name=soft:hard)",
+		},
 		cli.BoolFlag{
 			Name:  "profile",
 			Usage: "enable pprof remote profiler on localhost:6060",

lib/config.go

@@ -219,6 +219,9 @@ type RuntimeConfig struct {
 
 	// Sysctls to add to all containers.
 	DefaultSysctls []string `toml:"default_sysctls"`
+
+	// DefeultUlimits specifies the default ulimits to apply to containers
+	DefaultUlimits []string `toml:"default_ulimits"`
 }
 
 // ImageConfig represents the "crio.image" TOML config table.
@@ -356,6 +359,7 @@ func DefaultConfig() *Config {
 			DefaultCapabilities:      DefaultCapabilities,
 			LogLevel:                 "error",
 			DefaultSysctls:           []string{},
+			DefaultUlimits:           []string{},
 		},
 		ImageConfig: ImageConfig{
 			DefaultTransport:    defaultTransport,

server/container_create_linux.go

@@ -268,6 +268,14 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
 	specgen.HostSpecific = true
 	specgen.ClearProcessRlimits()
 
+	ulimits, err := getUlimitsFromConfig(s.config)
+	if err != nil {
+		return nil, err
+	}
+	for _, u := range ulimits {
+		specgen.AddProcessRlimits(u.name, u.hard, u.soft)
+	}
+
 	readOnlyRootfs := s.config.ReadOnly
 
 	var privileged bool

server/sandbox_run_linux.go

@@ -127,6 +127,16 @@ func (s *Server) runPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
 	if err != nil {
 		return nil, err
 	}
+	g.HostSpecific = true
+	g.ClearProcessRlimits()
+
+	ulimits, err := getUlimitsFromConfig(s.config)
+	if err != nil {
+		return nil, err
+	}
+	for _, u := range ulimits {
+		g.AddProcessRlimits(u.name, u.hard, u.soft)
+	}
 
 	// setup defaults for the pod sandbox
 	g.SetRootReadonly(true)

server/utils.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/cri-o/ocicni/pkg/ocicni"
+	units "github.com/docker/go-units"
 	"github.com/kubernetes-sigs/cri-o/lib/sandbox"
 	"github.com/kubernetes-sigs/cri-o/server/metrics"
 	"github.com/opencontainers/image-spec/specs-go/v1"
@@ -248,3 +249,26 @@ func validateSysctl(sysctl string, hostNet, hostIPC bool) error {
 	}
 	return errors.Errorf("%q not whitelisted", sysctl)
 }
+
+type ulimit struct {
+	name string
+	hard uint64
+	soft uint64
+}
+
+func getUlimitsFromConfig(config Config) ([]ulimit, error) {
+	var ulimits []ulimit
+	for _, u := range config.RuntimeConfig.DefaultUlimits {
+		ul, err := units.ParseUlimit(u)
+		if err != nil {
+			return nil, err
+		}
+		rl, err := ul.GetRlimit()
+		if err != nil {
+			return nil, err
+		}
+		// This sucks, but it's the runtime-tools interface
+		ulimits = append(ulimits, ulimit{name: "RLIMIT_" + strings.ToUpper(ul.Name), hard: rl.Hard, soft: rl.Soft})
+	}
+	return ulimits, nil
+}

test/command.bats

@@ -10,3 +10,16 @@ load helpers
 	echo "$output"
 	[ "$status" -ne 0 ]
 }
+
+@test "invalid ulimits" {
+	run ${CRIO_BINARY} --default-ulimits doesntexist=2042
+	echo $output
+	[ "$status" -ne 0 ]
+	[[ "$output" =~ "invalid ulimit type: doesntexist" ]]
+	run ${CRIO_BINARY} --default-ulimits nproc=2042:42
+	echo $output
+	[ "$status" -ne 0 ]
+	[[ "$output" =~ "ulimit soft limit must be less than or equal to hard limit: 2042 > 42" ]]
+	# can't cover everything here, ulimits parsing is tested in
+	# github.com/docker/go-units package
+}

test/ctr.bats

@@ -67,6 +67,48 @@ function teardown() {
 	stop_crio
 }
 
+@test "ulimits" {
+	ULIMITS="--default-ulimits nofile=42:42 --default-ulimits nproc=1024:2048" start_crio
+	run crictl runp "$TESTDATA"/sandbox_config.json
+	echo "$output"
+	[ "$status" -eq 0 ]
+	pod_id="$output"
+
+	ulimits=$(cat "$TESTDATA"/container_config.json | python -c 'import json,sys;obj=json.load(sys.stdin); obj["command"] = ["/bin/sh", "-c", "sleep 600"]; json.dump(obj, sys.stdout)')
+	echo "$ulimits" > "$TESTDIR"/container_config_ulimits.json
+	run crictl create "$pod_id" "$TESTDIR"/container_config_ulimits.json "$TESTDATA"/sandbox_config.json
+	echo "$output"
+	[ "$status" -eq 0 ]
+	ctr_id="$output"
+	run crictl start "$ctr_id"
+	echo "$output"
+	[ "$status" -eq 0 ]
+
+	run crictl exec --sync "$ctr_id" sh -c "ulimit -n"
+	echo "$output"
+	[ "$status" -eq 0 ]
+	[[ "$output" =~ "42" ]]
+	run crictl exec --sync "$ctr_id" sh -c "ulimit -p"
+	echo "$output"
+	[ "$status" -eq 0 ]
+	[[ "$output" =~ "1024" ]]
+
+	run crictl exec --sync "$ctr_id" sh -c "ulimit -Hp"
+	echo "$output"
+	[ "$status" -eq 0 ]
+	[[ "$output" =~ "2048" ]]
+
+	run crictl stopp "$pod_id"
+	echo "$output"
+	[ "$status" -eq 0 ]
+	run crictl rmp "$pod_id"
+	echo "$output"
+	[ "$status" -eq 0 ]
+	cleanup_ctrs
+	cleanup_pods
+	stop_crio
+}
+
 @test "ctr remove" {
 	start_crio
 	run crictl runp "$TESTDATA"/sandbox_config.json
@@ -1197,4 +1239,4 @@ function teardown() {
 	cleanup_ctrs
 	cleanup_pods
 	stop_crio
-}
+}

test/helpers.bash

@@ -26,6 +26,7 @@ APPARMOR_PROFILE=${APPARMOR_PROFILE:-crio-default}
 RUNTIME=${RUNTIME:-runc}
 UID_MAPPINGS=${UID_MAPPINGS:-}
 GID_MAPPINGS=${GID_MAPPINGS:-}
+ULIMITS=${ULIMITS:-}
 OVERRIDE_OPTIONS=${OVERRIDE_OPTIONS:-}
 RUNTIME_PATH=$(command -v $RUNTIME || true)
 RUNTIME_BINARY=${RUNTIME_PATH:-/usr/local/sbin/runc}
@@ -240,8 +241,8 @@ function start_crio() {
 	"$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name=quay.io/crio/image-volume-test:latest --import-from=dir:"$ARTIFACTS_PATH"/image-volume-test-image --signature-policy="$INTEGRATION_ROOT"/policy.json
 	"$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name=quay.io/crio/busybox:latest --import-from=dir:"$ARTIFACTS_PATH"/busybox-image --signature-policy="$INTEGRATION_ROOT"/policy.json
 	"$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name=quay.io/crio/stderr-test:latest --import-from=dir:"$ARTIFACTS_PATH"/stderr-test --signature-policy="$INTEGRATION_ROOT"/policy.json
-	"$CRIO_BINARY" ${DEFAULT_MOUNTS_OPTS} ${HOOKS_OPTS} --default-capabilities "$capabilities" --conmon "$CONMON_BINARY" --listen "$CRIO_SOCKET" --stream-port "$STREAM_PORT" --cgroup-manager "$CGROUP_MANAGER" --default-mounts-file "$TESTDIR/containers/mounts.conf" --registry "quay.io" --runtime "$RUNTIME_BINARY" --root "$TESTDIR/crio" --runroot "$TESTDIR/crio-run" $STORAGE_OPTIONS --seccomp-profile "$seccomp" --apparmor-profile "$apparmor" --cni-config-dir "$CRIO_CNI_CONFIG" --cni-plugin-dir "$CRIO_CNI_PLUGIN" --signature-policy "$INTEGRATION_ROOT"/policy.json --image-volumes "$IMAGE_VOLUMES" --pids-limit "$PIDS_LIMIT" --log-size-max "$LOG_SIZE_MAX_LIMIT" --uid-mappings "$UID_MAPPINGS" --gid-mappings "$GID_MAPPINGS" --default-sysctls "$TEST_SYSCTL" $OVERRIDE_OPTIONS --config /dev/null config >$CRIO_CONFIG
-	sed -r -e 's/^(#)?root =/root =/g' -e 's/^(#)?runroot =/runroot =/g' -e 's/^(#)?storage_driver =/storage_driver =/g' -e '/^(#)?storage_option = (\[)?[ \t]*$/,/^#?$/s/^(#)?//g' -e '/^(#)?registries = (\[)?[ \t]*$/,/^#?$/s/^(#)?//g' -i $CRIO_CONFIG
+	"$CRIO_BINARY" ${DEFAULT_MOUNTS_OPTS} ${HOOKS_OPTS} --default-capabilities "$capabilities" --conmon "$CONMON_BINARY" --listen "$CRIO_SOCKET" --stream-port "$STREAM_PORT" --cgroup-manager "$CGROUP_MANAGER" --default-mounts-file "$TESTDIR/containers/mounts.conf" --registry "quay.io" --runtime "$RUNTIME_BINARY" --root "$TESTDIR/crio" --runroot "$TESTDIR/crio-run" $STORAGE_OPTIONS --seccomp-profile "$seccomp" --apparmor-profile "$apparmor" --cni-config-dir "$CRIO_CNI_CONFIG" --cni-plugin-dir "$CRIO_CNI_PLUGIN" --signature-policy "$INTEGRATION_ROOT"/policy.json --image-volumes "$IMAGE_VOLUMES" --pids-limit "$PIDS_LIMIT" --log-size-max "$LOG_SIZE_MAX_LIMIT" $ULIMITS --uid-mappings "$UID_MAPPINGS" --gid-mappings "$GID_MAPPINGS" --default-sysctls "$TEST_SYSCTL" $OVERRIDE_OPTIONS --config /dev/null config >$CRIO_CONFIG
+	sed -r -e 's/^(#)?root =/root =/g' -e 's/^(#)?runroot =/runroot =/g' -e 's/^(#)?storage_driver =/storage_driver =/g' -e '/^(#)?storage_option = (\[)?[ \t]*$/,/^#?$/s/^(#)?//g' -e '/^(#)?registries = (\[)?[ \t]*$/,/^#?$/s/^(#)?//g' -e '/^(#)?default_ulimits = (\[)?[ \t]*$/,/^#?$/s/^(#)?//g' -i $CRIO_CONFIG
 	# Prepare the CNI configuration files, we're running with non host networking by default
 	if [[ -n "$5" ]]; then
 		netfunc="$5"