fix: change services OOM score

- Bump `cri` `oom_score_adj` to: `-500`. - Explicitly set kubelet oom score in `kubelet.yaml` config to -450. - Additionally adjust containers' OOM score using API. Fixes: #4378 Signed-off-by: Artem Chernyshev <artem.chernyshev@talos-systems.com>
siderolabs · Oct 21, 2021 · cff20ec · cff20ec
1 parent 666a2b6
commit cff20ec
Show file tree

Hide file tree

Showing 5 changed files with 23 additions and 18 deletions.
diff --git a/internal/app/machined/pkg/system/runner/containerd/containerd.go b/internal/app/machined/pkg/system/runner/containerd/containerd.go
@@ -19,7 +19,6 @@ import (
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/namespaces"
 	"github.com/containerd/containerd/oci"
-	"github.com/containerd/containerd/sys"
 
 	"github.com/talos-systems/talos/internal/app/machined/pkg/system/events"
 	"github.com/talos-systems/talos/internal/app/machined/pkg/system/runner"
@@ -125,7 +124,7 @@ func (c *containerdRunner) Close() error {
 
 // Run implements runner.Runner interface
 //
-//nolint:gocyclo,cyclop
+//nolint:gocyclo
 func (c *containerdRunner) Run(eventSink events.Recorder) error {
 	defer close(c.stopped)
 
@@ -180,21 +179,6 @@ func (c *containerdRunner) Run(eventSink events.Recorder) error {
 		return fmt.Errorf("failed to start task: %q: %w", c.args.ID, err)
 	}
 
-	if c.opts.OOMScoreAdj != 0 {
-		var processes []containerd.ProcessInfo
-
-		processes, err = task.Pids(c.ctx)
-		if err != nil {
-			eventSink(events.StateRunning, "Failed to get task %q childs: %w", c.args.ID, err)
-		}
-
-		for _, p := range processes {
-			if err = sys.AdjustOOMScore(int(p.Pid), c.opts.OOMScoreAdj); err != nil {
-				eventSink(events.StateRunning, "Failed to change OOMScoreAdj to process %s", p.Pid)
-			}
-		}
-	}
-
 	eventSink(events.StateRunning, "Started task %s (PID %d) for container %s", task.ID(), task.Pid(), c.container.ID())
 
 	statusC, err := task.Wait(c.ctx)
@@ -287,6 +271,12 @@ func (c *containerdRunner) newOCISpecOpts(image oci.Image) []oci.SpecOpts {
 		oci.WithNoNewPrivileges,
 	)
 
+	if c.opts.OOMScoreAdj != 0 {
+		specOpts = append(specOpts,
+			WithOOMScoreAdj(c.opts.OOMScoreAdj),
+		)
+	}
+
 	if c.opts.CgroupPath != "" {
 		specOpts = append(specOpts,
 			oci.WithCgroup(c.opts.CgroupPath),

diff --git a/internal/app/machined/pkg/system/runner/containerd/opts.go b/internal/app/machined/pkg/system/runner/containerd/opts.go
@@ -32,3 +32,12 @@ func WithRootfsPropagation(rp string) oci.SpecOpts {
 		return nil
 	}
 }
+
+// WithOOMScoreAdj sets the oom score.
+func WithOOMScoreAdj(score int) oci.SpecOpts {
+	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error {
+		s.Process.OOMScoreAdj = &score
+
+		return nil
+	}
+}
diff --git a/internal/app/machined/pkg/system/services/cri.go b/internal/app/machined/pkg/system/services/cri.go
@@ -77,7 +77,7 @@ func (c *CRI) Runner(r runtime.Runtime) (runner.Runner, error) {
 		args,
 		runner.WithLoggingManager(r.Logging()),
 		runner.WithEnv(env),
-		runner.WithOOMScoreAdj(-100),
+		runner.WithOOMScoreAdj(-500),
 		runner.WithCgroupPath(constants.CgroupRuntime),
 	),
 		restart.WithType(restart.Forever),

diff --git a/internal/app/machined/pkg/system/services/kubelet.go b/internal/app/machined/pkg/system/services/kubelet.go
@@ -214,6 +214,7 @@ func (k *Kubelet) Runner(r runtime.Runtime) (runner.Runner, error) {
 			oci.WithAllDevicesAllowed,
 			oci.WithCapabilities(capability.AllGrantableCapabilities()), // TODO: kubelet doesn't need all of these, we should consider limiting capabilities
 		),
+		runner.WithOOMScoreAdj(constants.KubeletOOMScoreAdj),
 	),
 		restart.WithType(restart.Forever),
 	), nil
@@ -255,6 +256,7 @@ func (k *Kubelet) HealthSettings(runtime.Runtime) *health.Settings {
 func newKubeletConfiguration(clusterDNS []string, dnsDomain string) *kubeletconfig.KubeletConfiguration {
 	f := false
 	t := true
+	oomScoreAdj := int32(constants.KubeletOOMScoreAdj)
 
 	return &kubeletconfig.KubeletConfiguration{
 		TypeMeta: metav1.TypeMeta{
@@ -264,6 +266,7 @@ func newKubeletConfiguration(clusterDNS []string, dnsDomain string) *kubeletconf
 		StaticPodPath:      constants.ManifestsDirectory,
 		Address:            "0.0.0.0",
 		Port:               constants.KubeletPort,
+		OOMScoreAdj:        &oomScoreAdj,
 		RotateCertificates: true,
 		Authentication: kubeletconfig.KubeletAuthentication{
 			X509: kubeletconfig.KubeletX509Authentication{

diff --git a/pkg/machinery/constants/constants.go b/pkg/machinery/constants/constants.go
@@ -193,6 +193,9 @@ const (
 	// KubeletPort is the kubelet port for secure API.
 	KubeletPort = 10250
 
+	// KubeletOOMScoreAdj oom_score_adj config.
+	KubeletOOMScoreAdj = -450
+
 	// KubeletPKIDir is the path to the directory where kubelet stores issued certificates and keys.
 	KubeletPKIDir = "/var/lib/kubelet/pki"