Merge pull request #131 from Netflix/oom-2

Introduce OOM settings for containers

executor/mock/standalone/standalone_test.go

@@ -43,7 +43,7 @@ var (
 	}
 	ubuntu = testImage{
 		name: "titusoss/ubuntu",
-		tag:  "20180501-1525157359",
+		tag:  "20180518-1526605880",
 	}
 	// TODO: Determine how this got built, and add it to the auto image builders?
 	byDigest = testImage{
@@ -94,6 +94,8 @@ func TestStandalone(t *testing.T) {
 		testTerminateTimeout,
 		testMakesPTY,
 		testTerminateTimeoutNotTooSlow,
+		testOOMAdj,
+		testOOMKill,
 	}
 	for _, fun := range testFunctions {
 		fullName := runtime.FuncForPC(reflect.ValueOf(fun).Pointer()).Name()
@@ -531,3 +533,45 @@ func testTerminateTimeoutNotTooSlow(t *testing.T, jobID string) {
 		t.Fatalf("Task wasn't killed quickly enough, in %s", killTime.String())
 	}
 }
+
+func testOOMAdj(t *testing.T, jobID string) {
+	ji := &mock.JobInput{
+		ImageName:  ubuntu.name,
+		Version:    ubuntu.tag,
+		Entrypoint: `/bin/bash -c 'cat /proc/1/oom_score | grep 999'`,
+		JobID:      jobID,
+	}
+	if !mock.RunJobExpectingSuccess(ji) {
+		t.Fail()
+	}
+}
+
+func testOOMKill(t *testing.T, jobID string) {
+	// Start the executor
+	jobRunner := mock.NewJobRunner()
+	defer jobRunner.StopExecutorAsync()
+
+	ji := &mock.JobInput{
+		ImageName:  ubuntu.name,
+		Version:    ubuntu.tag,
+		Entrypoint: `stress --vm 100 --vm-keep --vm-hang 100`,
+		JobID:      jobID,
+	}
+	jobResponse := jobRunner.StartJob(ji)
+
+	// Wait until the task is running
+
+	for status := range jobResponse.UpdateChan {
+
+		if mock.IsTerminalState(status.State) {
+			if status.State.String() != "TASK_FAILED" {
+				t.Fail()
+			}
+			if !strings.Contains(status.Mesg, "OOMKilled") {
+				t.Fatal("Task killed due to: ", status.Mesg)
+			}
+			return
+		}
+	}
+	t.Fail()
+}

executor/runtime/docker/docker.go

@@ -1520,6 +1520,10 @@ func (r *DockerRuntime) setupPostStartLogDirTiniHandleConnection2(parentCtx cont
 		}
 	}
 
+	if err := setupOOMAdj(c, cred); err != nil {
+		return err
+	}
+
 	if err := setupContainerNesting(parentCtx, c, cred); err != nil {
 		log.Error("Unable to setup container nesting: ", err)
 		return err

executor/runtime/docker/docker_linux.go

@@ -191,3 +191,21 @@ func setupContainerNesting(parentCtx context.Context, c *runtimeTypes.Container,
 
 	return ret
 }
+
+func setupOOMAdj(c *runtimeTypes.Container, cred ucred) error {
+	oomScore := 1000
+
+	if c.TitusInfo.OomScoreAdj != nil {
+		oomScore = int(c.TitusInfo.GetOomScoreAdj())
+	}
+
+	pid := strconv.FormatInt(int64(cred.pid), 10)
+	oomScoreAdjPath := filepath.Join("/proc", pid, "oom_score_adj")
+	file, err := os.OpenFile(oomScoreAdjPath, os.O_RDWR, 0000)
+	if err != nil {
+		return err
+	}
+	defer shouldClose(file)
+	_, err = file.WriteString(fmt.Sprintf("%d\n", oomScore))
+	return err
+}

executor/runtime/docker/docker_unsupported.go

@@ -40,3 +40,7 @@ func cleanupCgroups(cgroupPath string) error {
 func setupContainerNesting(parentCtx context.Context, c *runtimeTypes.Container, cred ucred) error {
 	return errUnsupported
 }
+
+func setupOOMAdj(c *runtimeTypes.Container, cred ucred) error {
+	return errUnsupported
+}

hack/test-images/ubuntu/Dockerfile

@@ -1,2 +1,2 @@
 FROM ubuntu:xenial
-RUN apt-get update && apt-get install -y curl libcap2-bin grep iproute2 httpie iputils-ping
+RUN apt-get update && apt-get install -y curl libcap2-bin grep iproute2 httpie iputils-ping stress