e2e: add tests for exec2 task driver (#22406)

* e2e: add tests for exec2 task driver * e2e: use envoy 1.29.4 because consul * e2e: add a bridge networking http test for exec driver * e2e: split up http test so curl always starts after the server
hashicorp · May 31, 2024 · 2054e87 · 2054e87
1 parent 86ee56b
commit 2054e87
Show file tree

Hide file tree

Showing 10 changed files with 472 additions and 4 deletions.
diff --git a/e2e/exec2/doc.go b/e2e/exec2/doc.go
@@ -0,0 +1,5 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+// Package exec2 contains test cases related to the exec2 task driver.
+package exec2
diff --git a/e2e/exec2/exec2_test.go b/e2e/exec2/exec2_test.go
@@ -0,0 +1,100 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package exec2
+
+import (
+	"fmt"
+	"regexp"
+	"testing"
+
+	"github.com/hashicorp/nomad/e2e/v3/cluster3"
+	"github.com/hashicorp/nomad/e2e/v3/jobs3"
+	"github.com/shoenig/test/must"
+)
+
+func TestExec2(t *testing.T) {
+	cluster3.Establish(t,
+		cluster3.Leader(),
+		cluster3.LinuxClients(1),
+	)
+
+	t.Run("testEnv", testEnv)
+	t.Run("testSecretsDir", testSecretsDir)
+	t.Run("testCountdash", testCountdash)
+	t.Run("testHTTP", testHTTP)
+}
+
+func testEnv(t *testing.T) {
+	job, cleanup := jobs3.Submit(t,
+		"./input/env.hcl",
+		jobs3.WaitComplete("group"),
+	)
+	t.Cleanup(cleanup)
+
+	logs := job.TaskLogs("group", "env")
+
+	// ensure the job id lines up
+	expect := fmt.Sprintf("NOMAD_JOB_ID=%s", job.JobID())
+	must.StrContains(t, logs.Stdout, expect)
+
+	// ensure dynamic user e.g.
+	// USER=nomad-85249
+	userRe := regexp.MustCompile(`nomad-\d+`)
+	must.RegexMatch(t, userRe, logs.Stdout)
+}
+
+func testSecretsDir(t *testing.T) {
+	job, cleanup := jobs3.Submit(t,
+		"./input/secrets.hcl",
+		jobs3.WaitComplete("group"),
+	)
+	t.Cleanup(cleanup)
+
+	// ensure we can read the workload identity token file
+	nomadTokenLogs := job.TaskLogs("group", "nomad-token")
+	tokenRe := regexp.MustCompile(`[\w_-]+`)
+	must.RegexMatch(t, tokenRe, nomadTokenLogs.Stdout)
+
+	// ensure we can read the written password.txt file
+	passwordLogs := job.TaskLogs("group", "password")
+	must.StrContains(t, passwordLogs.Stdout, "abc123")
+}
+
+func testCountdash(t *testing.T) {
+	job, cleanup := jobs3.Submit(t,
+		"./input/countdash.hcl",
+	)
+	t.Cleanup(cleanup)
+
+	apiEnvoyLogs := job.TaskLogs("api", "connect-proxy-count-api")
+	must.StrContains(t, apiEnvoyLogs.Stderr, "all clusters initialized. initializing init manager")
+
+	dashEnvoyLogs := job.TaskLogs("dashboard", "connect-proxy-count-dashboard")
+	must.StrContains(t, dashEnvoyLogs.Stderr, "all clusters initialized. initializing init manager")
+
+	apiLogs := job.TaskLogs("api", "backend")
+	must.StrContains(t, apiLogs.Stdout, "Serving at http://localhost:9001")
+
+	dashLogs := job.TaskLogs("dashboard", "dashboard")
+	must.StrContains(t, dashLogs.Stdout, "Using counting service at http://127.0.0.1:8080")
+}
+
+func testHTTP(t *testing.T) {
+	job, _ := jobs3.Submit(t,
+		"./input/http.hcl",
+		jobs3.DisableCleanup(),
+	)
+
+	job2, _ := jobs3.Submit(t,
+		"./input/http_curl.hcl",
+		jobs3.DisableCleanup(),
+	)
+
+	logs := job.TaskLogs("backend", "http")
+	must.StrContains(t, logs.Stderr, `"GET / HTTP/1.1" 200 -`)        // healthcheck
+	must.StrContains(t, logs.Stderr, `"GET /hi.html HTTP/1.1" 200 -`) // curl
+
+	logs2 := job2.TaskLogs("client", "curl")
+	must.StrContains(t, logs2.Stdout, "<body><p>Hello, friend!</p></body>")
+}
diff --git a/e2e/exec2/input/countdash.hcl b/e2e/exec2/input/countdash.hcl
@@ -0,0 +1,119 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+# This is a variation of countdash that uses exec2 for running the envoy
+# proxies manually.
+
+job "countdash" {
+  group "api" {
+    network {
+      mode = "bridge"
+    }
+
+    service {
+      name = "count-api"
+      port = "9001"
+
+      connect {
+        sidecar_service {}
+        sidecar_task {
+          driver = "exec2"
+          user   = "nobody"
+          config {
+            command = "/opt/bin/envoy"
+            args = [
+              "-c",
+              "${NOMAD_SECRETS_DIR}/envoy_bootstrap.json",
+              "-l",
+              "${meta.connect.log_level}",
+              "--concurrency",
+              "${meta.connect.proxy_concurrency}",
+              "--disable-hot-restart"
+            ]
+            # TODO(shoenig) should not need NOMAD_ values once
+            # https://github.com/hashicorp/nomad-driver-exec2/issues/29 is
+            # fixed.
+            unveil = ["rx:/opt/bin", "rwc:/dev/shm", "r:${NOMAD_TASK_DIR}", "r:${NOMAD_SECRETS_DIR}"]
+          }
+
+          resources {
+            cpu    = 1000
+            memory = 256
+          }
+        }
+      }
+    }
+
+    task "backend" {
+      driver = "docker"
+
+      config {
+        image = "docker.io/hashicorpdev/counter-api:v3"
+      }
+    }
+  }
+
+  group "dashboard" {
+    network {
+      mode = "bridge"
+
+      port "http" {
+        static = 9002
+        to     = 9002
+      }
+    }
+
+    service {
+      name = "count-dashboard"
+      port = "http"
+
+      connect {
+        sidecar_service {
+          proxy {
+            upstreams {
+              destination_name = "count-api"
+              local_bind_port  = 8080
+            }
+          }
+        }
+        sidecar_task {
+          driver = "exec2"
+          user   = "nobody"
+          config {
+            command = "/opt/bin/envoy"
+            args = [
+              "-c",
+              "${NOMAD_SECRETS_DIR}/envoy_bootstrap.json",
+              "-l",
+              "${meta.connect.log_level}",
+              "--concurrency",
+              "${meta.connect.proxy_concurrency}",
+              "--disable-hot-restart"
+            ]
+            # TODO(shoenig) should not need NOMAD_ values once
+            # https://github.com/hashicorp/nomad-driver-exec2/issues/29 is
+            # fixed.
+            unveil = ["rx:/opt/bin", "rwc:/dev/shm", "r:${NOMAD_TASK_DIR}", "r:${NOMAD_SECRETS_DIR}"]
+          }
+
+          resources {
+            cpu    = 1000
+            memory = 256
+          }
+        }
+      }
+    }
+
+    task "dashboard" {
+      driver = "docker"
+
+      env {
+        COUNTING_SERVICE_URL = "http://${NOMAD_UPSTREAM_ADDR_count_api}"
+      }
+
+      config {
+        image = "docker.io/hashicorpdev/counter-dashboard:v3"
+      }
+    }
+  }
+}
diff --git a/e2e/exec2/input/env.hcl b/e2e/exec2/input/env.hcl
@@ -0,0 +1,38 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+# This is a simple env job using the exec2 task driver.
+
+job "env" {
+  type = "batch"
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "group" {
+    reschedule {
+      attempts  = 0
+      unlimited = false
+    }
+
+    restart {
+      attempts = 0
+      mode     = "fail"
+    }
+
+    task "env" {
+      driver = "exec2"
+
+      config {
+        command = "env"
+      }
+
+      resources {
+        cpu    = 100
+        memory = 64
+      }
+    }
+  }
+}
diff --git a/e2e/exec2/input/http.hcl b/e2e/exec2/input/http.hcl
@@ -0,0 +1,74 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+# This job serves the NOMAD_TASK_DIR over http.
+
+job "http" {
+  type = "service"
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "backend" {
+    network {
+      mode = "bridge"
+      port "http" {
+        to = "9999"
+      }
+    }
+
+    task "http" {
+      driver = "exec2"
+
+      service {
+        name     = "python-http"
+        port     = "http"
+        provider = "nomad"
+        check {
+          name     = "hi"
+          type     = "http"
+          path     = "/"
+          interval = "3s"
+          timeout  = "1s"
+        }
+      }
+
+      config {
+        command = "python3"
+        args    = ["-m", "http.server", "9999", "--directory", "${NOMAD_TASK_DIR}"]
+      }
+
+      template {
+        destination = "local/hi.html"
+        data        = <<EOH
+      <!doctype html>
+      <html>
+        <title>example</title>
+        <body><p>Hello, friend!</p></body>
+      </html>
+      EOH
+      }
+
+      resources {
+        cpu    = 500
+        memory = 256
+      }
+    }
+
+    restart {
+      attempts = 0
+      mode     = "fail"
+    }
+
+    reschedule {
+      attempts  = 0
+      unlimited = false
+    }
+
+    update {
+      min_healthy_time = "5s"
+    }
+  }
+}
diff --git a/e2e/exec2/input/http_curl.hcl b/e2e/exec2/input/http_curl.hcl
@@ -0,0 +1,43 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+# This job makes requests to the "python-http" service.
+
+job "http_curl" {
+  type = "service"
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "client" {
+    task "curl" {
+      driver = "exec2"
+
+      config {
+        command = "bash"
+        args    = ["local/script.sh"]
+      }
+
+      template {
+        destination = "local/script.sh"
+        change_mode = "noop"
+        data        = <<EOF
+#!/usr/bin/env bash
+
+while true
+do
+
+{{ range nomadService "python-http" }}
+(curl -s -S -L "{{ .Address }}:{{ .Port }}/hi.html") || true
+{{ end }}
+
+sleep 2
+
+done
+EOF
+      }
+    }
+  }
+}