e2e: add tests for exec2 task driver

e2e/exec2/doc.go

@@ -0,0 +1,5 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+// Package exec2 contains test cases related to the exec2 task driver.
+package exec2

e2e/exec2/exec2_test.go

@@ -0,0 +1,80 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package exec2
+
+import (
+	"fmt"
+	"regexp"
+	"testing"
+
+	"github.com/hashicorp/nomad/e2e/v3/cluster3"
+	"github.com/hashicorp/nomad/e2e/v3/jobs3"
+	"github.com/shoenig/test/must"
+)
+
+func TestExec2(t *testing.T) {
+	cluster3.Establish(t,
+		cluster3.Leader(),
+		cluster3.LinuxClients(1),
+	)
+
+	t.Run("testEnv", testEnv)
+	t.Run("testSecretsDir", testSecretsDir)
+	t.Run("testCountdash", testCountdash)
+}
+
+func testEnv(t *testing.T) {
+	job, cleanup := jobs3.Submit(t,
+		"./input/env.hcl",
+		jobs3.WaitComplete("group"),
+	)
+	t.Cleanup(cleanup)
+
+	logs := job.TaskLogs("group", "env")
+
+	// ensure the job id lines up
+	expect := fmt.Sprintf("NOMAD_JOB_ID=%s", job.JobID())
+	must.StrContains(t, logs.Stdout, expect)
+
+	// ensure dynamic user e.g.
+	// USER=nomad-85249
+	userRe := regexp.MustCompile(`nomad-\d+`)
+	must.RegexMatch(t, userRe, logs.Stdout)
+}
+
+func testSecretsDir(t *testing.T) {
+	job, cleanup := jobs3.Submit(t,
+		"./input/secrets.hcl",
+		jobs3.WaitComplete("group"),
+	)
+	t.Cleanup(cleanup)
+
+	// ensure we can read the workload identity token file
+	nomadTokenLogs := job.TaskLogs("group", "nomad-token")
+	tokenRe := regexp.MustCompile(`[\w_-]+`)
+	must.RegexMatch(t, tokenRe, nomadTokenLogs.Stdout)
+
+	// ensure we can read the written password.txt file
+	passwordLogs := job.TaskLogs("group", "password")
+	must.StrContains(t, passwordLogs.Stdout, "abc123")
+}
+
+func testCountdash(t *testing.T) {
+	job, cleanup := jobs3.Submit(t,
+		"./input/countdash.hcl",
+	)
+	t.Cleanup(cleanup)
+
+	apiEnvoyLogs := job.TaskLogs("api", "connect-proxy-count-api")
+	must.StrContains(t, apiEnvoyLogs.Stderr, "all clusters initialized. initializing init manager")
+
+	dashEnvoyLogs := job.TaskLogs("dashboard", "connect-proxy-count-dashboard")
+	must.StrContains(t, dashEnvoyLogs.Stderr, "all clusters initialized. initializing init manager")
+
+	apiLogs := job.TaskLogs("api", "backend")
+	must.StrContains(t, apiLogs.Stdout, "Serving at http://localhost:9001")
+
+	dashLogs := job.TaskLogs("dashboard", "dashboard")
+	must.StrContains(t, dashLogs.Stdout, "Using counting service at http://127.0.0.1:8080")
+}

e2e/exec2/input/countdash.hcl

@@ -0,0 +1,113 @@
+job "countdash" {
+  group "api" {
+    network {
+      mode = "bridge"
+    }
+
+    service {
+      name = "count-api"
+      port = "9001"
+
+      connect {
+        sidecar_service {}
+        sidecar_task {
+          driver = "exec2"
+          user   = "nobody"
+          config {
+            command = "/opt/bin/envoy"
+            args = [
+              "-c",
+              "${NOMAD_SECRETS_DIR}/envoy_bootstrap.json",
+              "-l",
+              "${meta.connect.log_level}",
+              "--concurrency",
+              "${meta.connect.proxy_concurrency}",
+              "--disable-hot-restart"
+            ]
+            # TODO(shoenig) should not need NOMAD_ values once
+            # https://github.com/hashicorp/nomad-driver-exec2/issues/29 is
+            # fixed.
+            unveil = ["rx:/opt/bin", "rwc:/dev/shm", "r:${NOMAD_TASK_DIR}", "r:${NOMAD_SECRETS_DIR}"]
+          }
+
+          resources {
+            cpu    = 1000
+            memory = 256
+          }
+        }
+      }
+    }
+
+    task "backend" {
+      driver = "docker"
+
+      config {
+        image = "docker.io/hashicorpdev/counter-api:v3"
+      }
+    }
+  }
+
+  group "dashboard" {
+    network {
+      mode = "bridge"
+
+      port "http" {
+        static = 9002
+        to     = 9002
+      }
+    }
+
+    service {
+      name = "count-dashboard"
+      port = "http"
+
+      connect {
+        sidecar_service {
+          proxy {
+            upstreams {
+              destination_name = "count-api"
+              local_bind_port  = 8080
+            }
+          }
+        }
+        sidecar_task {
+          driver = "exec2"
+          user   = "nobody"
+          config {
+            command = "/opt/bin/envoy"
+            args = [
+              "-c",
+              "${NOMAD_SECRETS_DIR}/envoy_bootstrap.json",
+              "-l",
+              "${meta.connect.log_level}",
+              "--concurrency",
+              "${meta.connect.proxy_concurrency}",
+              "--disable-hot-restart"
+            ]
+            # TODO(shoenig) should not need NOMAD_ values once
+            # https://github.com/hashicorp/nomad-driver-exec2/issues/29 is
+            # fixed.
+            unveil = ["rx:/opt/bin", "rwc:/dev/shm", "r:${NOMAD_TASK_DIR}", "r:${NOMAD_SECRETS_DIR}"]
+          }
+
+          resources {
+            cpu    = 1000
+            memory = 256
+          }
+        }
+      }
+    }
+
+    task "dashboard" {
+      driver = "docker"
+
+      env {
+        COUNTING_SERVICE_URL = "http://${NOMAD_UPSTREAM_ADDR_count_api}"
+      }
+
+      config {
+        image = "docker.io/hashicorpdev/counter-dashboard:v3"
+      }
+    }
+  }
+}

e2e/exec2/input/env.hcl

@@ -0,0 +1,38 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+# This is a simple env job using the exec2 task driver.
+
+job "env" {
+  type = "batch"
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "group" {
+    reschedule {
+      attempts  = 0
+      unlimited = false
+    }
+
+    restart {
+      attempts = 0
+      mode     = "fail"
+    }
+
+    task "env" {
+      driver = "exec2"
+
+      config {
+        command = "env"
+      }
+
+      resources {
+        cpu    = 100
+        memory = 64
+      }
+    }
+  }
+}

e2e/exec2/input/secrets.hcl

@@ -0,0 +1,67 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+# This job writes and reads the secrets directory.
+
+job "secrets" {
+  type = "batch"
+
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "group" {
+    reschedule {
+      attempts  = 0
+      unlimited = false
+    }
+
+    restart {
+      attempts = 0
+      mode     = "fail"
+    }
+
+    task "nomad-token" {
+      driver = "exec2"
+      identity {
+        file = true
+      }
+      config {
+        command = "cat"
+        args    = ["${NOMAD_SECRETS_DIR}/nomad_token"]
+
+        # TODO(shoenig) should not need NOMAD_ values once
+        # https://github.com/hashicorp/nomad-driver-exec2/issues/29 is
+        # fixed.
+        unveil = ["r:${NOMAD_SECRETS_DIR}"]
+      }
+      resources {
+        cpu    = 100
+        memory = 64
+      }
+    }
+
+    task "password" {
+      driver = "exec2"
+      lifecycle {
+        hook    = "prestart"
+        sidecar = false
+      }
+      config {
+        command = "bash"
+        args    = ["-c", "echo abc123 > ${NOMAD_SECRETS_DIR}/password.txt && cat ${NOMAD_SECRETS_DIR}/password.txt"]
+
+        # TODO(shoenig) should not need NOMAD_ values once
+        # https://github.com/hashicorp/nomad-driver-exec2/issues/29 is
+        # fixed.
+        unveil = ["rwc:${NOMAD_SECRETS_DIR}"]
+      }
+      resources {
+        cpu    = 100
+        memory = 64
+      }
+    }
+  }
+}

e2e/terraform/Makefile

@@ -1,5 +1,6 @@
 PKG_PATH = $(shell pwd)/../../pkg/linux_amd64/nomad
-LICENSE_PATH ?=
+NOMAD_LICENSE_PATH ?=
+CONSUL_LICENSE_PATH ?=
 
 # deploy for quick local development testing
 
@@ -8,14 +9,16 @@ plan:
 		-var="nomad_local_binary=$(PKG_PATH)" \
 		-var="volumes=false" \
 		-var="client_count_ubuntu_jammy_amd64=3" \
-		-var="client_count_windows_2016_amd64=0"
+		-var="client_count_windows_2016_amd64=0" \
+		-var="consul_license=$(shell cat $(CONSUL_LICENSE_PATH))"
 
 apply:
 	terraform apply -auto-approve \
 		-var="nomad_local_binary=$(PKG_PATH)" \
 		-var="volumes=false" \
 		-var="client_count_ubuntu_jammy_amd64=3" \
-		-var="client_count_windows_2016_amd64=0"
+		-var="client_count_windows_2016_amd64=0" \
+		-var="consul_license=$(shell cat $(CONSUL_LICENSE_PATH))"
 
 clean: destroy tidy
 
@@ -32,7 +35,7 @@ plan_full:
 
 apply_full:
 	@terraform apply -auto-approve \
-		-var="nomad_license=$(shell cat $(LICENSE_PATH))"
+		-var="nomad_license=$(shell cat $(NOMAD_LICENSE_PATH))"
 
 clean_full: destroy_full tidy
 

e2e/terraform/etc/nomad.d/client-linux.hcl

@@ -51,3 +51,11 @@ plugin "nomad-pledge-driver" {
     pledge_executable = "/usr/local/bin/pledge"
   }
 }
+
+plugin "nomad-driver-exec2" {
+  config {
+    unveil_defaults = true
+    unveil_by_task  = true
+    unveil_paths    = ["r:/etc/mime.types"]
+  }
+}

e2e/terraform/packer/ubuntu-jammy-amd64/setup.sh

@@ -19,6 +19,7 @@ export DEBIAN_FRONTEND=noninteractive
 echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections
 
 mkdir_for_root /opt
+mkdir_for_root /opt/bin # for envoy
 mkdir_for_root /srv/data # for host volumes
 mkdir_for_root /opt/cni/bin
 
@@ -132,6 +133,16 @@ sudo mv /tmp/nomad-pledge-driver ${NOMAD_PLUGIN_DIR}
 sudo mv /tmp/pledge /usr/local/bin
 sudo chmod +x /usr/local/bin/pledge
 
+# Exec2
+echo "Installing Exec2 Driver"
+sudo hc-install install --path ${NOMAD_PLUGIN_DIR} --version v0.1.0-alpha.2 nomad-driver-exec2
+sudo chmod +x ${NOMAD_PLUGIN_DIR}/nomad-driver-exec2
+
+# Envoy
+echo "Installing Envoy"
+sudo curl -s -S -L -o /opt/bin/envoy https://github.com/envoyproxy/envoy/releases/download/v1.30.1/envoy-1.30.1-linux-x86_64
+sudo chmod +x /opt/bin/envoy
+
 # ECS
 if [ -a "/tmp/linux/nomad-driver-ecs" ]; then
     echo "Installing nomad-driver-ecs"