From 2054e871587408e3721046665495f96000636d62 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Fri, 31 May 2024 09:22:39 -0500 Subject: [PATCH] e2e: add tests for exec2 task driver (#22406) * e2e: add tests for exec2 task driver * e2e: use envoy 1.29.4 because consul * e2e: add a bridge networking http test for exec driver * e2e: split up http test so curl always starts after the server --- e2e/exec2/doc.go | 5 + e2e/exec2/exec2_test.go | 100 +++++++++++++++ e2e/exec2/input/countdash.hcl | 119 ++++++++++++++++++ e2e/exec2/input/env.hcl | 38 ++++++ e2e/exec2/input/http.hcl | 74 +++++++++++ e2e/exec2/input/http_curl.hcl | 43 +++++++ e2e/exec2/input/secrets.hcl | 67 ++++++++++ e2e/terraform/Makefile | 11 +- e2e/terraform/etc/nomad.d/client-linux.hcl | 8 ++ .../packer/ubuntu-jammy-amd64/setup.sh | 11 ++ 10 files changed, 472 insertions(+), 4 deletions(-) create mode 100644 e2e/exec2/doc.go create mode 100644 e2e/exec2/exec2_test.go create mode 100644 e2e/exec2/input/countdash.hcl create mode 100644 e2e/exec2/input/env.hcl create mode 100644 e2e/exec2/input/http.hcl create mode 100644 e2e/exec2/input/http_curl.hcl create mode 100644 e2e/exec2/input/secrets.hcl diff --git a/e2e/exec2/doc.go b/e2e/exec2/doc.go new file mode 100644 index 000000000000..f88e105f2983 --- /dev/null +++ b/e2e/exec2/doc.go @@ -0,0 +1,5 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +// Package exec2 contains test cases related to the exec2 task driver. +package exec2 diff --git a/e2e/exec2/exec2_test.go b/e2e/exec2/exec2_test.go new file mode 100644 index 000000000000..67cb3ab58b29 --- /dev/null +++ b/e2e/exec2/exec2_test.go @@ -0,0 +1,100 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package exec2 + +import ( + "fmt" + "regexp" + "testing" + + "github.com/hashicorp/nomad/e2e/v3/cluster3" + "github.com/hashicorp/nomad/e2e/v3/jobs3" + "github.com/shoenig/test/must" +) + +func TestExec2(t *testing.T) { + cluster3.Establish(t, + cluster3.Leader(), + cluster3.LinuxClients(1), + ) + + t.Run("testEnv", testEnv) + t.Run("testSecretsDir", testSecretsDir) + t.Run("testCountdash", testCountdash) + t.Run("testHTTP", testHTTP) +} + +func testEnv(t *testing.T) { + job, cleanup := jobs3.Submit(t, + "./input/env.hcl", + jobs3.WaitComplete("group"), + ) + t.Cleanup(cleanup) + + logs := job.TaskLogs("group", "env") + + // ensure the job id lines up + expect := fmt.Sprintf("NOMAD_JOB_ID=%s", job.JobID()) + must.StrContains(t, logs.Stdout, expect) + + // ensure dynamic user e.g. + // USER=nomad-85249 + userRe := regexp.MustCompile(`nomad-\d+`) + must.RegexMatch(t, userRe, logs.Stdout) +} + +func testSecretsDir(t *testing.T) { + job, cleanup := jobs3.Submit(t, + "./input/secrets.hcl", + jobs3.WaitComplete("group"), + ) + t.Cleanup(cleanup) + + // ensure we can read the workload identity token file + nomadTokenLogs := job.TaskLogs("group", "nomad-token") + tokenRe := regexp.MustCompile(`[\w_-]+`) + must.RegexMatch(t, tokenRe, nomadTokenLogs.Stdout) + + // ensure we can read the written password.txt file + passwordLogs := job.TaskLogs("group", "password") + must.StrContains(t, passwordLogs.Stdout, "abc123") +} + +func testCountdash(t *testing.T) { + job, cleanup := jobs3.Submit(t, + "./input/countdash.hcl", + ) + t.Cleanup(cleanup) + + apiEnvoyLogs := job.TaskLogs("api", "connect-proxy-count-api") + must.StrContains(t, apiEnvoyLogs.Stderr, "all clusters initialized. initializing init manager") + + dashEnvoyLogs := job.TaskLogs("dashboard", "connect-proxy-count-dashboard") + must.StrContains(t, dashEnvoyLogs.Stderr, "all clusters initialized. initializing init manager") + + apiLogs := job.TaskLogs("api", "backend") + must.StrContains(t, apiLogs.Stdout, "Serving at http://localhost:9001") + + dashLogs := job.TaskLogs("dashboard", "dashboard") + must.StrContains(t, dashLogs.Stdout, "Using counting service at http://127.0.0.1:8080") +} + +func testHTTP(t *testing.T) { + job, _ := jobs3.Submit(t, + "./input/http.hcl", + jobs3.DisableCleanup(), + ) + + job2, _ := jobs3.Submit(t, + "./input/http_curl.hcl", + jobs3.DisableCleanup(), + ) + + logs := job.TaskLogs("backend", "http") + must.StrContains(t, logs.Stderr, `"GET / HTTP/1.1" 200 -`) // healthcheck + must.StrContains(t, logs.Stderr, `"GET /hi.html HTTP/1.1" 200 -`) // curl + + logs2 := job2.TaskLogs("client", "curl") + must.StrContains(t, logs2.Stdout, "

Hello, friend!

") +} diff --git a/e2e/exec2/input/countdash.hcl b/e2e/exec2/input/countdash.hcl new file mode 100644 index 000000000000..f2cea8f4002c --- /dev/null +++ b/e2e/exec2/input/countdash.hcl @@ -0,0 +1,119 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +# This is a variation of countdash that uses exec2 for running the envoy +# proxies manually. + +job "countdash" { + group "api" { + network { + mode = "bridge" + } + + service { + name = "count-api" + port = "9001" + + connect { + sidecar_service {} + sidecar_task { + driver = "exec2" + user = "nobody" + config { + command = "/opt/bin/envoy" + args = [ + "-c", + "${NOMAD_SECRETS_DIR}/envoy_bootstrap.json", + "-l", + "${meta.connect.log_level}", + "--concurrency", + "${meta.connect.proxy_concurrency}", + "--disable-hot-restart" + ] + # TODO(shoenig) should not need NOMAD_ values once + # https://github.com/hashicorp/nomad-driver-exec2/issues/29 is + # fixed. + unveil = ["rx:/opt/bin", "rwc:/dev/shm", "r:${NOMAD_TASK_DIR}", "r:${NOMAD_SECRETS_DIR}"] + } + + resources { + cpu = 1000 + memory = 256 + } + } + } + } + + task "backend" { + driver = "docker" + + config { + image = "docker.io/hashicorpdev/counter-api:v3" + } + } + } + + group "dashboard" { + network { + mode = "bridge" + + port "http" { + static = 9002 + to = 9002 + } + } + + service { + name = "count-dashboard" + port = "http" + + connect { + sidecar_service { + proxy { + upstreams { + destination_name = "count-api" + local_bind_port = 8080 + } + } + } + sidecar_task { + driver = "exec2" + user = "nobody" + config { + command = "/opt/bin/envoy" + args = [ + "-c", + "${NOMAD_SECRETS_DIR}/envoy_bootstrap.json", + "-l", + "${meta.connect.log_level}", + "--concurrency", + "${meta.connect.proxy_concurrency}", + "--disable-hot-restart" + ] + # TODO(shoenig) should not need NOMAD_ values once + # https://github.com/hashicorp/nomad-driver-exec2/issues/29 is + # fixed. + unveil = ["rx:/opt/bin", "rwc:/dev/shm", "r:${NOMAD_TASK_DIR}", "r:${NOMAD_SECRETS_DIR}"] + } + + resources { + cpu = 1000 + memory = 256 + } + } + } + } + + task "dashboard" { + driver = "docker" + + env { + COUNTING_SERVICE_URL = "http://${NOMAD_UPSTREAM_ADDR_count_api}" + } + + config { + image = "docker.io/hashicorpdev/counter-dashboard:v3" + } + } + } +} diff --git a/e2e/exec2/input/env.hcl b/e2e/exec2/input/env.hcl new file mode 100644 index 000000000000..1dd2fa95c120 --- /dev/null +++ b/e2e/exec2/input/env.hcl @@ -0,0 +1,38 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +# This is a simple env job using the exec2 task driver. + +job "env" { + type = "batch" + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + group "group" { + reschedule { + attempts = 0 + unlimited = false + } + + restart { + attempts = 0 + mode = "fail" + } + + task "env" { + driver = "exec2" + + config { + command = "env" + } + + resources { + cpu = 100 + memory = 64 + } + } + } +} diff --git a/e2e/exec2/input/http.hcl b/e2e/exec2/input/http.hcl new file mode 100644 index 000000000000..e1a645459668 --- /dev/null +++ b/e2e/exec2/input/http.hcl @@ -0,0 +1,74 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +# This job serves the NOMAD_TASK_DIR over http. + +job "http" { + type = "service" + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + group "backend" { + network { + mode = "bridge" + port "http" { + to = "9999" + } + } + + task "http" { + driver = "exec2" + + service { + name = "python-http" + port = "http" + provider = "nomad" + check { + name = "hi" + type = "http" + path = "/" + interval = "3s" + timeout = "1s" + } + } + + config { + command = "python3" + args = ["-m", "http.server", "9999", "--directory", "${NOMAD_TASK_DIR}"] + } + + template { + destination = "local/hi.html" + data = < + + example +

Hello, friend!

+ + EOH + } + + resources { + cpu = 500 + memory = 256 + } + } + + restart { + attempts = 0 + mode = "fail" + } + + reschedule { + attempts = 0 + unlimited = false + } + + update { + min_healthy_time = "5s" + } + } +} diff --git a/e2e/exec2/input/http_curl.hcl b/e2e/exec2/input/http_curl.hcl new file mode 100644 index 000000000000..cbbb15053cb0 --- /dev/null +++ b/e2e/exec2/input/http_curl.hcl @@ -0,0 +1,43 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +# This job makes requests to the "python-http" service. + +job "http_curl" { + type = "service" + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + group "client" { + task "curl" { + driver = "exec2" + + config { + command = "bash" + args = ["local/script.sh"] + } + + template { + destination = "local/script.sh" + change_mode = "noop" + data = < ${NOMAD_SECRETS_DIR}/password.txt && cat ${NOMAD_SECRETS_DIR}/password.txt"] + + # TODO(shoenig) should not need NOMAD_ values once + # https://github.com/hashicorp/nomad-driver-exec2/issues/29 is + # fixed. + unveil = ["rwc:${NOMAD_SECRETS_DIR}"] + } + resources { + cpu = 100 + memory = 64 + } + } + } +} diff --git a/e2e/terraform/Makefile b/e2e/terraform/Makefile index 6bbb785dd46a..acdf419f11d1 100644 --- a/e2e/terraform/Makefile +++ b/e2e/terraform/Makefile @@ -1,5 +1,6 @@ PKG_PATH = $(shell pwd)/../../pkg/linux_amd64/nomad -LICENSE_PATH ?= +NOMAD_LICENSE_PATH ?= +CONSUL_LICENSE_PATH ?= # deploy for quick local development testing @@ -8,14 +9,16 @@ plan: -var="nomad_local_binary=$(PKG_PATH)" \ -var="volumes=false" \ -var="client_count_ubuntu_jammy_amd64=3" \ - -var="client_count_windows_2016_amd64=0" + -var="client_count_windows_2016_amd64=0" \ + -var="consul_license=$(shell cat $(CONSUL_LICENSE_PATH))" apply: terraform apply -auto-approve \ -var="nomad_local_binary=$(PKG_PATH)" \ -var="volumes=false" \ -var="client_count_ubuntu_jammy_amd64=3" \ - -var="client_count_windows_2016_amd64=0" + -var="client_count_windows_2016_amd64=0" \ + -var="consul_license=$(shell cat $(CONSUL_LICENSE_PATH))" clean: destroy tidy @@ -32,7 +35,7 @@ plan_full: apply_full: @terraform apply -auto-approve \ - -var="nomad_license=$(shell cat $(LICENSE_PATH))" + -var="nomad_license=$(shell cat $(NOMAD_LICENSE_PATH))" clean_full: destroy_full tidy diff --git a/e2e/terraform/etc/nomad.d/client-linux.hcl b/e2e/terraform/etc/nomad.d/client-linux.hcl index 3b8dce400a49..ef9a33efedb9 100644 --- a/e2e/terraform/etc/nomad.d/client-linux.hcl +++ b/e2e/terraform/etc/nomad.d/client-linux.hcl @@ -51,3 +51,11 @@ plugin "nomad-pledge-driver" { pledge_executable = "/usr/local/bin/pledge" } } + +plugin "nomad-driver-exec2" { + config { + unveil_defaults = true + unveil_by_task = true + unveil_paths = ["r:/etc/mime.types"] + } +} diff --git a/e2e/terraform/packer/ubuntu-jammy-amd64/setup.sh b/e2e/terraform/packer/ubuntu-jammy-amd64/setup.sh index c48b31088410..e3e71b7dae39 100755 --- a/e2e/terraform/packer/ubuntu-jammy-amd64/setup.sh +++ b/e2e/terraform/packer/ubuntu-jammy-amd64/setup.sh @@ -19,6 +19,7 @@ export DEBIAN_FRONTEND=noninteractive echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections mkdir_for_root /opt +mkdir_for_root /opt/bin # for envoy mkdir_for_root /srv/data # for host volumes mkdir_for_root /opt/cni/bin @@ -132,6 +133,16 @@ sudo mv /tmp/nomad-pledge-driver ${NOMAD_PLUGIN_DIR} sudo mv /tmp/pledge /usr/local/bin sudo chmod +x /usr/local/bin/pledge +# Exec2 +echo "Installing Exec2 Driver" +sudo hc-install install --path ${NOMAD_PLUGIN_DIR} --version v0.1.0-alpha.2 nomad-driver-exec2 +sudo chmod +x ${NOMAD_PLUGIN_DIR}/nomad-driver-exec2 + +# Envoy +echo "Installing Envoy" +sudo curl -s -S -L -o /opt/bin/envoy https://github.com/envoyproxy/envoy/releases/download/v1.29.4/envoy-1.29.4-linux-x86_64 +sudo chmod +x /opt/bin/envoy + # ECS if [ -a "/tmp/linux/nomad-driver-ecs" ]; then echo "Installing nomad-driver-ecs"