Ensure dockertests are running against most up to date benchmark binary instead of latest docker tag

Makefile

@@ -23,6 +23,7 @@ LATEST_TAG?=$(REGISTRY_NAME)/$(PRODUCT_NAME):latest
 BUILD_DIR=dist
 GOFMT_FILES?=$$(find . -name '*.go' | grep -v vendor)
 LDFLAGS?="-X '$(PKG).Version=v$(VERSION)'"
+BENCHMARK_IMAGES?=$$(docker images --format="{{json .}}" | jq -r 'select(.Tag=="latest-benchmark-testing" or .Tag=="<none>")'.ID)
 
 dist:
 	mkdir -p $(DIST)
@@ -32,6 +33,14 @@ dist:
 bin: dist
 	CGO_ENABLED=0 GOARCH=$(ARCH) GOOS=$(OS) go build -o $(BIN)
 
+PHONY: dockerbin
+dockerbin: dist
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o dist/linux/amd64/vault-benchmark
+
+PHONY: cleanupimages
+cleanupimages: 
+	docker rmi $(BENCHMARK_IMAGES)
+
 .PHONY: build
 build:
 	@$(CURDIR)/scripts/crt-build.sh build

helper/dockertest/dockerjobs/benchmark_run.go

@@ -6,23 +6,46 @@ package dockerjobs
 import (
 	"context"
 	"fmt"
+	"io"
+	"os"
 	"testing"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/client"
 	dockhelper "github.com/hashicorp/vault/sdk/helper/docker"
 )
 
 func CreateVaultBenchmarkContainer(t *testing.T, vaultAddr string, vaultToken string, configFile string) (func(), int64) {
 	ctx := context.Background()
+	binary := os.Getenv("BENCHMARK_BINARY")
+
+	if binary == "" {
+		t.Skip("only running docker test when $BENCHMARK_BINARY present")
+	}
+
+	api, err := dockhelper.NewDockerAPI()
+
+	if err != nil {
+		t.Fatalf("Error starting docker api: %s", err)
+	}
+
+	imageRepo := "hashicorp/vault-benchmark"
+	imageTag := "latest"
+	tag, err := setupBenchmarkImage(ctx, imageRepo, imageTag, binary, api)
+
+	if err != nil {
+		t.Fatalf("Error setting up benchmark image: %s", err)
+	}
+
 	volume := map[string]string{
 		"configs/": "/etc/",
 	}
 
 	runOpts := dockhelper.RunOptions{
 		ContainerName:   "vault-benchmark",
-		ImageRepo:       "docker.mirror.hashicorp.services/hashicorp/vault-benchmark",
-		ImageTag:        "0.1",
+		ImageRepo:       imageRepo,
+		ImageTag:        tag,
 		DoNotAutoRemove: true,
 		Env:             []string{fmt.Sprintf("VAULT_ADDR=%s", vaultAddr), fmt.Sprintf("VAULT_TOKEN=%s", vaultToken)},
 		CopyFromTo:      volume,
@@ -41,7 +64,6 @@ func CreateVaultBenchmarkContainer(t *testing.T, vaultAddr string, vaultToken st
 
 	exitCh, errCh := runner.DockerAPI.ContainerWait(ctx, service.Container.ID, container.WaitConditionNotRunning)
 
-
 	// wait until benchmark exit
 	var exitCode int64
 	select {
@@ -62,3 +84,40 @@ func CreateVaultBenchmarkContainer(t *testing.T, vaultAddr string, vaultToken st
 
 	return cleanup, exitCode
 }
+
+func setupBenchmarkImage(ctx context.Context, imageRepo string, imageTag string, binary string, dAPI *client.Client) (string, error) {
+	suffix := "benchmark-testing"
+
+	tag := imageTag + "-" + suffix
+
+	f, err := os.Open(binary)
+	if err != nil {
+		return "", err
+	}
+
+	data, err := io.ReadAll(f)
+	if err != nil {
+		return "", err
+	}
+
+	bCtx := dockhelper.NewBuildContext()
+	bCtx["vault-benchmark"] = &dockhelper.FileContents{
+		Data: data,
+		Mode: 0o755,
+	}
+
+	containerFile := fmt.Sprintf(`
+FROM %s:%s
+COPY vault-benchmark /bin/vault-benchmark
+`, imageRepo, imageTag)
+
+	_, err = dockhelper.BuildImage(ctx, dAPI, containerFile, bCtx,
+		dockhelper.BuildRemove(true), dockhelper.BuildForceRemove(true),
+		dockhelper.BuildPullParent(true),
+		dockhelper.BuildTags([]string{imageRepo + ":" + tag}))
+	if err != nil {
+		return "", err
+	}
+
+	return tag, nil
+}

test-fixtures/configs/userpass.hcl

@@ -10,7 +10,6 @@ test "userpass_auth" "userpass_test1" {
     config {
         username = "test-user"
         password = "password"
-        token_bound_cidrs = ["1.2.3.4/32"]
         token_no_default_policy = true
         token_num_uses = 500
         token_period = "3m"