diff --git a/.github/workflows/checks-codecov.yaml b/.github/workflows/checks-codecov.yaml index e69b83d2b..213933f02 100644 --- a/.github/workflows/checks-codecov.yaml +++ b/.github/workflows/checks-codecov.yaml @@ -109,11 +109,15 @@ jobs: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Restore Cache - uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + - name: Cache Go build and module artifacts + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: - key: main - path: '**' + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: go-acceptance-${{ runner.os }}-${{ hashFiles('go.sum', 'tools/go.sum', 'tools/kubectl/go.sum') }} + restore-keys: | + go-acceptance-${{ runner.os }}- - name: Setup Go environment uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 @@ -121,6 +125,19 @@ jobs: go-version-file: go.mod cache: false + - name: Install tkn CLI + run: | + TKN_VERSION=$(grep 'tektoncd/cli' tools/go.mod | awk '{print $2}' | sed 's/^v//') + curl -fsSL "https://github.com/tektoncd/cli/releases/download/v${TKN_VERSION}/tkn_${TKN_VERSION}_Linux_x86_64.tar.gz" \ + | sudo tar xz -C /usr/local/bin tkn + + - name: Install kubectl + run: | + KUBECTL_VERSION=$(grep 'k8s.io/kubernetes' tools/kubectl/go.mod | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+') + sudo curl -fsSL "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \ + -o /usr/local/bin/kubectl + sudo chmod +x /usr/local/bin/kubectl + - name: Update podman run: | "${GITHUB_WORKSPACE}/hack/ubuntu-podman-update.sh" diff --git a/Makefile b/Makefile index 41d77aaf2..07b9c34a5 100644 --- a/Makefile +++ b/Makefile @@ -121,7 +121,9 @@ ACCEPTANCE_TIMEOUT:=20m .PHONY: acceptance acceptance: ## Run all acceptance tests - @ACCEPTANCE_WORKDIR="$$(mktemp -d)"; \ + @SECONDS=0; \ + echo "[`date '+%H:%M:%S'`] Starting acceptance tests"; \ + ACCEPTANCE_WORKDIR="$$(mktemp -d)"; \ cleanup() { \ cp "$${ACCEPTANCE_WORKDIR}"/features/__snapshots__/* "$(ROOT_DIR)"/features/__snapshots__/; \ }; \ @@ -129,9 +131,13 @@ acceptance: ## Run all acceptance tests trap cleanup EXIT; \ cp -R . "$$ACCEPTANCE_WORKDIR"; \ cd "$$ACCEPTANCE_WORKDIR" && \ - $(MAKE) build && \ + $(MAKE) build E2E_INSTRUMENTATION=true && \ + echo "[`date '+%H:%M:%S'`] Build done, running tests"; \ export GOCOVERDIR="$${ACCEPTANCE_WORKDIR}/coverage"; \ - cd acceptance && go test -timeout $(ACCEPTANCE_TIMEOUT) ./... ; go tool covdata textfmt -i=$${GOCOVERDIR} -o="$(ROOT_DIR)/coverage-acceptance.out" + cd acceptance && go test -timeout $(ACCEPTANCE_TIMEOUT) ./... && test_passed=1 || test_passed=0; \ + echo "[`date '+%H:%M:%S'`] Tests finished in $$((SECONDS/60))m$$((SECONDS%60))s"; \ + go tool covdata textfmt -i=$${GOCOVERDIR} -o="$(ROOT_DIR)/coverage-acceptance.out" || true; \ + [ "$$test_passed" = "1" ] # Add @focus above the feature you're hacking on to use this # (Mainly for use with the feature-% target below) @@ -340,9 +346,10 @@ TASKS ?= tasks/verify-enterprise-contract/0.1/verify-enterprise-contract.yaml,ta ifneq (,$(findstring localhost:,$(TASK_REPO))) SKOPEO_ARGS=--src-tls-verify=false --dest-tls-verify=false endif +TKN ?= $(shell command -v tkn 2>/dev/null || echo "go run -modfile tools/go.mod github.com/tektoncd/cli/cmd/tkn") .PHONY: task-bundle task-bundle: ## Push the Tekton Task bundle to an image repository - @go run -modfile tools/go.mod github.com/tektoncd/cli/cmd/tkn bundle push $(TASK_REPO):$(TASK_TAG) $(addprefix -f ,$(TASKS)) --annotate org.opencontainers.image.revision="$(TASK_TAG)" + @$(TKN) bundle push $(TASK_REPO):$(TASK_TAG) $(addprefix -f ,$(TASKS)) --annotate org.opencontainers.image.revision="$(TASK_TAG)" .PHONY: task-bundle-snapshot task-bundle-snapshot: task-bundle ## Push task bundle and then tag with "snapshot" diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 77df1f4ec..7756517c1 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -20,6 +20,7 @@ import ( "context" "flag" "fmt" + "io" "os" "path/filepath" "runtime" @@ -28,6 +29,7 @@ import ( "github.com/cucumber/godog" "github.com/gkampitakis/go-snaps/snaps" + "k8s.io/klog/v2" "github.com/conforma/cli/acceptance/cli" "github.com/conforma/cli/acceptance/conftest" @@ -55,17 +57,23 @@ var restore = flag.Bool("restore", false, "restore last persisted environment") var noColors = flag.Bool("no-colors", false, "disable colored output") +var verbose = flag.Bool("verbose", false, "show stdout/stderr in failure output") + // specify a subset of scenarios to run filtering by given tags var tags = flag.String("tags", "", "select scenarios to run based on tags") // random seed to use var seed = flag.Int64("seed", -1, "random seed to use for the tests") +// godog output formatter (pretty, progress, cucumber, junit, events) +var format = flag.String("format", "", "godog output formatter (default: progress, or set EC_ACCEPTANCE_FORMAT)") + // failedScenario tracks information about a failed scenario type failedScenario struct { Name string Location string Error error + LogFile string } // scenarioTracker tracks failed scenarios across all test runs @@ -74,17 +82,18 @@ type scenarioTracker struct { failedScenarios []failedScenario } -func (st *scenarioTracker) addFailure(name, location string, err error) { +func (st *scenarioTracker) addFailure(name, location, logFile string, err error) { st.mu.Lock() defer st.mu.Unlock() st.failedScenarios = append(st.failedScenarios, failedScenario{ Name: name, Location: location, Error: err, + LogFile: logFile, }) } -func (st *scenarioTracker) printSummary(t *testing.T) { +func (st *scenarioTracker) printSummary() { st.mu.Lock() defer st.mu.Unlock() @@ -99,8 +108,8 @@ func (st *scenarioTracker) printSummary(t *testing.T) { for i, fs := range st.failedScenarios { fmt.Fprintf(os.Stderr, "%d. %s\n", i+1, fs.Name) fmt.Fprintf(os.Stderr, " Location: %s\n", fs.Location) - if fs.Error != nil { - fmt.Fprintf(os.Stderr, " Error: %v\n", fs.Error) + if fs.LogFile != "" { + fmt.Fprintf(os.Stderr, " Log file: %s\n", fs.LogFile) } if i < len(st.failedScenarios)-1 { fmt.Fprintf(os.Stderr, "\n") @@ -136,29 +145,22 @@ func initializeScenario(sc *godog.ScenarioContext) { }) sc.After(func(ctx context.Context, scenario *godog.Scenario, scenarioErr error) (context.Context, error) { - // Log scenario end with status - write to /dev/tty to bypass capture - if tty, err := os.OpenFile("/dev/tty", os.O_WRONLY, 0); err == nil { - // Strip the working directory prefix to show relative paths - uri := scenario.Uri - if cwd, err := os.Getwd(); err == nil { - if rel, err := filepath.Rel(cwd, uri); err == nil { - uri = rel - } - } - - if scenarioErr != nil { - fmt.Fprintf(tty, "✗ FAILED: %s (%s)\n", scenario.Name, uri) - } else { - fmt.Fprintf(tty, "✓ PASSED: %s (%s)\n", scenario.Name, uri) - } - tty.Close() - } + logger, ctx := log.LoggerFor(ctx) + + logFile := logger.LogFile() + logger.Close() if scenarioErr != nil { - tracker.addFailure(scenario.Name, scenario.Uri, scenarioErr) + tracker.addFailure(scenario.Name, scenario.Uri, logFile, scenarioErr) } _, err := testenv.Persist(ctx) + + if scenarioErr == nil { + // Clean up log files for passing scenarios + os.Remove(logFile) + } + return ctx, err }) } @@ -176,6 +178,7 @@ func setupContext(t *testing.T) context.Context { ctx = context.WithValue(ctx, testenv.PersistStubEnvironment, *persist) ctx = context.WithValue(ctx, testenv.RestoreStubEnvironment, *restore) ctx = context.WithValue(ctx, testenv.NoColors, *noColors) + ctx = context.WithValue(ctx, testenv.VerboseOutput, *verbose) return ctx } @@ -196,8 +199,16 @@ func TestFeatures(t *testing.T) { ctx := setupContext(t) + godogFormat := "progress:/dev/null" + if f := os.Getenv("EC_ACCEPTANCE_FORMAT"); f != "" { + godogFormat = f + } + if *format != "" { + godogFormat = *format + } + opts := godog.Options{ - Format: "pretty", + Format: godogFormat, Paths: []string{featuresDir}, Randomize: *seed, Concurrency: runtime.NumCPU(), @@ -216,18 +227,23 @@ func TestFeatures(t *testing.T) { exitCode := suite.Run() - // Print summary of failed scenarios - tracker.printSummary(t) - if exitCode != 0 { - // Exit directly without t.Fatal to avoid verbose Go test output - os.Exit(1) + t.Fatalf("acceptance test suite failed with exit code %d", exitCode) } } func TestMain(t *testing.M) { + // Suppress k8s client-side throttling warnings that pollute test output. + // LogToStderr(false) is required because klog defaults to writing directly + // to stderr, ignoring any writer set via SetOutput. + klog.LogToStderr(false) + klog.SetOutput(io.Discard) + v := t.Run() + // Print summaries after all go test output so they appear last + tracker.printSummary() + // After all tests have run `go-snaps` can check for not used snapshots if _, err := snaps.Clean(t); err != nil { fmt.Println("Error cleaning snaps:", err) diff --git a/acceptance/cli/cli.go b/acceptance/cli/cli.go index 8d345d0c9..8e4cfab70 100644 --- a/acceptance/cli/cli.go +++ b/acceptance/cli/cli.go @@ -560,7 +560,12 @@ func theStandardErrorShouldContain(ctx context.Context, expected *godog.DocStrin return nil } - return fmt.Errorf("expected error:\n%s\nnot found in standard error:\n%s", expected, stderr) + var b bytes.Buffer + if diffErr := diff.Text("stderr", "expected", status.stderr, expectedStdErr, &b); diffErr != nil { + return fmt.Errorf("expected error:\n%s\nnot found in standard error:\n%s", expected, stderr) + } + + return fmt.Errorf("expected and actual stderr differ:\n%s", b.String()) } // theStandardOutputShouldMatchBaseline reads the expected text from a file instead of directly @@ -714,40 +719,44 @@ func EcStatusFrom(ctx context.Context) (*status, error) { // logExecution logs the details of the execution and offers hits as how to // troubleshoot test failures by using persistent environment func logExecution(ctx context.Context) { - noColors := testenv.NoColorOutput(ctx) - if c.SUPPORT_COLOR != !noColors { - c.SUPPORT_COLOR = !noColors - } - s, err := ecStatusFrom(ctx) if err != nil { return // the ec wasn't invoked no status was stored } - output := &strings.Builder{} - outputSegment := func(name string, v any) { - output.WriteString("\n\n") - output.WriteString(c.Underline(c.Bold(name))) - output.WriteString(fmt.Sprintf("\n%v", v)) + noColors := testenv.NoColorOutput(ctx) + if c.SUPPORT_COLOR != !noColors { + c.SUPPORT_COLOR = !noColors } - outputSegment("Command", s.Cmd) - outputSegment("State", fmt.Sprintf("Exit code: %d\nPid: %d", s.ProcessState.ExitCode(), s.ProcessState.Pid())) - outputSegment("Environment", strings.Join(s.Env, "\n")) - var varsStr []string - for k, v := range s.vars { - varsStr = append(varsStr, fmt.Sprintf("%s=%s", k, v)) - } - outputSegment("Variables", strings.Join(varsStr, "\n")) - if s.stdout.Len() == 0 { - outputSegment("Stdout", c.Italic("* No standard output")) - } else { - outputSegment("Stdout", c.Green(s.stdout.String())) - } - if s.stderr.Len() == 0 { - outputSegment("Stdout", c.Italic("* No standard error")) - } else { - outputSegment("Stderr", c.Red(s.stderr.String())) + verbose, _ := ctx.Value(testenv.VerboseOutput).(bool) + if verbose { + output := &strings.Builder{} + outputSegment := func(name string, v any) { + output.WriteString("\n\n") + output.WriteString(c.Underline(c.Bold(name))) + output.WriteString(fmt.Sprintf("\n%v", v)) + } + + outputSegment("Command", s.Cmd) + outputSegment("State", fmt.Sprintf("Exit code: %d\nPid: %d", s.ProcessState.ExitCode(), s.ProcessState.Pid())) + outputSegment("Environment", strings.Join(s.Env, "\n")) + var varsStr []string + for k, v := range s.vars { + varsStr = append(varsStr, fmt.Sprintf("%s=%s", k, v)) + } + outputSegment("Variables", strings.Join(varsStr, "\n")) + if s.stdout.Len() == 0 { + outputSegment("Stdout", c.Italic("* No standard output")) + } else { + outputSegment("Stdout", c.Green(s.stdout.String())) + } + if s.stderr.Len() == 0 { + outputSegment("Stderr", c.Italic("* No standard error")) + } else { + outputSegment("Stderr", c.Red(s.stderr.String())) + } + fmt.Print(output.String()) } if testenv.Persisted(ctx) { @@ -758,12 +767,11 @@ func logExecution(ctx context.Context) { } } - output.WriteString("\n" + c.Bold("NOTE") + ": " + fmt.Sprintf("The test environment is persisted, to recreate the failure run:\n%s %s\n\n", strings.Join(environment, " "), strings.Join(s.Cmd.Args, " "))) + fmt.Printf("\n%s: The test environment is persisted, to recreate the failure run:\n%s %s\n\n", + c.Bold("NOTE"), strings.Join(environment, " "), strings.Join(s.Cmd.Args, " ")) } else { - output.WriteString("\n" + c.Bold("HINT") + ": To recreate the failure re-run the test with `-args -persist` to persist the stubbed environment\n\n") + fmt.Printf("\n%s: To recreate the failure re-run the test with `-args -persist` to persist the stubbed environment, or `-args -verbose` for detailed execution output\n\n", c.Bold("HINT")) } - - fmt.Print(output.String()) } func matchSnapshot(ctx context.Context) error { @@ -852,7 +860,9 @@ func AddStepsTo(sc *godog.ScenarioContext) { sc.Step(`^a file named "([^"]*)" containing$`, createGenericFile) sc.Step(`^a track bundle file named "([^"]*)" containing$`, createTrackBundleFile) sc.After(func(ctx context.Context, sc *godog.Scenario, err error) (context.Context, error) { - logExecution(ctx) + if err != nil { + logExecution(ctx) + } return ctx, nil }) diff --git a/acceptance/conftest/conftest.go b/acceptance/conftest/conftest.go index f617b9fc3..5c487a603 100644 --- a/acceptance/conftest/conftest.go +++ b/acceptance/conftest/conftest.go @@ -91,7 +91,12 @@ func runConftest(ctx context.Context, command, produces string, content *godog.D var stderr bytes.Buffer cmd.Stdout = &stdout cmd.Stderr = &stderr + var cmdErr error defer func() { + if cmdErr == nil { + return + } + noColors := testenv.NoColorOutput(ctx) if c.SUPPORT_COLOR != !noColors { c.SUPPORT_COLOR = !noColors @@ -105,8 +110,8 @@ func runConftest(ctx context.Context, command, produces string, content *godog.D fmt.Printf("\n\t%s", strings.ReplaceAll(stderr.String(), "\n", "\n\t")) }() - if err := cmd.Run(); err != nil { - return fmt.Errorf("failure running conftest: %w", err) + if cmdErr = cmd.Run(); cmdErr != nil { + return fmt.Errorf("failure running conftest: %w", cmdErr) } buff, err := os.ReadFile(path.Join(dir, produces)) diff --git a/acceptance/git/git.go b/acceptance/git/git.go index bcc632849..a683207a5 100644 --- a/acceptance/git/git.go +++ b/acceptance/git/git.go @@ -59,6 +59,7 @@ type gitState struct { RepositoriesDir string CertificatePath string LatestCommit string + Container testcontainers.Container `json:"-"` } func (g gitState) Key() any { @@ -188,6 +189,8 @@ func startStubGitServer(ctx context.Context) (context.Context, error) { return ctx, err } + state.Container = git + port, err := git.MappedPort(ctx, "443/tcp") if err != nil { return ctx, err @@ -314,7 +317,7 @@ func AddStepsTo(sc *godog.ScenarioContext) { sc.Step(`^stub git daemon running$`, startStubGitServer) sc.Step(`^a git repository named "([^"]*)" with$`, createGitRepository) - // removes all git repositories from the filesystem + // removes all git repositories from the filesystem and terminates the container sc.After(func(ctx context.Context, finished *godog.Scenario, scenarioErr error) (context.Context, error) { if testenv.Persisted(ctx) { return ctx, nil @@ -329,6 +332,13 @@ func AddStepsTo(sc *godog.ScenarioContext) { return ctx, nil } + if state.Container != nil { + if err := state.Container.Terminate(ctx); err != nil { + logger, _ := log.LoggerFor(ctx) + logger.Warnf("failed to terminate git container: %v", err) + } + } + os.RemoveAll(state.RepositoriesDir) return ctx, nil diff --git a/acceptance/go.mod b/acceptance/go.mod index db377fe3c..c43120bff 100644 --- a/acceptance/go.mod +++ b/acceptance/go.mod @@ -33,10 +33,12 @@ require ( github.com/wiremock/go-wiremock v1.11.0 github.com/yudai/gojsondiff v1.0.0 golang.org/x/exp v0.0.0-20250911091902-df9299821621 + golang.org/x/sync v0.20.0 gopkg.in/go-jose/go-jose.v2 v2.6.3 k8s.io/api v0.35.3 k8s.io/apimachinery v0.35.3 k8s.io/client-go v0.35.3 + k8s.io/klog/v2 v2.130.1 oras.land/oras-go/v2 v2.6.0 sigs.k8s.io/kind v0.26.0 sigs.k8s.io/kustomize/api v0.20.1 @@ -209,7 +211,6 @@ require ( github.com/skeema/knownhosts v1.3.1 // indirect github.com/spf13/cobra v1.10.2 // indirect github.com/spf13/pflag v1.0.10 // indirect - github.com/stretchr/objx v0.5.2 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect github.com/theupdateframework/go-tuf v0.7.0 // indirect github.com/theupdateframework/go-tuf/v2 v2.4.1 // indirect @@ -246,7 +247,6 @@ require ( golang.org/x/mod v0.33.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.42.0 // indirect golang.org/x/term v0.41.0 // indirect golang.org/x/text v0.35.0 // indirect @@ -265,7 +265,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.34.3 // indirect k8s.io/cli-runtime v0.34.2 // indirect - k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect knative.dev/pkg v0.0.0-20250415155312-ed3e2158b883 // indirect diff --git a/acceptance/kubernetes/kind/acceptance.Dockerfile b/acceptance/kubernetes/kind/acceptance.Dockerfile new file mode 100644 index 000000000..98d5a986a --- /dev/null +++ b/acceptance/kubernetes/kind/acceptance.Dockerfile @@ -0,0 +1,35 @@ +# Copyright The Conforma Contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +# Minimal image for acceptance tests. The ec and kubectl binaries are +# pre-built on the host and injected here to avoid the multi-stage Go +# compilation that the production Dockerfile uses. +FROM registry.access.redhat.com/ubi9/ubi-minimal:latest@sha256:83006d535923fcf1345067873524a3980316f51794f01d8655be55d6e9387183 + +RUN microdnf upgrade --assumeyes --nodocs --setopt=keepcache=0 --refresh && microdnf -y --nodocs --setopt=keepcache=0 install gzip jq ca-certificates + +ARG EC_BINARY +ARG KUBECTL_BINARY + +COPY ${EC_BINARY} /usr/local/bin/ec +COPY ${KUBECTL_BINARY} /usr/local/bin/kubectl +COPY hack/reduce-snapshot.sh /usr/local/bin/ + +RUN ln -s /usr/local/bin/ec /usr/local/bin/conforma + +USER 1001 + +ENTRYPOINT ["/usr/local/bin/ec"] diff --git a/acceptance/kubernetes/kind/image.go b/acceptance/kubernetes/kind/image.go index eb0fa2d3d..6750852b9 100644 --- a/acceptance/kubernetes/kind/image.go +++ b/acceptance/kubernetes/kind/image.go @@ -20,6 +20,8 @@ import ( "archive/tar" "compress/gzip" "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "io" @@ -30,32 +32,187 @@ import ( imagespecv1 "github.com/opencontainers/image-spec/specs-go/v1" v1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1" + "golang.org/x/sync/errgroup" + corev1 "k8s.io/api/core/v1" "oras.land/oras-go/v2" orasFile "oras.land/oras-go/v2/content/file" "oras.land/oras-go/v2/registry/remote" "sigs.k8s.io/yaml" + "github.com/conforma/cli/acceptance/log" "github.com/conforma/cli/acceptance/testenv" ) -// buildCliImage runs `make push-image` to build and push the image to the Kind -// cluster. The image is pushed to -// `localhost:/cli:latest--`, see push-image -// Makefile target for details. The registry is running without TLS, so we need -// `--tls-verify=false` here. - +// buildCliImage builds the ec and kubectl binaries locally, then constructs a +// minimal container image and pushes it to the Kind cluster registry. The image +// is pushed to `localhost:/cli:latest--`. Building the +// binaries on the host leverages the warm Go build cache, avoiding the +// redundant Go compilation that the multi-stage production Dockerfile performs. +// +// A content hash of the build inputs is computed and compared against a cache +// marker file. When the hash matches, the build is skipped entirely. func (k *kindCluster) buildCliImage(ctx context.Context) error { - cmd := exec.CommandContext(ctx, "make", "push-image", fmt.Sprintf("IMAGE_REPO=localhost:%d/cli", k.registryPort), "PODMAN_OPTS=--tls-verify=false") /* #nosec */ + currentHash, err := computeSourceHash() + var cacheFile string + if err != nil { + // On hash failure, fall through to a full build + fmt.Printf("[WARN] Failed to compute source hash, rebuilding: %v\n", err) + } else { + cacheFile = fmt.Sprintf("/tmp/ec-cli-image-cache-%d.hash", k.registryPort) + if cached, err := os.ReadFile(cacheFile); err == nil && string(cached) == currentHash { + fmt.Println("[INFO] CLI image cache hit, skipping build") + return nil + } + } + + // Build into a directory not excluded by .dockerignore (which excludes + // dist/) and not conflicting with the versioned binary from make build. + buildDir := ".acceptance-build" + if err := os.MkdirAll(buildDir, 0755); err != nil { + return fmt.Errorf("creating build directory: %w", err) + } + defer os.RemoveAll(buildDir) + + // Derive version the same way as the Makefile + versionCmd := exec.CommandContext(ctx, "hack/derive-version.sh") // #nosec G204 + versionOut, err := versionCmd.CombinedOutput() + if err != nil { + fmt.Printf("[WARN] Failed to derive version, building without: %v\n", err) + versionOut = nil + } + version := strings.TrimSpace(string(versionOut)) + + // Build ec binary locally + ldflags := "-s -w" + if version != "" { + ldflags += " -X github.com/conforma/cli/internal/version.Version=" + version + } + ecBinary := filepath.Join(buildDir, "ec") + ecBuildCmd := exec.CommandContext(ctx, "go", "build", "-trimpath", "--mod=readonly", fmt.Sprintf("-ldflags=%s", ldflags), "-o", ecBinary) // #nosec G204 + if out, err := ecBuildCmd.CombinedOutput(); err != nil { + fmt.Printf("[ERROR] Failed to build ec binary, %q returned an error: %v\nCommand output:\n", ecBuildCmd, err) + fmt.Print(string(out)) + return err + } + + // Use pre-built kubectl from PATH if available, otherwise build from source + kubectlBinary := filepath.Join(buildDir, "kubectl") + if kubectlPath, err := exec.LookPath("kubectl"); err == nil { + fmt.Printf("[INFO] Using pre-built kubectl from %s\n", kubectlPath) + if err := copyFile(kubectlPath, kubectlBinary); err != nil { + return fmt.Errorf("copying kubectl binary: %w", err) + } + } else { + kubectlBuildCmd := exec.CommandContext(ctx, "go", "build", "-trimpath", "--mod=readonly", "-modfile", "tools/kubectl/go.mod", "-o", kubectlBinary, "k8s.io/kubernetes/cmd/kubectl") // #nosec G204 + if out, err := kubectlBuildCmd.CombinedOutput(); err != nil { + fmt.Printf("[ERROR] Failed to build kubectl binary, %q returned an error: %v\nCommand output:\n", kubectlBuildCmd, err) + fmt.Print(string(out)) + return err + } + } + + // Build the container image using the minimal acceptance Dockerfile + imgTag, err := getTag(ctx) + if err != nil { + return fmt.Errorf("getting image tag: %w", err) + } + imageRef := fmt.Sprintf("localhost:%d/cli:%s", k.registryPort, imgTag) + + buildImgCmd := exec.CommandContext(ctx, "podman", "build", // #nosec G204 + "-t", imageRef, + "-f", "acceptance/kubernetes/kind/acceptance.Dockerfile", + "--build-arg", fmt.Sprintf("EC_BINARY=%s", ecBinary), + "--build-arg", fmt.Sprintf("KUBECTL_BINARY=%s", kubectlBinary), + ".") + if out, err := buildImgCmd.CombinedOutput(); err != nil { + fmt.Printf("[ERROR] Failed to build CLI image, %q returned an error: %v\nCommand output:\n", buildImgCmd, err) + fmt.Print(string(out)) + return err + } - if out, err := cmd.CombinedOutput(); err != nil { - fmt.Printf("[ERROR] Unable to build and push the CLI image, %q returned an error: %v\nCommand output:\n", cmd, err) + // Push the image to the Kind registry (no TLS) + pushCmd := exec.CommandContext(ctx, "podman", "push", "--tls-verify=false", imageRef) // #nosec G204 + if out, err := pushCmd.CombinedOutput(); err != nil { + fmt.Printf("[ERROR] Failed to push CLI image, %q returned an error: %v\nCommand output:\n", pushCmd, err) fmt.Print(string(out)) return err } + // Write cache hash only after a successful build + if cacheFile != "" { + _ = os.WriteFile(cacheFile, []byte(currentHash), 0644) // #nosec G306 + } + return nil } +// computeSourceHash computes a SHA-256 hash of all build inputs for the CLI +// image: Go source files, go.mod, go.sum, Dockerfile, build.sh, Makefile, and +// hack/reduce-snapshot.sh. Returns a hex-encoded digest string. +func computeSourceHash() (string, error) { + h := sha256.New() + + // Hash individual build files + buildFiles := []string{ + "go.mod", + "go.sum", + "Dockerfile", + "build.sh", + "Makefile", + "hack/reduce-snapshot.sh", + "tools/kubectl/go.mod", + "tools/kubectl/go.sum", + "acceptance/kubernetes/kind/acceptance.Dockerfile", + } + for _, f := range buildFiles { + if err := hashFile(h, f); err != nil { + if errors.Is(err, os.ErrNotExist) { + continue + } + return "", fmt.Errorf("hashing %s: %w", f, err) + } + } + + // Hash all .go source files + if err := filepath.WalkDir(".", func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + + // Skip vendor, .git, and acceptance test directories + if d.IsDir() && (d.Name() == "vendor" || d.Name() == ".git" || d.Name() == "acceptance") { + return filepath.SkipDir + } + + if !d.IsDir() && strings.HasSuffix(path, ".go") { + if err := hashFile(h, path); err != nil { + return err + } + } + + return nil + }); err != nil { + return "", fmt.Errorf("walking source tree: %w", err) + } + + return hex.EncodeToString(h.Sum(nil)), nil +} + +// hashFile adds the contents of a file to the given hash, prefixed by its path +// for domain separation. +func hashFile(h io.Writer, path string) error { + fmt.Fprintf(h, "file:%s\n", path) + + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + + _, err = io.Copy(h, f) + return err +} + // buildTaskBundleImage runs `make task-bundle` for each version of the Task in // the `$REPOSITORY_ROOT/task` directory to push the Tekton Task bundle to the // registry running on the Kind cluster. The image is pushed to image reference: @@ -117,7 +274,12 @@ func (k *kindCluster) buildTaskBundleImage(ctx context.Context) error { for i, step := range steps { if strings.Contains(step.Image, "/cli:") { steps[i].Image = img + steps[i].ImagePullPolicy = corev1.PullIfNotPresent } + // Strip resource requests to avoid scheduling waterfall in acceptance tests. + // Each TaskRun pod requests 2600m CPU, limiting concurrent pods on the Kind + // node and causing scheduling delays up to 5 minutes. + steps[i].ComputeResources = corev1.ResourceRequirements{} } out, err := yaml.Marshal(taskDefinition) @@ -139,17 +301,21 @@ func (k *kindCluster) buildTaskBundleImage(ctx context.Context) error { } } + g, gCtx := errgroup.WithContext(ctx) for version, tasks := range taskBundles { - tasksPath := strings.Join(tasks, ",") - cmd := exec.CommandContext(ctx, "make", "task-bundle", fmt.Sprintf("TASK_REPO=localhost:%d/ec-task-bundle", k.registryPort), fmt.Sprintf("TASKS=%s", tasksPath), fmt.Sprintf("TASK_TAG=%s", version)) /* #nosec */ - if out, err := cmd.CombinedOutput(); err != nil { - fmt.Printf("[ERROR] Unable to build and push the Task bundle image, %q returned an error: %v\nCommand output:\n", cmd, err) - fmt.Print(string(out)) - return err - } + g.Go(func() error { + tasksPath := strings.Join(tasks, ",") + cmd := exec.CommandContext(gCtx, "make", "task-bundle", fmt.Sprintf("TASK_REPO=localhost:%d/ec-task-bundle", k.registryPort), fmt.Sprintf("TASKS=%s", tasksPath), fmt.Sprintf("TASK_TAG=%s", version)) /* #nosec */ + if out, err := cmd.CombinedOutput(); err != nil { + fmt.Printf("[ERROR] Unable to build and push the Task bundle image, %q returned an error: %v\nCommand output:\n", cmd, err) + fmt.Print(string(out)) + return err + } + return nil + }) } - return nil + return g.Wait() } // builds a snapshot oci artifact for use with build trusted artifacts @@ -185,7 +351,8 @@ func (k *kindCluster) BuildSnapshotArtifact(ctx context.Context, content string) if t != nil { t.snapshotDigest = fileDescriptor.Digest.String() } - fmt.Printf("file descriptor for %s: %v\n", name, fileDescriptor) + logger, _ := log.LoggerFor(ctx) + logger.Logf("file descriptor for %s: %v", name, fileDescriptor) } artifactType := "application/vnd.test.artifact" @@ -196,7 +363,8 @@ func (k *kindCluster) BuildSnapshotArtifact(ctx context.Context, content string) if err != nil { return ctx, fmt.Errorf("failed creating manifestDescriptor: %w", err) } - fmt.Println("manifest descriptor:", manifestDescriptor) + logger, _ := log.LoggerFor(ctx) + logger.Log("manifest descriptor:", manifestDescriptor) tag := "latest" if err = fs.Tag(ctx, manifestDescriptor, tag); err != nil { @@ -208,7 +376,7 @@ func (k *kindCluster) BuildSnapshotArtifact(ctx context.Context, content string) if err != nil { return ctx, fmt.Errorf("failed to create repo: %w", err) } - fmt.Println("artifactRepo:", artifactRepo) + logger.Log("artifactRepo:", artifactRepo) // the registry is insecure repo.PlainHTTP = true @@ -217,7 +385,7 @@ func (k *kindCluster) BuildSnapshotArtifact(ctx context.Context, content string) if err != nil { return ctx, fmt.Errorf("failed to copy %s: %w", filePath, err) } - fmt.Println("snapshotDigest:", orasDesc.Digest) + logger.Log("snapshotDigest:", orasDesc.Digest) return ctx, nil } @@ -232,6 +400,24 @@ func getTag(ctx context.Context) (string, error) { return fmt.Sprintf("latest-%s", strings.Replace(strings.TrimSuffix(string(archOut), "\n"), "/", "-", -1)), nil } +// copyFile copies a file from src to dst, preserving the executable permission. +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + + out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755) // #nosec G302 + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, in) + return err +} + // Tar and gzip a file. Used with trusted artifacts. func tarGzipFile(source, target string) error { srcFile, err := os.Open(source) diff --git a/acceptance/kubernetes/kind/kind.go b/acceptance/kubernetes/kind/kind.go index abf104542..51326a61b 100644 --- a/acceptance/kubernetes/kind/kind.go +++ b/acceptance/kubernetes/kind/kind.go @@ -31,6 +31,7 @@ import ( "sync" "github.com/phayes/freeport" + "golang.org/x/sync/errgroup" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -238,21 +239,43 @@ func Start(givenCtx context.Context) (ctx context.Context, kCluster types.Cluste return } - err = applyConfiguration(ctx, &kCluster, yaml) + err = applyResources(ctx, &kCluster, yaml) if err != nil { logger.Errorf("Unable apply cluster configuration: %v", err) return } - err = kCluster.buildCliImage(ctx) - if err != nil { - logger.Errorf("Unable to build CLI image: %v", err) + // Set up ConfigMap RBAC early so all scenarios have it + // regardless of execution order + if err = kCluster.ensureConfigMapRBAC(ctx); err != nil { + logger.Errorf("Unable to create ConfigMap RBAC: %v", err) return } - err = kCluster.buildTaskBundleImage(ctx) + // Wait for the in-cluster registry (needed by image builds) + err = waitForAvailableDeploymentsIn(ctx, &kCluster, "image-registry") if err != nil { - logger.Errorf("Unable to build Task image: %v", err) + logger.Errorf("Unable to wait for image registry: %v", err) + return + } + + // Run image builds concurrently with Tekton deployment + g, gCtx := errgroup.WithContext(ctx) + + g.Go(func() error { + return kCluster.buildCliImage(gCtx) + }) + + g.Go(func() error { + return kCluster.buildTaskBundleImage(gCtx) + }) + + g.Go(func() error { + return waitForAvailableDeploymentsIn(gCtx, &kCluster, "tekton-pipelines") + }) + + if err = g.Wait(); err != nil { + logger.Errorf("Unable to complete cluster setup: %v", err) return } @@ -295,44 +318,64 @@ func renderTestConfiguration(k *kindCluster) (yaml []byte, err error) { return kustomize.Render(path.Join("test")) } -// applyConfiguration runs equivalent of kubectl apply for each document in the -// definitions YAML -func applyConfiguration(ctx context.Context, k *kindCluster, definitions []byte) (err error) { +// applyResources runs equivalent of kubectl apply for each document in the +// definitions YAML. Cluster-scoped resources (Namespaces, CRDs, ClusterRoles, +// etc.) are applied sequentially first, then namespaced resources are applied +// in parallel. +func applyResources(ctx context.Context, k *kindCluster, definitions []byte) error { + type resource struct { + obj unstructured.Unstructured + mapping *meta.RESTMapping + } + + // Parse all documents + var clusterScoped, namespaceScoped []resource reader := util.NewYAMLReader(bufio.NewReader(bytes.NewReader(definitions))) for { - var definition []byte - definition, err = reader.Read() + definition, err := reader.Read() if err != nil { if err == io.EOF { break } - return + return err } var obj unstructured.Unstructured - if err = yaml.Unmarshal(definition, &obj); err != nil { - return + if err := yaml.Unmarshal(definition, &obj); err != nil { + return err } - var mapping *meta.RESTMapping - if mapping, err = k.mapper.RESTMapping(obj.GroupVersionKind().GroupKind()); err != nil { - return + mapping, err := k.mapper.RESTMapping(obj.GroupVersionKind().GroupKind()) + if err != nil { + return err } - var c dynamic.ResourceInterface = k.dynamic.Resource(mapping.Resource) if mapping.Scope.Name() == meta.RESTScopeNameNamespace { - c = c.(dynamic.NamespaceableResourceInterface).Namespace(obj.GetNamespace()) + namespaceScoped = append(namespaceScoped, resource{obj: obj, mapping: mapping}) + } else { + clusterScoped = append(clusterScoped, resource{obj: obj, mapping: mapping}) } + } - _, err = c.Apply(ctx, obj.GetName(), &obj, metav1.ApplyOptions{FieldManager: "application/apply-patch"}) - if err != nil { - return + // Apply cluster-scoped resources sequentially (ordering matters for CRDs, Namespaces) + for _, r := range clusterScoped { + c := k.dynamic.Resource(r.mapping.Resource) + if _, err := c.Apply(ctx, r.obj.GetName(), &r.obj, metav1.ApplyOptions{FieldManager: "application/apply-patch"}); err != nil { + return err } } - err = waitForAvailableDeploymentsIn(ctx, k, "tekton-pipelines", "image-registry") + // Apply namespaced resources in parallel + g, gCtx := errgroup.WithContext(ctx) + for _, r := range namespaceScoped { + g.Go(func() error { + c := k.dynamic.Resource(r.mapping.Resource).Namespace(r.obj.GetNamespace()) + _, err := c.Apply(gCtx, r.obj.GetName(), &r.obj, metav1.ApplyOptions{FieldManager: "application/apply-patch"}) + return err + }) + } - return + return g.Wait() } // waitForAvailableDeploymentsIn makes sure that all deployments in the provided diff --git a/acceptance/kubernetes/kind/kubernetes.go b/acceptance/kubernetes/kind/kubernetes.go index 0535ad82b..2f86095e5 100644 --- a/acceptance/kubernetes/kind/kubernetes.go +++ b/acceptance/kubernetes/kind/kubernetes.go @@ -372,7 +372,7 @@ func (k *kindCluster) CreateNamespace(ctx context.Context) (context.Context, err return ctx, err } - return ctx, applyConfiguration(ctx, k, yaml) + return ctx, applyResources(ctx, k, yaml) } // stringParam generates a Tekton Parameter optionally expanding certain variables diff --git a/acceptance/log/log.go b/acceptance/log/log.go index e55c58738..55d4b7288 100644 --- a/acceptance/log/log.go +++ b/acceptance/log/log.go @@ -14,18 +14,17 @@ // // SPDX-License-Identifier: Apache-2.0 -// Package log forwards logs to testing.T.Log* methods +// Package log provides per-scenario file-based logging for acceptance tests package log import ( "context" "fmt" - "strings" + "os" + "sync" "sync/atomic" "sigs.k8s.io/kind/pkg/log" - - "github.com/conforma/cli/acceptance/testenv" ) type loggerKeyType int @@ -34,18 +33,22 @@ const loggerKey loggerKeyType = 0 var counter atomic.Uint32 +// DelegateLogger is the interface used internally to write log output type DelegateLogger interface { Log(args ...any) Logf(format string, args ...any) } +// Logger is the interface used by acceptance test packages for logging type Logger interface { DelegateLogger + Close() Enabled() bool Error(message string) Errorf(format string, args ...any) Info(message string) Infof(format string, args ...any) + LogFile() string Name(name string) Printf(format string, v ...any) V(level log.Level) log.InfoLogger @@ -53,107 +56,77 @@ type Logger interface { Warnf(format string, args ...any) } +// fileLogger writes log output to a file, one per scenario +type fileLogger struct { + mu sync.Mutex + file *os.File +} + +func (f *fileLogger) Log(args ...any) { + f.mu.Lock() + defer f.mu.Unlock() + fmt.Fprintln(f.file, args...) +} + +func (f *fileLogger) Logf(format string, args ...any) { + f.mu.Lock() + defer f.mu.Unlock() + fmt.Fprintf(f.file, format+"\n", args...) +} + +func (f *fileLogger) Close() { + f.mu.Lock() + defer f.mu.Unlock() + f.file.Close() +} + type logger struct { id uint32 name string t DelegateLogger -} - -// shouldSuppress checks if a log message should be suppressed -// Suppresses verbose container operation logs to reduce noise -func shouldSuppress(msg string) bool { - suppressPatterns := []string{ - "Creating container for image", - "Container created:", - "Starting container:", - "Container started:", - "Waiting for container id", - "Container is ready:", - "Skipping global cluster destruction", - "Released cluster to group", - "Destroying global cluster", - "Waiting for all consumers to finish", - "Last global cluster consumer finished", - } - - for _, pattern := range suppressPatterns { - if strings.Contains(msg, pattern) { - return true - } - } - return false + path string } // Log logs given arguments func (l logger) Log(args ...any) { msg := fmt.Sprint(args...) - if shouldSuppress(msg) { - return - } l.t.Logf("(%010d: %s) %s", l.id, l.name, msg) } // Logf logs using given format and specified arguments func (l logger) Logf(format string, args ...any) { msg := fmt.Sprintf(format, args...) - if shouldSuppress(msg) { - return - } l.t.Logf("(%010d: %s) %s", l.id, l.name, msg) } // Printf logs using given format and specified arguments func (l logger) Printf(format string, args ...any) { msg := fmt.Sprintf(format, args...) - if shouldSuppress(msg) { - return - } l.t.Logf("(%010d: %s) %s", l.id, l.name, msg) } func (l logger) Warn(message string) { - if shouldSuppress(message) { - return - } l.Logf("[WARN ] %s", message) } func (l logger) Warnf(format string, args ...any) { - msg := fmt.Sprintf(format, args...) - if shouldSuppress(msg) { - return - } - l.Logf("[WARN ] %s", msg) + l.Logf("[WARN ] %s", fmt.Sprintf(format, args...)) } func (l logger) Error(message string) { - if shouldSuppress(message) { - return - } l.Logf("[ERROR] %s", message) } func (l logger) Errorf(format string, args ...any) { - msg := fmt.Sprintf(format, args...) - if shouldSuppress(msg) { - return - } - l.Logf("[ERROR] %s", msg) + l.Logf("[ERROR] %s", fmt.Sprintf(format, args...)) } func (l logger) Info(message string) { - if shouldSuppress(message) { - return - } l.Logf("[INFO ] %s", message) } func (l logger) Infof(format string, args ...any) { - msg := fmt.Sprintf(format, args...) - if shouldSuppress(msg) { - return - } - l.Logf("[INFO ] %s", msg) + l.Logf("[INFO ] %s", fmt.Sprintf(format, args...)) } func (l logger) V(_ log.Level) log.InfoLogger { @@ -168,23 +141,39 @@ func (l *logger) Name(name string) { l.name = name } -// LoggerFor returns the logger for the provided Context, it is -// expected that a *testing.T instance is stored in the Context -// under the TestingKey key +// LogFile returns the path to the per-scenario log file +func (l *logger) LogFile() string { + return l.path +} + +// Close closes the underlying log file +func (l *logger) Close() { + if fl, ok := l.t.(*fileLogger); ok { + fl.Close() + } +} + +// LoggerFor returns the logger for the provided Context. Each call for +// a new context creates a per-scenario temp file for log isolation. func LoggerFor(ctx context.Context) (Logger, context.Context) { if logger, ok := ctx.Value(loggerKey).(Logger); ok { return logger, ctx } - delegate, ok := ctx.Value(testenv.TestingT).(DelegateLogger) - if !ok { - panic("No testing.T found in context") + id := counter.Add(1) + + f, err := os.CreateTemp("", fmt.Sprintf("scenario-%010d-*.log", id)) + if err != nil { + panic(fmt.Sprintf("failed to create scenario log file: %v", err)) } + delegate := &fileLogger{file: f} + logger := logger{ t: delegate, - id: counter.Add(1), + id: id, name: "*", + path: f.Name(), } return &logger, context.WithValue(ctx, loggerKey, &logger) diff --git a/acceptance/log/log_test.go b/acceptance/log/log_test.go index a7b1f23f0..b4645eb5a 100644 --- a/acceptance/log/log_test.go +++ b/acceptance/log/log_test.go @@ -16,55 +16,116 @@ //go:build unit -// Package log forwards logs to testing.T.Log* methods +// Package log provides per-scenario file-based logging for acceptance tests package log import ( "context" + "os" + "strings" "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - - "github.com/conforma/cli/acceptance/testenv" + "github.com/stretchr/testify/require" ) -type mockDelegateLogger struct { - mock.Mock +func TestLoggerWritesToFile(t *testing.T) { + ctx := context.Background() + + loggerA, _ := LoggerFor(ctx) + loggerA.Name("ScenarioA") + defer loggerA.Close() + defer os.Remove(loggerA.LogFile()) + + loggerA.Log("hello from A") + loggerA.Logf("formatted %s", "message") + loggerA.Info("info msg") + loggerA.Warn("warn msg") + loggerA.Error("error msg") + loggerA.Close() + + content, err := os.ReadFile(loggerA.LogFile()) + require.NoError(t, err) + + lines := string(content) + assert.Contains(t, lines, "hello from A") + assert.Contains(t, lines, "formatted message") + assert.Contains(t, lines, "[INFO ]") + assert.Contains(t, lines, "[WARN ]") + assert.Contains(t, lines, "[ERROR]") } -func (m *mockDelegateLogger) Log(args ...any) { - m.Called(args) +func TestLoggerCaching(t *testing.T) { + ctx := context.Background() + + loggerA, ctx := LoggerFor(ctx) + defer loggerA.Close() + defer os.Remove(loggerA.LogFile()) + + // Second call with same context returns the cached logger + loggerB, _ := LoggerFor(ctx) + + assert.Equal(t, loggerA, loggerB) } -func (m *mockDelegateLogger) Logf(format string, args ...any) { - m.Called(format, args) +func TestLoggerUniqueness(t *testing.T) { + ctxA := context.Background() + ctxB := context.Background() + + loggerA, _ := LoggerFor(ctxA) + defer loggerA.Close() + defer os.Remove(loggerA.LogFile()) + + loggerB, _ := LoggerFor(ctxB) + defer loggerB.Close() + defer os.Remove(loggerB.LogFile()) + + assert.NotEqual(t, loggerA.(*logger).id, loggerB.(*logger).id) + assert.NotEqual(t, loggerA.LogFile(), loggerB.LogFile()) } -func TestLogger(t *testing.T) { - dl := mockDelegateLogger{} - ctx := context.WithValue(context.Background(), testenv.TestingT, &dl) +func TestLoggerIsolation(t *testing.T) { + ctxA := context.Background() + ctxB := context.Background() - loggerA, ctx := LoggerFor(ctx) + loggerA, _ := LoggerFor(ctxA) loggerA.Name("A") + defer loggerA.Close() + defer os.Remove(loggerA.LogFile()) - assert.Equal(t, loggerA, ctx.Value(loggerKey)) + loggerB, _ := LoggerFor(ctxB) + loggerB.Name("B") + defer loggerB.Close() + defer os.Remove(loggerB.LogFile()) - dl.On("Logf", "(%010d: %s) %s", []any{uint32(1), "A", "hello"}) + loggerA.Log("only in A") + loggerB.Log("only in B") - loggerA.Logf("%s", "hello") + loggerA.Close() + loggerB.Close() - dl = mockDelegateLogger{} - ctx = context.WithValue(context.Background(), testenv.TestingT, &dl) + contentA, err := os.ReadFile(loggerA.LogFile()) + require.NoError(t, err) + contentB, err := os.ReadFile(loggerB.LogFile()) + require.NoError(t, err) - loggerB, ctx := LoggerFor(ctx) - loggerB.Name("B") + assert.Contains(t, string(contentA), "only in A") + assert.NotContains(t, string(contentA), "only in B") + assert.Contains(t, string(contentB), "only in B") + assert.NotContains(t, string(contentB), "only in A") +} - assert.Equal(t, loggerB, ctx.Value(loggerKey)) +func TestLogFileCreatesTemporaryFile(t *testing.T) { + ctx := context.Background() - dl.On("Logf", "(%010d: %s) %s", []any{uint32(2), "B", "hey"}) + l, _ := LoggerFor(ctx) + defer l.Close() + defer os.Remove(l.LogFile()) - loggerB.Log("hey") + path := l.LogFile() + assert.True(t, strings.Contains(path, "scenario-")) + assert.True(t, strings.HasSuffix(path, ".log")) - assert.NotEqual(t, loggerA.(*logger).id, loggerB.(*logger).id) + _, err := os.Stat(path) + assert.NoError(t, err) } diff --git a/acceptance/registry/registry.go b/acceptance/registry/registry.go index b8abdb9f2..3ff7dbeab 100644 --- a/acceptance/registry/registry.go +++ b/acceptance/registry/registry.go @@ -50,6 +50,7 @@ const registryStateKey = key(0) type registryState struct { HostAndPort string + Container testcontainers.Container `json:"-"` } func (g registryState) Key() any { @@ -117,6 +118,8 @@ func startStubRegistry(ctx context.Context) (context.Context, error) { return ctx, err } + state.Container = registry + port, err := registry.MappedPort(ctx, "5000/tcp") if err != nil { return ctx, err @@ -323,4 +326,24 @@ func Register(ctx context.Context, hostAndPort string) (context.Context, error) func AddStepsTo(sc *godog.ScenarioContext) { sc.Step(`^stub registry running$`, startStubRegistry) sc.Step(`^registry image "([^"]*)" should contain a layer with$`, assertImageContent) + + sc.After(func(ctx context.Context, finished *godog.Scenario, scenarioErr error) (context.Context, error) { + if testenv.Persisted(ctx) { + return ctx, nil + } + + if !testenv.HasState[registryState](ctx) { + return ctx, nil + } + + state := testenv.FetchState[registryState](ctx) + if state.Container != nil { + if err := state.Container.Terminate(ctx); err != nil { + logger, _ := log.LoggerFor(ctx) + logger.Warnf("failed to terminate registry container: %v", err) + } + } + + return ctx, nil + }) } diff --git a/acceptance/testenv/testenv.go b/acceptance/testenv/testenv.go index 6a500aed2..854898823 100644 --- a/acceptance/testenv/testenv.go +++ b/acceptance/testenv/testenv.go @@ -39,6 +39,7 @@ const ( PersistStubEnvironment testEnv = iota // key to a bool flag telling if the environment is persisted RestoreStubEnvironment // key to a bool flag telling if the environment is restored NoColors // key to a bool flag telling if the colors should be used in output + VerboseOutput // key to a bool flag telling if verbose output (stdout/stderr) should be shown on failure TestingT // key to the *testing.T instance in Context persistedEnv // key to a map of persisted environment states RekorImpl // key to a implementation of the Rekor interface, used to prevent import cycles diff --git a/acceptance/wiremock/wiremock.go b/acceptance/wiremock/wiremock.go index 08f7a94b2..292f5a071 100644 --- a/acceptance/wiremock/wiremock.go +++ b/acceptance/wiremock/wiremock.go @@ -85,7 +85,8 @@ type unmatchedRequest struct { } type wiremockState struct { - URL string + URL string + Container testcontainers.Container `json:"-"` } func (g wiremockState) Key() any { @@ -225,6 +226,8 @@ func StartWiremock(ctx context.Context) (context.Context, error) { return ctx, fmt.Errorf("unable to run GenericContainer: %v", err) } + state.Container = w + port, err := w.MappedPort(ctx, "8080/tcp") if err != nil { return ctx, err @@ -279,34 +282,37 @@ func IsRunning(ctx context.Context) bool { return state.Up() } -// AddStepsTo makes sure that nay unmatched requests, i.e. requests that are not -// stubbed get reported at the end of a scenario run -// TODO: reset stub state after the scenario (given not persisted flag is set) +// AddStepsTo makes sure that any unmatched requests, i.e. requests that are not +// stubbed get reported at the end of a scenario run, and terminates the container +// after the scenario completes func AddStepsTo(sc *godog.ScenarioContext) { sc.After(func(ctx context.Context, finished *godog.Scenario, scenarioErr error) (context.Context, error) { - if !IsRunning(ctx) { - return ctx, nil - } - - w, err := wiremockFrom(ctx) - if err != nil { - // wiremock wasn't launched, we don't need to proceed - return ctx, err + if IsRunning(ctx) { + if w, err := wiremockFrom(ctx); err == nil { + if unmatched, err := w.UnmatchedRequests(); err == nil && len(unmatched) > 0 { + logger, _ := log.LoggerFor(ctx) + logger.Log("Found unmatched WireMock requests:") + for i, u := range unmatched { + logger.Logf("[%d]: %s", i, u) + } + } + } } - unmatched, err := w.UnmatchedRequests() - if err != nil { - return ctx, err + if testenv.Persisted(ctx) { + return ctx, nil } - if len(unmatched) == 0 { + if !testenv.HasState[wiremockState](ctx) { return ctx, nil } - logger, ctx := log.LoggerFor(ctx) - logger.Log("Found unmatched WireMock requests:") - for i, u := range unmatched { - logger.Logf("[%d]: %s", i, u) + state := testenv.FetchState[wiremockState](ctx) + if state.Container != nil { + if err := state.Container.Terminate(ctx); err != nil { + logger, _ := log.LoggerFor(ctx) + logger.Warnf("failed to terminate wiremock container: %v", err) + } } return ctx, nil diff --git a/features/__snapshots__/validate_input.snap b/features/__snapshots__/validate_input.snap index 149c21ded..b7cecb89d 100755 --- a/features/__snapshots__/validate_input.snap +++ b/features/__snapshots__/validate_input.snap @@ -101,7 +101,7 @@ Error: success criteria not met --- [multiple data source top level key clash:stderr - 1] -Error: error validating file input.json: evaluating policy: load: load documents: 1 error occurred during loading: ${TEMP}/ec-work-${RANDOM}/dat${RANDOM}/${RANDOM}/data.yaml: merge error +Error: error validating file ${TMPDIR}/input.json: evaluating policy: load: load documents: 1 error occurred during loading: ${TEMP}/ec-work-${RANDOM}/dat${RANDOM}/${RANDOM}/data.yaml: merge error --- @@ -118,7 +118,7 @@ Error: error validating file pipeline_definition.yaml: evaluating policy: no reg ec-version: ${EC_VERSION} effective-time: "${TIMESTAMP}" filepaths: -- filepath: input.json +- filepath: ${TMPDIR}/input.json success: true success-count: 0 successes: null diff --git a/features/validate_input.feature b/features/validate_input.feature index 7dd0bce91..c50762fd5 100644 --- a/features/validate_input.feature +++ b/features/validate_input.feature @@ -119,7 +119,7 @@ Feature: validate input # In this situation a merge happens and we get second # level keys from both sources. Scenario: multiple data source top level key map merging - Given a file named "policy.yaml" containing + Given a file named "${TMPDIR}/policy.yaml" containing """ sources: - data: @@ -128,11 +128,11 @@ Feature: validate input policy: - "file::acceptance/examples/data-merges/policy" """ - Given a file named "input.json" containing + Given a file named "${TMPDIR}/input.json" containing """ {} """ - When ec command is run with "validate input --file input.json --policy policy.yaml -o yaml" + When ec command is run with "validate input --file ${TMPDIR}/input.json --policy ${TMPDIR}/policy.yaml -o yaml" Then the exit status should be 0 Then the output should match the snapshot @@ -140,7 +140,7 @@ Feature: validate input # two different data sources, but its value is not a map. # In this situation ec throws a "merge error" error. Scenario: multiple data source top level key clash - Given a file named "policy.yaml" containing + Given a file named "${TMPDIR}/policy.yaml" containing """ sources: - data: @@ -149,10 +149,10 @@ Feature: validate input policy: - "file::acceptance/examples/data-merges/policy" """ - Given a file named "input.json" containing + Given a file named "${TMPDIR}/input.json" containing """ {} """ - When ec command is run with "validate input --file input.json --policy policy.yaml -o yaml" + When ec command is run with "validate input --file ${TMPDIR}/input.json --policy ${TMPDIR}/policy.yaml -o yaml" Then the exit status should be 1 Then the output should match the snapshot diff --git a/hack/ubi-base-image-bump.sh b/hack/ubi-base-image-bump.sh index 5278b4cef..5585cb66b 100755 --- a/hack/ubi-base-image-bump.sh +++ b/hack/ubi-base-image-bump.sh @@ -30,7 +30,7 @@ NEW_DIGEST=$(skopeo inspect --raw docker://$UBI_MINIMAL | sha256sum | awk '{prin echo "Found $UBI_MINIMAL:latest@$NEW_DIGEST" # Update docker files -DOCKER_FILES=(Dockerfile Dockerfile.dist) +DOCKER_FILES=(Dockerfile Dockerfile.dist acceptance/kubernetes/kind/acceptance.Dockerfile) for d in "${DOCKER_FILES[@]}" ; do echo "Updating $d" sed -E "s!^FROM $UBI_MINIMAL@sha256:[0-9a-f]{64}\$!FROM $UBI_MINIMAL@sha256:$NEW_DIGEST!" -i $d