From 00995b04f1806ac83016cf3ebafc85ca56a31a0a Mon Sep 17 00:00:00 2001 From: Dejan K Date: Tue, 21 Oct 2025 15:26:50 +0200 Subject: [PATCH 1/2] docs(test-results): add AGENTS.md and DOCUMENTATION.md for LLM agents --- test-results/AGENTS.md | 24 ++++++++++++ test-results/DOCUMENTATION.md | 74 +++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 test-results/AGENTS.md create mode 100644 test-results/DOCUMENTATION.md diff --git a/test-results/AGENTS.md b/test-results/AGENTS.md new file mode 100644 index 00000000..1e5aad9e --- /dev/null +++ b/test-results/AGENTS.md @@ -0,0 +1,24 @@ +# Repository Guidelines + +## Project Structure & Module Organization +The CLI entrypoint is `main.go`, and subcommands sit in `cmd/` (for example `cmd/publish.go`). Shared logic belongs in `pkg/` packages such as `parsers`, `fileloader`, and `logger`; add new utilities here to keep commands thin. Reference docs live in `docs/`, helper scripts in `scripts/`, and runtime fixtures in `priv/`—update those assets whenever payload formats or examples shift. + +## Build, Test, and Development Commands +Use Go 1.24+ locally. Key tasks: +- `make run arg="publish --help"` runs the CLI with custom arguments. +- `make test` invokes `gotestsum` inside the `cli` container and writes `junit-report.xml`. +- `make lint` pipes `revive` and `staticcheck` output into JSON files for CI surfacing. +- `make regen.golden` refreshes parser golden data after intentional output changes. +Run `make test.setup` once to build the Docker image before containerised tasks. + +## Coding Style & Naming Conventions +Rely on `gofmt` defaults (tabs, short package names) and keep exported names purposeful. Honour the revive rules defined in `lint.toml`; any suppression needs reviewer sign-off. Commands should expose `cobra.Command` variables named after the verb (`publishCmd`, `compileCmd`) to match existing patterns. JSON payloads use lowerCamelCase tags—adjust fixtures in `priv/` whenever structures change. + +## Testing Guidelines +Place `_test.go` files beside implementations and favour table-driven tests, mirroring suites under `pkg/parsers`. Use `testify/require` for clear failures. `go test ./...` offers fast local feedback, while `make test.cover` produces `coverage.lcov` plus a Markdown summary via `scripts/lcov-to-md.sh`. For integration flows, follow the README examples that combine `test-results publish` with `--ignore-missing` when pipelines generate optional reports. + +## Commit & Pull Request Guidelines +Commits should be active-voice summaries with optional scopes and PR numbers, for example `feat(parser): add junit v2 adapter (#513)`. Keep related work within a single commit to simplify reviews. Pull requests need a short description, linked tickets, confirmation of the commands you ran, and artefact samples (screenshots, generated JSON) when behaviour changes. Call out breaking CLI changes explicitly so release notes stay accurate. + +## Security & Dependency Checks +Security scanning depends on the shared Semaphore toolbox. Run `make check.static` for static analysis and `make check.deps` for dependency audits; both populate `/tmp/monorepo`, so clear it if the cache becomes stale. If you must accept a finding, document the rationale and planned follow-up directly in the pull request. diff --git a/test-results/DOCUMENTATION.md b/test-results/DOCUMENTATION.md new file mode 100644 index 00000000..6d25c30d --- /dev/null +++ b/test-results/DOCUMENTATION.md @@ -0,0 +1,74 @@ +# Test Results CLI – Internal Notes + +## Purpose +- CLI used in Semaphore pipelines to parse raw test artifacts, normalize them into a common JSON schema, and push both detailed and summary reports to artifact storage. +- Supports multiple JUnit dialects and Go linters, providing consistent IDs and metadata enrichment for downstream dashboards. + +## Top-Level Layout +- `cmd/` – Cobra commands (`publish`, `compile`, `combine`, `gen-pipeline-report`, `command-metrics`, `resource-metrics`) that expose user-facing verbs and orchestrate work. +- `pkg/cli/` – shared execution helpers: flag handling, file discovery, parsing, JSON marshaling, gzip compression, artifact uploads, and stats reporting. +- `pkg/parser/` – core data model (`TestResults`, suites, cases), deterministic ID generation, JSON helpers, and XML utilities. +- `pkg/parsers/` – concrete parser implementations (multiple JUnit variants, Go revive/staticcheck, etc.) plus fixtures/goldens. +- `pkg/logger/` – thin logrus wrapper that enforces the `* message` format and centralizes log level control. +- `pkg/fileloader/` and `pkg/compression/` – memoized file readers and gzip utilities. +- `scripts/`, `Dockerfile.dev`, `docker-compose.yml` – development helpers; `docs/` contains design notes (e.g., ID generation); `priv/` holds sample payloads used in tests. + +## Command Flow +1. `main.go` boots `cmd.Execute()`, initializing Cobra/viper and persistent flags (`--parser`, `--name`, `--verbose`, etc.). +2. Each subcommand calls `cli.SetLogLevel` to honor verbosity/trace flags and then composes operations: + - Load candidate files (`cli.LoadFiles`) and resolve parser selection (`cli.FindParserForFile` or `--parser` override). + - Parse to `parser.Result`, decorate with metadata (`cli.DecorateResults`) and regenerate IDs if names/prefixes change. + - Marshal to JSON, optionally gzip (`cli.WriteToFile`, `cli.WriteToFilePath`), and publish via `cli.PushArtifacts`. + - `publish` merges reports, trims stdout/stderr, uploads job/workflow summaries, and pushes raw XML files when requested. + - `compile` stops after producing a local JSON artifact. + - `gen-pipeline-report` aggregates previously published job reports for the pipeline ID in `SEMAPHORE_PIPELINE_ID`. + - `combine` merges existing JSON summaries; metrics commands (`command-metrics`, `resource-metrics`) emit telemetry payloads. + +## Parser & ID Architecture +- `parser.Parser` interface defines `Parse`, `IsApplicable`, name/description metadata, and supported extensions. +- Concrete implementations under `pkg/parsers/` normalize framework-specific quirks: + - `junit_generic` handles baseline XML; specialized files shim mocha, rspec, golang, phpunit, exunit. + - Linters (revive, staticcheck) parse JSON streams into suites/cases. +- ID strategy (documented in `docs/id-generation.md`): + - UUIDv3 seeded by existing IDs, names, framework labels, and parent hierarchy to stay deterministic across runs. + - Suites namespace child IDs by parent result; failed tests include state to differentiate retries. +- Helpers trim output length, detect flaky states, and inject Semaphore metadata (job, workflow, repo identifiers) from environment variables. + +## Artifact & Storage Integration +- Upload layer targets Semaphore artifact storage via `cli.PushArtifacts`, grouping by `job` or `workflow`. +- CLI writes a merged `test-results/junit.json` plus per-input `junit-.xml` (unless `--no-raw`). +- Summaries (`summary.json`) include aggregated counts and upload statistics; stats accumulate via `cli.ArtifactStats`. +- Temporary files land under the OS temp dir; cleanup uses `defer os.Remove` and `os.RemoveAll`. + +## Configuration & Environment +- Viper auto-loads `$HOME/.test-results.yaml` when present; command flags override config. +- Key env vars: `SEMAPHORE_PIPELINE_ID`, `SEMAPHORE_WORKFLOW_ID`, `SEMAPHORE_JOB_ID`, `SEMAPHORE_AGENT_MACHINE_TYPE`, etc.; parsers read them to enrich metadata. +- Flags to know: `--parser`, `--ignore-missing`, `--no-compress`, `--suite-prefix`, `--omit-output-for-passed`, `--trim-output-to`, `--name`. + +## Development Workflow +- Requires Go 1.24+. Use `make test.setup` once to build the `cli` Docker image and fetch module deps. +- Everyday commands: + - `make run arg="publish --help"` – smoke-run CLI without building binary. + - `make test` – executes `gotestsum` inside Docker and exports `junit-report.xml`. + - `make lint` – runs revive and staticcheck, producing `revive.json` and `staticcheck.json`. + - `make regen.golden` – refresh parser golden fixtures after intentional format changes. + - `make test.cover` – generates `coverage.lcov` and Markdown summary via `scripts/lcov-to-md.sh`. +- Local iteration without Docker: `go test ./...` and `go run main.go ...` respect the same flags. + +## Testing & Fixtures +- Parser tests live beside implementations using table-driven cases and the shared helpers in `pkg/parsers/test_helpers.go`. +- Golden files live in `priv/` and `pkg/parsers/testdata` (embedded string literals); keep them in sync when tweaking parsers. +- Metrics commands have log-based fixtures under `priv/command-metrics` and `priv/resource-metrics`. +- Use `make regen.golden` plus `git diff` to validate intentional output deltas. + +## Utilities & Cross-Cutting Concerns +- Logging: `pkg/logger` wraps logrus; `cli.SetLogLevel` maps `--verbose`, `--trace` to logrus levels. +- Compression: `pkg/compression` toggles gzip; CLI defaults to compressed output for artifacts. +- File caching: `pkg/fileloader.Load` memoizes `bytes.Reader` instances to avoid rereading the same path during batch operations. +- Metrics: `cmd/command-metrics` and `cmd/resource-metrics` parse Semaphore job logs to emit structured telemetry; they rely on shared parsing in `pkg/cli`. + +## Tips for Future Changes +- When adding a parser, implement the `parser.Parser` interface, register it in `pkg/parsers/parsers.go`, and supply fixtures + golden tests. +- Any change to JSON schema or summary aggregation should update sample payloads in `priv/` and documented behaviour in `README.md`. +- Artifact uploads are sensitive to path conventions (`test-results/junit.json`, `test-results/.json`); double-check destinations when modifying. +- For CLI UX tweaks, adjust Cobra flags and keep `README.md` examples aligned; leverage `cmd/root.go` for persistent options. From 3e716c0651e4b55636a77b1dd7bc7e0750223bb3 Mon Sep 17 00:00:00 2001 From: Dejan K Date: Tue, 21 Oct 2025 15:36:03 +0200 Subject: [PATCH 2/2] feat(test-results): implement sanitizeDirective function and add tests for command metrics --- test-results/cmd/command-metrics.go | 46 +++++++- test-results/cmd/command-metrics_test.go | 132 +++++++++++++++++++++++ 2 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 test-results/cmd/command-metrics_test.go diff --git a/test-results/cmd/command-metrics.go b/test-results/cmd/command-metrics.go index f1d5a61b..a11b223b 100644 --- a/test-results/cmd/command-metrics.go +++ b/test-results/cmd/command-metrics.go @@ -6,10 +6,53 @@ import ( "os" "path/filepath" "strings" + "unicode" "github.com/spf13/cobra" ) +const maxDirectiveLabelLength = 80 + +func sanitizeDirective(raw string) string { + replacer := strings.NewReplacer( + "\r\n", " ", + "\n", " ", + "\r", " ", + "\t", " ", + "[", "(", + "]", ")", + "{", "(", + "}", ")", + ) + + clean := replacer.Replace(raw) + clean = strings.Map(func(r rune) rune { + if unicode.IsControl(r) { + return -1 + } + return r + }, clean) + + // Collapse repeated whitespace to a single space. + clean = strings.Join(strings.Fields(clean), " ") + clean = strings.TrimSpace(clean) + + if clean == "" { + return "(unnamed command)" + } + + runes := []rune(clean) + if len(runes) > maxDirectiveLabelLength { + truncationLimit := maxDirectiveLabelLength + if truncationLimit > 3 { + truncationLimit = truncationLimit - 3 + } + clean = string(runes[:truncationLimit]) + "..." + } + + return clean +} + var commandMetricsCmd = &cobra.Command{ Use: "command-metrics", Short: "Generates a command summary markdown report from agent metrics", @@ -60,7 +103,8 @@ var commandMetricsCmd = &cobra.Command{ if duration < 1 { duration = 1 } - out += fmt.Sprintf(" %s[%ds] :step%d, %d, %ds\n", node.Directive, duration, i, node.StartedAt, duration) + label := sanitizeDirective(node.Directive) + out += fmt.Sprintf(" %s[%ds] :step%d, %d, %ds\n", label, duration, i, node.StartedAt, duration) } out += "```\n" diff --git a/test-results/cmd/command-metrics_test.go b/test-results/cmd/command-metrics_test.go new file mode 100644 index 00000000..29d34b7a --- /dev/null +++ b/test-results/cmd/command-metrics_test.go @@ -0,0 +1,132 @@ +package cmd + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSanitizeDirective(t *testing.T) { + t.Parallel() + + longInput := strings.Repeat("a", 81) + longExpected := strings.Repeat("a", 77) + "..." + + testCases := []struct { + name string + input string + expected string + }{ + { + name: "trims whitespace and collapses", + input: "\t echo hello \nworld ", + expected: "echo hello world", + }, + { + name: "replaces brackets", + input: "if [ \"$VAR\" == 1 ] { echo ok }", + expected: "if ( \"$VAR\" == 1 ) ( echo ok )", + }, + { + name: "drops control characters", + input: "echo \x00hi", + expected: "echo hi", + }, + { + name: "truncates long labels", + input: longInput, + expected: longExpected, + }, + { + name: "provides fallback for empty values", + input: " ", + expected: "(unnamed command)", + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + require.Equal(t, tc.expected, sanitizeDirective(tc.input)) + }) + } +} + +func TestCommandMetricsGeneratesMermaidTimeline(t *testing.T) { + tempDir := t.TempDir() + + jobLogPath := filepath.Join(tempDir, "job_log_sample.json") + var entries []map[string]any + + entries = append(entries, map[string]any{ + "event": "cmd_finished", + "directive": "if [ \"$A\" == \"main\" ]; then\necho hi\nfi", + "started_at": int64(10), + "finished_at": int64(12), + }) + + entries = append(entries, map[string]any{ + "event": "cmd_finished", + "directive": "", + "started_at": int64(12), + "finished_at": int64(12), + }) + + entries = append(entries, map[string]any{ + "event": "cmd_finished", + "directive": strings.Repeat("b", 100), + "started_at": int64(20), + "finished_at": int64(25), + }) + + entries = append(entries, map[string]any{ + "event": "cmd_started", + "directive": "ignored", + "started_at": int64(30), + "finished_at": int64(40), + }) + + var builder strings.Builder + for _, entry := range entries { + payload, err := json.Marshal(entry) + require.NoError(t, err) + builder.Write(payload) + builder.WriteString("\n") + } + + err := os.WriteFile(jobLogPath, []byte(builder.String()), 0600) + require.NoError(t, err) + + outputPath := filepath.Join(tempDir, "report.md") + err = os.WriteFile(outputPath, []byte("# Existing content\n"), 0600) + require.NoError(t, err) + + srcFlag := commandMetricsCmd.Flags().Lookup("src") + require.NotNil(t, srcFlag) + + defer func() { + _ = commandMetricsCmd.Flags().Set("src", srcFlag.DefValue) + }() + + err = commandMetricsCmd.Flags().Set("src", jobLogPath) + require.NoError(t, err) + + err = commandMetricsCmd.RunE(commandMetricsCmd, []string{outputPath}) + require.NoError(t, err) + + content, err := os.ReadFile(outputPath) + require.NoError(t, err) + text := string(content) + + require.Contains(t, text, "# Existing content") + require.Contains(t, text, "## 🧭 Job Timeline") + + require.Contains(t, text, `if ( "$A" == "main" ); then echo hi fi[2s] :step0, 10, 2s`) + require.Contains(t, text, `(unnamed command)[1s] :step1, 12, 1s`) + require.Contains(t, text, strings.Repeat("b", 77)+`...[5s] :step2, 20, 5s`) +}