From 225ebdb80253a76245495b2d7a065fbb7372825c Mon Sep 17 00:00:00 2001 From: Christopher Date: Wed, 20 May 2026 15:13:20 +1000 Subject: [PATCH] docs(examples): add test vars templating example Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- examples/features/README.md | 2 + .../features/test-vars-templating/README.md | 16 ++++++++ .../evals/dataset.eval.yaml | 40 +++++++++++++++++++ .../evaluation/validation/eval-validator.ts | 1 + .../validation/eval-validator.test.ts | 23 +++++++++++ 5 files changed, 82 insertions(+) create mode 100644 examples/features/test-vars-templating/README.md create mode 100644 examples/features/test-vars-templating/evals/dataset.eval.yaml diff --git a/examples/features/README.md b/examples/features/README.md index 1f05d275..10a5f30d 100644 --- a/examples/features/README.md +++ b/examples/features/README.md @@ -73,6 +73,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the | [suite-level-input](suite-level-input/) | Prepend a shared system prompt to every test in the suite | | [suite-level-input-files](suite-level-input-files/) | Share file attachments across every test in the suite | | [env-interpolation](env-interpolation/) | Inject environment variables into eval config with `${{ VAR }}` | +| [test-vars-templating](test-vars-templating/) | Inject per-test `vars` into `{{name}}` templates in eval fields | --- @@ -169,6 +170,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the | [sdk-programmatic-api](sdk-programmatic-api/) | TypeScript SDK | | [suite-level-input](suite-level-input/) | Dataset & input | | [suite-level-input-files](suite-level-input-files/) | Dataset & input | +| [test-vars-templating](test-vars-templating/) | Dataset & input | | [threshold-grader](threshold-grader/) | LLM grading | | [tool-evaluation-plugins](tool-evaluation-plugins/) | Tool & agent evaluation | | [tool-trajectory-advanced](tool-trajectory-advanced/) | Tool & agent evaluation | diff --git a/examples/features/test-vars-templating/README.md b/examples/features/test-vars-templating/README.md new file mode 100644 index 00000000..44a29636 --- /dev/null +++ b/examples/features/test-vars-templating/README.md @@ -0,0 +1,16 @@ +# Per-Test Vars Templating + +Demonstrates `tests[].vars` with `{{name}}` placeholders in eval files. + +## Usage + +```bash +agentv eval examples/features/test-vars-templating/evals/dataset.eval.yaml +``` + +## Features + +- **Per-test data**: each test defines its own `vars` object +- **Template substitution**: `{{question}}` and dotted paths like `{{expected.answer}}` +- **Suite-level templates**: shared `input` can reference per-test vars too +- **Separate from env interpolation**: `{{question}}` uses test data, `${{ VAR }}` uses environment variables diff --git a/examples/features/test-vars-templating/evals/dataset.eval.yaml b/examples/features/test-vars-templating/evals/dataset.eval.yaml new file mode 100644 index 00000000..4c9885fc --- /dev/null +++ b/examples/features/test-vars-templating/evals/dataset.eval.yaml @@ -0,0 +1,40 @@ +# Per-test vars templating example +# +# tests[].vars provides per-test data for {{name}} placeholders in eval fields. +# Placeholders support dotted paths like {{expected.answer}}. +# +# Usage: +# agentv eval examples/features/test-vars-templating/evals/dataset.eval.yaml + +description: Demonstrates tests[].vars templating in eval fields + +execution: + target: llm + +input: + - role: system + content: "You are a concise assistant answering {{category}} questions." + +tests: + - id: capital-france + vars: + category: geography + question: What is the capital of France? + expected: + answer: Paris + criteria: "Answers {{question}} correctly" + input: "Question: {{question}}" + expected_output: "{{expected.answer}}" + + - id: greet-ada + vars: + category: etiquette + person: + name: Ada + expected: + answer: Hello, Ada! + criteria: "Greets {{person.name}} warmly" + input: + - role: user + content: "Say hello to {{person.name}}." + expected_output: "{{expected.answer}}" diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index d3b71ddf..78774fd0 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -69,6 +69,7 @@ const DEPRECATED_TOP_LEVEL_FIELDS = new Map([ /** Known fields at the test level. */ const KNOWN_TEST_FIELDS = new Set([ 'id', + 'vars', 'criteria', 'input', 'input_files', diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index 7992160b..9879d64e 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -145,6 +145,29 @@ describe('validateEvalFile', () => { expect(result.errors).toHaveLength(0); }); + it('accepts vars without unknown-field warnings', async () => { + const filePath = path.join(tempDir, 'test-vars.yaml'); + await writeFile( + filePath, + `tests: + - id: test-1 + vars: + question: "What is 2+2?" + expected: + answer: "4" + criteria: "Answers {{question}} correctly" + input: "Question: {{question}}" + expected_output: "{{expected.answer}}" +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(true); + const warnings = result.errors.filter((e) => e.severity === 'warning'); + expect(warnings).toHaveLength(0); + }); + describe('assert field validation', () => { it('validates assert array items have type field', async () => { const filePath = path.join(tempDir, 'assert-missing-type.yaml');