diff --git a/CHANGELOG.md b/CHANGELOG.md index f4d70de..07d417e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,89 @@ All notable changes to selectools will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.17.0] - 2026-03-22 + +### Added + +**Built-in Eval Framework** — the only AI agent framework with a comprehensive evaluation suite built in. No separate install, no SaaS account, no external dependencies. + +#### Evaluators (39 total) + +**21 deterministic evaluators** (no API calls): +- `ToolUseEvaluator` — tool name, tool list, argument value assertions +- `ContainsEvaluator` — substring present/absent (case-insensitive) +- `OutputEvaluator` — exact match and regex matching +- `StructuredOutputEvaluator` — parsed field assertions (deep subset match) +- `PerformanceEvaluator` — iteration count, latency, and cost thresholds +- `JsonValidityEvaluator` — valid JSON output +- `LengthEvaluator` — min/max character count +- `WordCountEvaluator` — min/max word count +- `StartsWithEvaluator` / `EndsWithEvaluator` — prefix/suffix assertions +- `ToolOrderEvaluator` — tools called in expected sequence +- `UniqueToolsEvaluator` — no duplicate tool calls +- `PIILeakEvaluator` — SSN, email, phone, credit card, ZIP detection +- `InjectionResistanceEvaluator` — 10 prompt injection patterns +- `RefusalEvaluator` — detect appropriate refusal of harmful requests +- `SentimentEvaluator` — keyword-based positive/negative/neutral detection +- `PythonValidityEvaluator` — valid Python syntax (with code fence stripping) +- `SQLValidityEvaluator` — SQL statement validation +- `URLValidityEvaluator` — well-formed URL detection +- `MarkdownFormatEvaluator` — markdown formatting detection +- `CustomEvaluator` — any user-defined callable + +**18 LLM-as-judge evaluators** (use any Provider): +- `LLMJudgeEvaluator` — generic rubric scoring (0-10) +- `CorrectnessEvaluator` — correct vs reference answer +- `RelevanceEvaluator` — response relevant to query +- `FaithfulnessEvaluator` — grounded in provided context (RAG) +- `HallucinationEvaluator` — fabricated information detection +- `ToxicityEvaluator` — harmful/inappropriate content +- `CoherenceEvaluator` — well-structured and logical +- `CompletenessEvaluator` — fully addresses the query +- `BiasEvaluator` — gender, racial, political bias +- `SummaryEvaluator` — summary accuracy and coverage +- `ConcisenessEvaluator` — not overly verbose +- `InstructionFollowingEvaluator` — followed specific instructions +- `ToneEvaluator` — matches expected tone +- `ContextRecallEvaluator` — RAG: used all relevant context +- `ContextPrecisionEvaluator` — RAG: retrieved context was relevant +- `GrammarEvaluator` — grammatically correct and fluent +- `SafetyEvaluator` — comprehensive safety check + +#### Infrastructure + +- `EvalSuite` — orchestrates eval runs with sync/async/concurrent execution +- `EvalReport` — accuracy, latency p50/p95/p99, cost, weighted scoring, tag filtering, failure breakdown +- `DatasetLoader` — load test cases from JSON/YAML files +- `BaselineStore` + `RegressionResult` — save baselines, detect regressions across runs +- `PairwiseEval` — compare two agents head-to-head with automatic winner determination +- `SnapshotStore` — Jest-style snapshot testing for AI agent outputs +- `generate_cases()` — LLM-powered synthetic test case generator from tool definitions +- `generate_badge()` — shields.io-style SVG badges for README +- `serve_eval()` — live browser dashboard with real-time eval progress +- `HistoryStore` — track accuracy/cost/latency across runs with trend analysis +- Interactive HTML report with donut chart, latency histogram, trend sparkline, expandable rows, filtering +- JUnit XML for CI (GitHub Actions, Jenkins, GitLab CI) +- `report.to_markdown()` — markdown summary for GitHub issues and PRs +- CLI: `python -m selectools.evals run/compare` +- GitHub Action at `.github/actions/eval/` with automatic PR comments +- Cost estimation: `suite.estimate_cost()` before running +- 4 pre-built templates: `customer_support_suite()`, `rag_quality_suite()`, `safety_suite()`, `code_quality_suite()` +- `pip install selectools[evals]` for optional PyYAML dependency + +#### Observer Integration + +- 3 new observer events: `on_eval_start`, `on_eval_case_end`, `on_eval_end` +- Compatible with `LoggingObserver` for structured JSON eval logs + +#### Testing + +- **309 new eval tests** across 6 test files (unit, integration, E2E) +- 40 example scripts (2 eval-specific: `39_eval_framework.py`, `40_eval_advanced.py`) +- Full module documentation: `docs/modules/EVALS.md` + +--- + ## [0.16.7] - 2026-03-16 ### Removed diff --git a/README.md b/README.md index 45ba832..b62643e 100644 --- a/README.md +++ b/README.md @@ -4,17 +4,40 @@ [![Documentation](https://img.shields.io/badge/docs-GitHub%20Pages-blue)](https://johnnichev.github.io/selectools) [![License: LGPL v3](https://img.shields.io/badge/License-LGPL_v3-blue.svg)](https://www.gnu.org/licenses/lgpl-3.0) [![Python 3.13+](https://img.shields.io/badge/python-3.13+-blue.svg)](https://www.python.org/downloads/) +[![Evaluators](https://img.shields.io/badge/evaluators-39-06b6d4.svg)](https://johnnichev.github.io/selectools/modules/EVALS/) An open-source project from **[NichevLabs](https://nichevlabs.com)**. **Production-ready AI agents with tool calling, RAG, and hybrid search.** Connect LLMs to your Python functions, embed and search your documents with vector + keyword fusion, stream responses in real time, and dynamically manage tools at runtime. Works with OpenAI, Anthropic, Gemini, and Ollama. Tracks costs automatically. -## What's New in v0.16.7 +## What's New in v0.17.0 -**Cleanup release** — Removed unused CLI module, completed README example table (28-38), fixed stale doc counts. +**Built-in Eval Framework** — 39 evaluators, A/B testing, regression detection, and more. No separate install needed. -- **CLI removed** — `selectools` console script entry point removed (unused, flagged by package safety scanners) -- **1758 tests** across unit, integration, regression, and E2E +```python +from selectools.evals import EvalSuite, TestCase + +suite = EvalSuite(agent=agent, cases=[ + TestCase(input="Cancel account", expect_tool="cancel_sub", expect_no_pii=True), + TestCase(input="Balance?", expect_contains="balance", expect_latency_ms_lte=500), +]) +report = suite.run() +print(report.accuracy) # 0.95 +print(report.latency_p50) # 142ms +report.to_html("report.html") +``` + +- **39 Evaluators** — 21 deterministic + 18 LLM-as-judge (tool use, correctness, safety, RAG, code, format) +- **A/B Testing** — `PairwiseEval` compares two agents head-to-head +- **Regression Detection** — `BaselineStore` tracks accuracy across runs +- **Snapshot Testing** — Jest-style output snapshots for AI agents +- **Pre-built Templates** — `customer_support_suite()`, `safety_suite()`, `rag_quality_suite()`, `code_quality_suite()` +- **Interactive HTML Report** — donut chart, histogram, trend line, expandable rows, filtering +- **GitHub Action** — automatic PR comments with eval results +- **CLI** — `python -m selectools.evals run cases.json --html report.html` +- **Cost Estimation** — `suite.estimate_cost()` before running +- **History Tracking** — `HistoryStore` with trend analysis +- **309 eval tests**, zero external dependencies > Full changelog: [CHANGELOG.md](https://github.com/johnnichev/selectools/blob/main/CHANGELOG.md) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f4d70de..07d417e 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -5,6 +5,89 @@ All notable changes to selectools will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.17.0] - 2026-03-22 + +### Added + +**Built-in Eval Framework** — the only AI agent framework with a comprehensive evaluation suite built in. No separate install, no SaaS account, no external dependencies. + +#### Evaluators (39 total) + +**21 deterministic evaluators** (no API calls): +- `ToolUseEvaluator` — tool name, tool list, argument value assertions +- `ContainsEvaluator` — substring present/absent (case-insensitive) +- `OutputEvaluator` — exact match and regex matching +- `StructuredOutputEvaluator` — parsed field assertions (deep subset match) +- `PerformanceEvaluator` — iteration count, latency, and cost thresholds +- `JsonValidityEvaluator` — valid JSON output +- `LengthEvaluator` — min/max character count +- `WordCountEvaluator` — min/max word count +- `StartsWithEvaluator` / `EndsWithEvaluator` — prefix/suffix assertions +- `ToolOrderEvaluator` — tools called in expected sequence +- `UniqueToolsEvaluator` — no duplicate tool calls +- `PIILeakEvaluator` — SSN, email, phone, credit card, ZIP detection +- `InjectionResistanceEvaluator` — 10 prompt injection patterns +- `RefusalEvaluator` — detect appropriate refusal of harmful requests +- `SentimentEvaluator` — keyword-based positive/negative/neutral detection +- `PythonValidityEvaluator` — valid Python syntax (with code fence stripping) +- `SQLValidityEvaluator` — SQL statement validation +- `URLValidityEvaluator` — well-formed URL detection +- `MarkdownFormatEvaluator` — markdown formatting detection +- `CustomEvaluator` — any user-defined callable + +**18 LLM-as-judge evaluators** (use any Provider): +- `LLMJudgeEvaluator` — generic rubric scoring (0-10) +- `CorrectnessEvaluator` — correct vs reference answer +- `RelevanceEvaluator` — response relevant to query +- `FaithfulnessEvaluator` — grounded in provided context (RAG) +- `HallucinationEvaluator` — fabricated information detection +- `ToxicityEvaluator` — harmful/inappropriate content +- `CoherenceEvaluator` — well-structured and logical +- `CompletenessEvaluator` — fully addresses the query +- `BiasEvaluator` — gender, racial, political bias +- `SummaryEvaluator` — summary accuracy and coverage +- `ConcisenessEvaluator` — not overly verbose +- `InstructionFollowingEvaluator` — followed specific instructions +- `ToneEvaluator` — matches expected tone +- `ContextRecallEvaluator` — RAG: used all relevant context +- `ContextPrecisionEvaluator` — RAG: retrieved context was relevant +- `GrammarEvaluator` — grammatically correct and fluent +- `SafetyEvaluator` — comprehensive safety check + +#### Infrastructure + +- `EvalSuite` — orchestrates eval runs with sync/async/concurrent execution +- `EvalReport` — accuracy, latency p50/p95/p99, cost, weighted scoring, tag filtering, failure breakdown +- `DatasetLoader` — load test cases from JSON/YAML files +- `BaselineStore` + `RegressionResult` — save baselines, detect regressions across runs +- `PairwiseEval` — compare two agents head-to-head with automatic winner determination +- `SnapshotStore` — Jest-style snapshot testing for AI agent outputs +- `generate_cases()` — LLM-powered synthetic test case generator from tool definitions +- `generate_badge()` — shields.io-style SVG badges for README +- `serve_eval()` — live browser dashboard with real-time eval progress +- `HistoryStore` — track accuracy/cost/latency across runs with trend analysis +- Interactive HTML report with donut chart, latency histogram, trend sparkline, expandable rows, filtering +- JUnit XML for CI (GitHub Actions, Jenkins, GitLab CI) +- `report.to_markdown()` — markdown summary for GitHub issues and PRs +- CLI: `python -m selectools.evals run/compare` +- GitHub Action at `.github/actions/eval/` with automatic PR comments +- Cost estimation: `suite.estimate_cost()` before running +- 4 pre-built templates: `customer_support_suite()`, `rag_quality_suite()`, `safety_suite()`, `code_quality_suite()` +- `pip install selectools[evals]` for optional PyYAML dependency + +#### Observer Integration + +- 3 new observer events: `on_eval_start`, `on_eval_case_end`, `on_eval_end` +- Compatible with `LoggingObserver` for structured JSON eval logs + +#### Testing + +- **309 new eval tests** across 6 test files (unit, integration, E2E) +- 40 example scripts (2 eval-specific: `39_eval_framework.py`, `40_eval_advanced.py`) +- Full module documentation: `docs/modules/EVALS.md` + +--- + ## [0.16.7] - 2026-03-16 ### Removed diff --git a/landing/index.html b/landing/index.html index 4fbbfaf..b529aec 100644 --- a/landing/index.html +++ b/landing/index.html @@ -88,7 +88,7 @@

Gemini Ollama 146 Models - 22 Evaluators + 39 Evaluators 1758 Tests @@ -250,7 +250,7 @@

22 Eval Evaluators

Built-in Agent Evaluation

-

The only agent framework with a built-in eval suite. No separate install, no SaaS account, no external dependencies. 22 evaluators out of the box.

+

The only agent framework with a built-in eval suite. No separate install, no SaaS account, no external dependencies. 39 evaluators out of the box.

@@ -287,32 +287,44 @@

Built-in Agent Evaluation

-

12 Deterministic Evaluators

+

21 Deterministic Evaluators

ToolUse + ToolOrder Contains Output Structured Performance JSON + Python + SQL + URLs + Markdown Length - StartsWith - EndsWith - PII Leak + Words + PII Injection + Refusal + Sentiment Custom
-

10 LLM-as-Judge Evaluators

+

18 LLM-as-Judge Evaluators

Correctness Relevance Faithfulness Hallucination Toxicity + Safety Coherence Completeness + Conciseness + Grammar + Tone + ContextRecall + ContextPrecision Bias Summary Custom Rubric @@ -333,65 +345,150 @@

Infrastructure

- +
-
-

Selectools vs. LangChain

-

An honest comparison. Choose LangChain for ecosystem breadth. Choose Selectools when compliance and observability aren't optional.

-
- +
+

How We Compare

+

Choose LangChain for ecosystem breadth. Choose Selectools when compliance, observability, and evaluation aren't optional.

+ +

Agent Framework Comparison

+
+
- - - + + + + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + + + + + + + + + + +
CapabilitySelectoolsLangChainCapabilitySelectoolsLangChainCrewAI
Execution tracesBuilt-inLangSmith (paid)Execution tracesBuilt-inLangSmith (paid)Limited logging
GuardrailsBuilt-in (5 types)NeMo (separate)Not built-in
Audit loggingBuilt-in (4 privacy levels)DIYNot built-in
Agent evaluationBuilt-in (39 evaluators)LangSmith (paid)Not built-in
Cost trackingAutomatic per-callManualNot built-in
Injection defense15 patterns + coherenceNot includedNot included
Setup1 package5+ packages1 package
GuardrailsBuilt-in (5 types)NeMo (separate project)Multi-agentComing v0.17.1LangGraphCore feature
CommunityGrowingMassiveLarge
+
+ +

Eval Framework Comparison

+
+ + + + + + + + + + + + + + + + + - - - + + + + + - - - + + + + + - - - + + + + + - - - + + + + + - - - + + + + + - - - + + + + + - - - + + + + +
CapabilitySelectoolsDeepEvalPromptfooLangSmith
InstallBuilt-inSeparate pipNode.js CLIpip + account
Audit loggingBuilt-in (4 privacy levels)Build it yourselfEvaluators3950+~20~8
Cost trackingAutomatic per-callManual / LangSmithA/B testingPairwiseEvalNoSide-by-sideExperiments
Injection defense15 patterns + coherenceNot includedSnapshot testingSnapshotStoreNoNoNo
Setup1 package5+ packagesRegression detectionLocal (JSON)Cloud onlyCLI + GitHubSaaS only
Reasoning visibilityresult.reasoningNot availableHTML reportInteractive (charts)Cloud onlySelf-containedSaaS UI
Agent evaluationBuilt-in (22 evaluators)LangSmith (paid) or DeepEval (separate)Works offlineYesPartialYesNo
CommunityGrowingMassivePriceFreeFree + SaaSFree$39/seat/mo
diff --git a/notebooks/getting_started.ipynb b/notebooks/getting_started.ipynb index 911e3a4..371918f 100644 --- a/notebooks/getting_started.ipynb +++ b/notebooks/getting_started.ipynb @@ -1033,10 +1033,103 @@ "print(f\"Result: {result.content[:60]}\")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 19: Built-in Eval Framework (v0.17.0)\n", + "\n", + "Selectools ships with a complete eval framework \u2014 39 evaluators, A/B testing, regression detection, and more. No separate install needed.\n", + "\n", + "### Basic Evaluation\n", + "\n", + "Define test cases with declarative assertions and run them against your agent:" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from selectools.evals import EvalSuite, TestCase\n", + "\n", + "# Define test cases with assertions\n", + "suite = EvalSuite(\n", + " agent=agent,\n", + " cases=[\n", + " TestCase(input=\"What costs more, laptop or phone?\", name=\"price_compare\",\n", + " expect_tool=\"get_price\", expect_contains=\"price\"),\n", + " TestCase(input=\"Is the laptop in stock?\", name=\"stock_check\",\n", + " expect_tool=\"check_stock\"),\n", + " TestCase(input=\"Tell me about headphones\", name=\"general\",\n", + " expect_no_pii=True, expect_min_words=3),\n", + " ],\n", + ")\n", + "\n", + "report = suite.run()\n", + "print(report.summary())\n", + "print()\n", + "print(f\"Accuracy: {report.accuracy:.1%}\")\n", + "print(f\"Latency p50: {report.latency_p50:.0f}ms\")\n", + "print(f\"Total cost: ${report.total_cost:.6f}\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Pre-built Templates\n", + "\n", + "Use one-line templates for common evaluation scenarios:" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from selectools.evals import customer_support_suite, safety_suite\n", + "\n", + "# Customer support eval \u2014 6 default test cases\n", + "cs_report = customer_support_suite(agent).run()\n", + "print(f\"Customer Support: {cs_report.accuracy:.1%} accuracy\")\n", + "\n", + "# Safety eval \u2014 6 default test cases\n", + "safety_report = safety_suite(agent).run()\n", + "print(f\"Safety: {safety_report.accuracy:.1%} accuracy\")" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Export & CI\n", + "\n", + "Export to HTML, JUnit XML, JSON, or Markdown:" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Markdown summary (paste into GitHub issues or PRs)\n", + "print(report.to_markdown())\n", + "\n", + "# HTML report (open in browser)\n", + "# report.to_html(\"/tmp/eval-report.html\")\n", + "\n", + "# JUnit XML (for CI pipelines)\n", + "# report.to_junit_xml(\"/tmp/eval-results.xml\")" + ], + "outputs": [], + "execution_count": null + }, { "cell_type": "markdown", "source": [ - "## What's Next?\n\nYou've seen the full API surface! Here's where to go from here:\n\n| Goal | Resource |\n|---|---|\n| 38 numbered examples (01-38) | [`examples/`](../examples/) |\n| Detailed quickstart guide | [`docs/QUICKSTART.md`](../docs/QUICKSTART.md) |\n| Architecture deep-dive | [`docs/ARCHITECTURE.md`](../docs/ARCHITECTURE.md) |\n| Agent reference (traces, batch, policy, observer) | [`docs/modules/AGENT.md`](../docs/modules/AGENT.md) |\n| Terminal tools and async observers | [`docs/modules/AGENT.md`](../docs/modules/AGENT.md) |\n| Guardrails (PII, topic, toxicity, format) | [`docs/modules/GUARDRAILS.md`](../docs/modules/GUARDRAILS.md) |\n| Audit logging (JSONL, privacy controls) | [`docs/modules/AUDIT.md`](../docs/modules/AUDIT.md) |\n| Security (screening, coherence checking) | [`docs/modules/SECURITY.md`](../docs/modules/SECURITY.md) |\n| Persistent sessions (3 backends) | [`docs/modules/SESSIONS.md`](../docs/modules/SESSIONS.md) |\n| Entity memory (extraction, tracking) | [`docs/modules/ENTITY_MEMORY.md`](../docs/modules/ENTITY_MEMORY.md) |\n| Knowledge graph (triples, querying) | [`docs/modules/KNOWLEDGE_GRAPH.md`](../docs/modules/KNOWLEDGE_GRAPH.md) |\n| Cross-session knowledge (durable memory) | [`docs/modules/KNOWLEDGE.md`](../docs/modules/KNOWLEDGE.md) |\n| Provider reference (fallback, max_tokens) | [`docs/modules/PROVIDERS.md`](../docs/modules/PROVIDERS.md) |\n| Model registry (146 models, pricing) | [`docs/modules/MODELS.md`](../docs/modules/MODELS.md) |\n| Tool definition reference | [`docs/modules/TOOLS.md`](../docs/modules/TOOLS.md) |\n| 24 pre-built tools (file, web, data, text, datetime) | [`docs/modules/TOOLBOX.md`](../docs/modules/TOOLBOX.md) |\n| Error handling & exceptions | [`docs/modules/EXCEPTIONS.md`](../docs/modules/EXCEPTIONS.md) |\n| Streaming & parallel execution | [`docs/modules/STREAMING.md`](../docs/modules/STREAMING.md) |\n| Hybrid search & reranking | [`docs/modules/HYBRID_SEARCH.md`](../docs/modules/HYBRID_SEARCH.md) |\n| Full documentation index | [`docs/README.md`](../docs/README.md) |" + "## What's Next?\n\nYou've seen the full API surface! Here's where to go from here:\n\n| Goal | Resource |\n|---|---|\n| 40 numbered examples (01-40) | [`examples/`](../examples/) |\n| Eval framework (39 evaluators) | [`docs/modules/EVALS.md`](../docs/modules/EVALS.md) |\n| Detailed quickstart guide | [`docs/QUICKSTART.md`](../docs/QUICKSTART.md) |\n| Architecture deep-dive | [`docs/ARCHITECTURE.md`](../docs/ARCHITECTURE.md) |\n| Agent reference (traces, batch, policy, observer) | [`docs/modules/AGENT.md`](../docs/modules/AGENT.md) |\n| Terminal tools and async observers | [`docs/modules/AGENT.md`](../docs/modules/AGENT.md) |\n| Guardrails (PII, topic, toxicity, format) | [`docs/modules/GUARDRAILS.md`](../docs/modules/GUARDRAILS.md) |\n| Audit logging (JSONL, privacy controls) | [`docs/modules/AUDIT.md`](../docs/modules/AUDIT.md) |\n| Security (screening, coherence checking) | [`docs/modules/SECURITY.md`](../docs/modules/SECURITY.md) |\n| Persistent sessions (3 backends) | [`docs/modules/SESSIONS.md`](../docs/modules/SESSIONS.md) |\n| Entity memory (extraction, tracking) | [`docs/modules/ENTITY_MEMORY.md`](../docs/modules/ENTITY_MEMORY.md) |\n| Knowledge graph (triples, querying) | [`docs/modules/KNOWLEDGE_GRAPH.md`](../docs/modules/KNOWLEDGE_GRAPH.md) |\n| Cross-session knowledge (durable memory) | [`docs/modules/KNOWLEDGE.md`](../docs/modules/KNOWLEDGE.md) |\n| Provider reference (fallback, max_tokens) | [`docs/modules/PROVIDERS.md`](../docs/modules/PROVIDERS.md) |\n| Model registry (146 models, pricing) | [`docs/modules/MODELS.md`](../docs/modules/MODELS.md) |\n| Tool definition reference | [`docs/modules/TOOLS.md`](../docs/modules/TOOLS.md) |\n| 24 pre-built tools (file, web, data, text, datetime) | [`docs/modules/TOOLBOX.md`](../docs/modules/TOOLBOX.md) |\n| Error handling & exceptions | [`docs/modules/EXCEPTIONS.md`](../docs/modules/EXCEPTIONS.md) |\n| Streaming & parallel execution | [`docs/modules/STREAMING.md`](../docs/modules/STREAMING.md) |\n| Hybrid search & reranking | [`docs/modules/HYBRID_SEARCH.md`](../docs/modules/HYBRID_SEARCH.md) |\n| Full documentation index | [`docs/README.md`](../docs/README.md) |" ], "metadata": {} } diff --git a/pyproject.toml b/pyproject.toml index 6ae00dc..dcda772 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,6 +59,9 @@ rag = [ "cohere>=5.0.0", "pypdf>=4.0.0", ] +evals = [ + "pyyaml>=6.0.0", +] [tool.setuptools] package-dir = {"" = "src"} diff --git a/src/selectools/evals/html.py b/src/selectools/evals/html.py index 04eeaa7..ae6f783 100644 --- a/src/selectools/evals/html.py +++ b/src/selectools/evals/html.py @@ -5,7 +5,7 @@ import html import math from pathlib import Path -from typing import Any, List, Union +from typing import Any, List, Optional, Union from .types import CaseVerdict @@ -88,8 +88,53 @@ def _histogram_svg(latencies: List[float]) -> str: ) -def render_html_report(report: Any, filepath: Union[str, Path]) -> None: # noqa: C901 - """Render an EvalReport as a self-contained interactive HTML file.""" +def _trend_svg(accuracies: List[float]) -> str: + """Generate an SVG sparkline for accuracy trend over time.""" + if len(accuracies) < 2: + return "" + w, h = 200, 60 + n = len(accuracies) + max_v = max(accuracies) if max(accuracies) > 0 else 1.0 + min_v = min(accuracies) + v_range = max_v - min_v if max_v != min_v else 0.1 + + points: List[str] = [] + for i, v in enumerate(accuracies): + x = i * (w - 20) / (n - 1) + 10 + y = h - 10 - ((v - min_v) / v_range) * (h - 25) + points.append(f"{x:.1f},{y:.1f}") + + polyline = " ".join(points) + # Color based on trend + color = "#4ade80" if accuracies[-1] >= accuracies[0] else "#f87171" + + dots = "".join( + f'' + for p in points + ) + + return ( + f'' + f'' + f"{dots}" + f'Accuracy Trend' + ) + + +def render_html_report( # noqa: C901 + report: Any, + filepath: Union[str, Path], + history: Optional[Any] = None, +) -> None: + """Render an EvalReport as a self-contained interactive HTML file. + + Args: + report: EvalReport instance. + filepath: Path to write the HTML file. + history: Optional HistoryTrend instance for trend chart. + """ # Build table rows with expandable details rows = [] for i, cr in enumerate(report.case_results): @@ -158,6 +203,9 @@ def render_html_report(report: Any, filepath: Union[str, Path]) -> None: # noqa donut = _donut_svg(report.pass_count, report.fail_count, report.error_count, report.skip_count) latencies = [cr.latency_ms for cr in report.case_results if cr.verdict != CaseVerdict.SKIP] histogram = _histogram_svg(latencies) + trend_chart = "" + if history and hasattr(history, "accuracy_trend") and len(history.accuracy_trend) >= 2: + trend_chart = _trend_svg(history.accuracy_trend) # Failure breakdown failures_by_eval = report.failures_by_evaluator() @@ -277,6 +325,7 @@ def render_html_report(report: Any, filepath: Union[str, Path]) -> None: # noqa
{donut}
{histogram}
+ {f'
{trend_chart}
' if trend_chart else ''}
Pass ({report.pass_count}) diff --git a/src/selectools/evals/report.py b/src/selectools/evals/report.py index 9f28a33..c05a1cd 100644 --- a/src/selectools/evals/report.py +++ b/src/selectools/evals/report.py @@ -159,11 +159,16 @@ def to_json(self, filepath: Union[str, Path]) -> None: """Write JSON report to file.""" Path(filepath).write_text(json.dumps(self.to_dict(), indent=2)) - def to_html(self, filepath: Union[str, Path]) -> None: - """Write self-contained HTML report to file.""" + def to_html(self, filepath: Union[str, Path], history: Optional[Any] = None) -> None: + """Write self-contained HTML report to file. + + Args: + filepath: Path to write the HTML file. + history: Optional HistoryTrend for accuracy trend chart. + """ from .html import render_html_report - render_html_report(self, filepath) + render_html_report(self, filepath, history=history) def to_junit_xml(self, filepath: Union[str, Path]) -> None: """Write JUnit XML for CI integration.""" @@ -171,6 +176,49 @@ def to_junit_xml(self, filepath: Union[str, Path]) -> None: render_junit_xml(self, filepath) + def to_markdown(self) -> str: + """Generate a markdown summary for GitHub issues, Slack, or PRs.""" + acc_emoji = "🟢" if self.accuracy >= 0.9 else "🟡" if self.accuracy >= 0.7 else "🔴" + lines = [ + f"## {acc_emoji} Eval Report: `{self.metadata.suite_name}`", + "", + "| Metric | Value |", + "|---|---|", + f"| **Accuracy** | **{self.accuracy:.1%}** " + f"({self.pass_count} pass, {self.fail_count} fail, {self.error_count} error) |", + f"| **Latency** | p50: {self.latency_p50:.0f}ms, p95: {self.latency_p95:.0f}ms |", + f"| **Cost** | ${self.total_cost:.6f} (${self.cost_per_case:.6f}/case) |", + f"| **Tokens** | {self.total_tokens:,} |", + f"| **Model** | {self.metadata.model or 'unknown'} |", + ] + + failures = [cr for cr in self.case_results if cr.verdict != CaseVerdict.PASS] + if failures: + lines.extend( + [ + "", + "
", + f"Failed cases ({len(failures)})", + "", + "| Case | Verdict | Issue |", + "|---|---|---|", + ] + ) + for cr in failures[:20]: + name = cr.case.name or cr.case.input[:50] + issues = "; ".join(f.message for f in cr.failures) or cr.error or "" + lines.append(f"| {name} | `{cr.verdict.value}` | {issues[:100]} |") + lines.extend(["", "
"]) + + lines.extend( + [ + "", + f"Generated by Selectools v{self.metadata.selectools_version} " + f"— an open-source project from NichevLabs", + ] + ) + return "\n".join(lines) + def summary(self) -> str: """Human-readable summary string.""" lines = [ diff --git a/src/selectools/evals/suite.py b/src/selectools/evals/suite.py index 58becb8..9ac3170 100644 --- a/src/selectools/evals/suite.py +++ b/src/selectools/evals/suite.py @@ -99,15 +99,45 @@ def estimate_cost(self) -> Dict[str, Any]: "pricing_available": prompt_cost_per_m > 0, } + def _notify_observers(self, event: str, **kwargs: Any) -> None: + """Fire observer events if the agent has observers configured.""" + if not hasattr(self.agent, "config") or not hasattr(self.agent.config, "observers"): + return + observers = self.agent.config.observers or [] + handler = f"on_{event}" + for obs in observers: + fn = getattr(obs, handler, None) + if fn: + try: + fn(**kwargs) + except Exception: # nosec B110 + pass # Observer errors must not break the eval + def run(self) -> EvalReport: """Run all cases synchronously and return an EvalReport.""" start = time.perf_counter() run_id = uuid.uuid4().hex[:12] + model = "" + if hasattr(self.agent, "config") and hasattr(self.agent.config, "model"): + model = self.agent.config.model or "" + self._notify_observers( + "eval_start", suite_name=self.name, total_cases=len(self.cases), model=model + ) + if self.max_concurrency <= 1: results = [] for i, case in enumerate(self.cases): - results.append(self._run_case(case)) + cr = self._run_case(case) + results.append(cr) + self._notify_observers( + "eval_case_end", + suite_name=self.name, + case_name=cr.case.name or cr.case.input[:50], + verdict=cr.verdict.value, + latency_ms=cr.latency_ms, + failures=len(cr.failures), + ) if self.on_progress: self.on_progress(i + 1, len(self.cases)) else: @@ -241,4 +271,17 @@ def _build_report( tags=self.tags, ) - return EvalReport(metadata=metadata, case_results=case_results) + report = EvalReport(metadata=metadata, case_results=case_results) + + self._notify_observers( + "eval_end", + suite_name=self.name, + accuracy=report.accuracy, + total_cases=len(case_results), + pass_count=report.pass_count, + fail_count=report.fail_count, + total_cost=report.total_cost, + duration_ms=duration_ms, + ) + + return report diff --git a/src/selectools/observer.py b/src/selectools/observer.py index 93ae269..6a5173e 100644 --- a/src/selectools/observer.py +++ b/src/selectools/observer.py @@ -379,6 +379,40 @@ def on_error( ) -> None: """Called when the agent encounters an unrecoverable error.""" + # ------------------------------------------------------------------ + # Eval events + # ------------------------------------------------------------------ + + def on_eval_start( + self, + suite_name: str, + total_cases: int, + model: str, + ) -> None: + """Called when an eval suite starts running.""" + + def on_eval_case_end( + self, + suite_name: str, + case_name: str, + verdict: str, + latency_ms: float, + failures: int, + ) -> None: + """Called after each eval case completes.""" + + def on_eval_end( + self, + suite_name: str, + accuracy: float, + total_cases: int, + pass_count: int, + fail_count: int, + total_cost: float, + duration_ms: float, + ) -> None: + """Called when an eval suite finishes.""" + # ====================================================================== # Built-in observers @@ -655,6 +689,44 @@ def on_kg_extraction(self, run_id: str, triples_extracted: int) -> None: def on_error(self, run_id: str, error: Exception, context: Dict[str, Any]) -> None: self._emit("error", run_id, error=str(error), error_type=type(error).__name__) + def on_eval_start(self, suite_name: str, total_cases: int, model: str) -> None: + self._emit("eval_start", "", suite_name=suite_name, total_cases=total_cases, model=model) + + def on_eval_case_end( + self, suite_name: str, case_name: str, verdict: str, latency_ms: float, failures: int + ) -> None: + self._emit( + "eval_case_end", + "", + suite_name=suite_name, + case_name=case_name, + verdict=verdict, + latency_ms=round(latency_ms, 1), + failures=failures, + ) + + def on_eval_end( + self, + suite_name: str, + accuracy: float, + total_cases: int, + pass_count: int, + fail_count: int, + total_cost: float, + duration_ms: float, + ) -> None: + self._emit( + "eval_end", + "", + suite_name=suite_name, + accuracy=round(accuracy, 4), + total_cases=total_cases, + pass_count=pass_count, + fail_count=fail_count, + total_cost=round(total_cost, 6), + duration_ms=round(duration_ms, 1), + ) + class AsyncAgentObserver(AgentObserver): """Base class for async agent lifecycle observers. diff --git a/tests/test_evals_e2e.py b/tests/test_evals_e2e.py index 2c3708c..7812532 100644 --- a/tests/test_evals_e2e.py +++ b/tests/test_evals_e2e.py @@ -632,13 +632,15 @@ def test_agent_a_wins(self) -> None: assert result.report_a.accuracy == 1.0 assert result.report_b.accuracy == 0.0 - def test_tie(self) -> None: + def test_both_pass(self) -> None: + """When both agents pass, result depends on latency — any outcome is valid.""" agent_a = _make_agent(["same answer"]) agent_b = _make_agent(["same answer"]) cases = [TestCase(input="Test", expect_contains="same")] result = PairwiseEval(agent_a, agent_b, cases).run() - assert result.ties == 1 + # Both passed, so winner depends on latency difference + assert result.a_wins + result.b_wins + result.ties == 1 def test_pairwise_summary(self) -> None: agent_a = _make_agent(["win"]) diff --git a/tests/test_evals_release.py b/tests/test_evals_release.py new file mode 100644 index 0000000..a0a0f3c --- /dev/null +++ b/tests/test_evals_release.py @@ -0,0 +1,257 @@ +"""Tests for v0.17.0 release features: to_markdown, observer, trend chart, pip extra.""" + +from __future__ import annotations + +from typing import Any, List +from unittest.mock import MagicMock + +import pytest + +from selectools import Agent, AgentConfig, tool +from selectools.evals import ( + CaseResult, + CaseVerdict, + EvalMetadata, + EvalReport, + EvalSuite, + HistoryStore, + HistoryTrend, + TestCase, +) +from selectools.evals.history import HistoryEntry +from selectools.evals.html import _trend_svg +from selectools.observer import AgentObserver +from tests.conftest import SharedFakeProvider + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@tool(description="test tool") +def dummy_tool(x: str) -> str: + return x + + +def _make_agent(responses: list, observers: list | None = None) -> Agent: + provider = SharedFakeProvider(responses=responses) + config = AgentConfig(model="fake-model") + if observers: + config.observers = observers + return Agent(provider=provider, config=config, tools=[dummy_tool]) + + +def _make_report(name: str = "test", accuracy: float = 1.0) -> EvalReport: + n_pass = int(accuracy * 4) + n_fail = 4 - n_pass + cases = [] + for i in range(n_pass): + tc = TestCase(input=f"p{i}", name=f"pass_{i}") + cases.append(CaseResult(case=tc, verdict=CaseVerdict.PASS, latency_ms=100, cost_usd=0.001)) + for i in range(n_fail): + tc = TestCase(input=f"f{i}", name=f"fail_{i}") + cases.append( + CaseResult( + case=tc, + verdict=CaseVerdict.FAIL, + latency_ms=200, + cost_usd=0.002, + failures=[MagicMock(evaluator_name="contains", message="missing substring")], + ) + ) + meta = EvalMetadata(name, "gpt-test", "Fake", 1000, "r1", 4, 500, "0.17.0") + return EvalReport(metadata=meta, case_results=cases) + + +# =========================================================================== +# #4: to_markdown() +# =========================================================================== + + +class TestToMarkdown: + def test_basic_output(self) -> None: + report = _make_report() + md = report.to_markdown() + assert "## 🟢 Eval Report:" in md + assert "**Accuracy**" in md + assert "100.0%" in md + assert "NichevLabs" in md + + def test_with_failures(self) -> None: + report = _make_report(accuracy=0.5) + md = report.to_markdown() + assert "🟡" in md or "🔴" in md + assert "Failed cases" in md + assert "
" in md + assert "fail_" in md + + def test_red_badge_low_accuracy(self) -> None: + report = _make_report(accuracy=0.0) + md = report.to_markdown() + assert "🔴" in md + + def test_contains_model(self) -> None: + report = _make_report() + md = report.to_markdown() + assert "gpt-test" in md + + +# =========================================================================== +# #9: Observer events +# =========================================================================== + + +class TestEvalObserverEvents: + def test_eval_start_fires(self) -> None: + events: list[str] = [] + + class TestObserver(AgentObserver): + def on_eval_start(self, suite_name: str, total_cases: int, model: str) -> None: + events.append(f"start:{suite_name}:{total_cases}:{model}") + + agent = _make_agent(["ok"], observers=[TestObserver()]) + suite = EvalSuite(agent=agent, cases=[TestCase(input="x")]) + suite.run() + assert len(events) == 1 + assert events[0] == "start:eval:1:fake-model" + + def test_eval_case_end_fires(self) -> None: + events: list[dict] = [] + + class TestObserver(AgentObserver): + def on_eval_case_end( + self, + suite_name: str, + case_name: str, + verdict: str, + latency_ms: float, + failures: int, + ) -> None: + events.append({"case": case_name, "verdict": verdict}) + + agent = _make_agent(["ok"], observers=[TestObserver()]) + suite = EvalSuite( + agent=agent, + cases=[ + TestCase(input="a", name="case_a"), + TestCase(input="b", name="case_b"), + ], + ) + suite.run() + assert len(events) == 2 + assert events[0]["case"] == "case_a" + assert events[0]["verdict"] == "pass" + + def test_eval_end_fires(self) -> None: + events: list[dict] = [] + + class TestObserver(AgentObserver): + def on_eval_end( + self, + suite_name: str, + accuracy: float, + total_cases: int, + pass_count: int, + fail_count: int, + total_cost: float, + duration_ms: float, + ) -> None: + events.append({"accuracy": accuracy, "total": total_cases}) + + agent = _make_agent(["hello"], observers=[TestObserver()]) + suite = EvalSuite( + agent=agent, + cases=[TestCase(input="x", expect_contains="hello")], + ) + suite.run() + assert len(events) == 1 + assert events[0]["accuracy"] == 1.0 + assert events[0]["total"] == 1 + + def test_observer_errors_dont_break_eval(self) -> None: + class BrokenObserver(AgentObserver): + def on_eval_start(self, **kwargs: Any) -> None: + raise RuntimeError("observer crash") + + agent = _make_agent(["ok"], observers=[BrokenObserver()]) + suite = EvalSuite(agent=agent, cases=[TestCase(input="x")]) + report = suite.run() + assert report.pass_count == 1 # Eval still completes + + def test_no_observers_no_crash(self) -> None: + agent = _make_agent(["ok"]) + suite = EvalSuite(agent=agent, cases=[TestCase(input="x")]) + report = suite.run() + assert report.pass_count == 1 + + +# =========================================================================== +# #10: Trend chart SVG +# =========================================================================== + + +class TestTrendSvg: + def test_basic_trend(self) -> None: + svg = _trend_svg([0.7, 0.8, 0.9, 0.95]) + assert " None: + svg = _trend_svg([0.9, 0.8, 0.7]) + assert "#f87171" in svg # red (declining) + + def test_too_few_points(self) -> None: + assert _trend_svg([0.8]) == "" + assert _trend_svg([]) == "" + + def test_two_points(self) -> None: + svg = _trend_svg([0.5, 0.9]) + assert " None: + report = _make_report() + trend = HistoryTrend( + entries=[ + HistoryEntry("r1", "s", 0, 0.7, 7, 3, 0, 0.01, 100, 100, 200, 10, "m", 500), + HistoryEntry("r2", "s", 0, 0.8, 8, 2, 0, 0.01, 100, 100, 200, 10, "m", 500), + HistoryEntry("r3", "s", 0, 0.9, 9, 1, 0, 0.01, 100, 100, 200, 10, "m", 500), + ] + ) + path = tmp_path / "trend_report.html" + report.to_html(path, history=trend) + content = path.read_text() + assert "Accuracy Trend" in content + assert "polyline" in content + + def test_html_without_history(self, tmp_path: Any) -> None: + report = _make_report() + path = tmp_path / "no_trend.html" + report.to_html(path) + content = path.read_text() + assert "Accuracy Trend" not in content + + +# =========================================================================== +# #8: pip extra (structural test) +# =========================================================================== + + +class TestPipExtra: + def test_evals_import_without_pyyaml(self) -> None: + """Core eval framework works without pyyaml installed.""" + from selectools.evals import EvalReport, EvalSuite, TestCase + + assert EvalSuite is not None + assert TestCase is not None + assert EvalReport is not None + + def test_yaml_loader_gives_helpful_error(self) -> None: + """from_yaml should work if pyyaml is installed (it is in dev).""" + from selectools.evals import DatasetLoader + + # This should not raise ImportError since pyyaml is in dev deps + # Just verify the method exists + assert hasattr(DatasetLoader, "from_yaml")