chore(evaluators): refresh contrib template packaging scaffold

evaluators/contrib/template/Makefile

@@ -0,0 +1,31 @@
+.PHONY: help test lint lint-fix typecheck check build
+
+PACKAGE_DIR := $(notdir $(abspath $(CURDIR)))
+COVERAGE_XML := ../../../coverage-evaluators-$(PACKAGE_DIR).xml
+
+help:
+	@echo "Agent Control contrib evaluator template"
+	@echo ""
+	@echo "  make test       - run pytest"
+	@echo "  make lint       - run ruff check"
+	@echo "  make lint-fix   - run ruff check --fix"
+	@echo "  make typecheck  - run mypy"
+	@echo "  make check      - run lint, typecheck, and tests"
+	@echo "  make build      - build package"
+
+test:
+	uv run --with pytest --with pytest-asyncio --with pytest-cov pytest tests --cov=src --cov-report=xml:$(COVERAGE_XML) -q
+
+lint:
+	uv run --with ruff ruff check --config ../../../pyproject.toml src/
+
+lint-fix:
+	uv run --with ruff ruff check --config ../../../pyproject.toml --fix src/
+
+typecheck:
+	uv run --with mypy mypy --config-file ../../../pyproject.toml src/
+
+check: lint typecheck test
+
+build:
+	uv build

evaluators/contrib/template/README.md

@@ -1,5 +1,121 @@
-# Evaluator Template
+# Contrib Evaluator Template
 
-Starter template for creating a custom evaluator package.
+This directory is scaffolding for a new contrib evaluator package.
+
+It is intentionally excluded from repo automation until you convert it into a real package. In
+particular, `template/` does not participate in root `make check`, CI, semantic-release, or
+publishing because it ships a `pyproject.toml.template` placeholder instead of a real
+`pyproject.toml`.
+
+## Naming contract
+
+Pick `<name>` as a short lowercase single-word identifier such as `galileo`, `cisco`, or
+`budget`. That same value should appear in the steady-state package shape:
+
+- directory: `evaluators/contrib/<name>/`
+- pip package: `agent-control-evaluator-<name>`
+- Python module: `agent_control_evaluator_<name>`
+- extra name: `agent-control-evaluators[<name>]`
+
+The template uses `{{NAME}}` for that package identifier. It does not use `{{ORG}}`.
+
+Keep the public evaluator reference separate from the package identifier:
+
+- `{{ENTRY_POINT}}` is the user-facing evaluator name and should match
+  `EvaluatorMetadata.name` in your package code.
+- Single-evaluator packages can keep that public name flat, such as `budget`.
+- Packages that expose a family of evaluator ids should namespace it, such as
+  `cisco.ai_defense` or `galileo.luna2`.
+
+## Scaffold a new contrib package
+
+1. Copy the template and rename the manifest:
+
+   ```bash
+   cp -r evaluators/contrib/template evaluators/contrib/<name>
+   mv evaluators/contrib/<name>/pyproject.toml.template \
+     evaluators/contrib/<name>/pyproject.toml
+   ```
+
+2. Replace placeholders in `pyproject.toml`:
+
+   - `{{NAME}}` -> contrib package identifier
+   - `{{ENTRY_POINT}}` -> public evaluator reference / `EvaluatorMetadata.name`
+   - `{{EVALUATOR}}` -> evaluator module path segment (for example `budget` or `ai_defense`)
+   - `{{CLASS}}` -> evaluator class name
+   - `{{AUTHOR}}` -> authoring team
+
+   For a package with one primary evaluator, `{{ENTRY_POINT}}` is often just `<name>`. For a
+   package that groups provider-specific evaluators, use `<name>.<evaluator_id>`.
+
+   The template starts new packages at `0.1.0`; change that if your release plan differs.
+   Also replace the copied `README.md` with package-specific install, configuration, and usage
+   docs before your first build or publish. Then confirm the package `version` reflects your
+   release plan and that the `agent-control-evaluators` / `agent-control-models` dependency
+   floors match the compatibility floor you intend to support. Keep those dependency floors
+   aligned with the builtin extra you add below before you commit the new package.
+
+3. Add package code and tests:
+
+   - `src/agent_control_evaluator_<name>/`
+   - `tests/`
+
+4. Validate the package locally:
+
+   ```bash
+   make lint
+   make lint-fix
+   make typecheck
+   make test
+   make check
+   make build
+   ```
+
+## Canonical install docs
+
+Contributor-facing and user-facing package docs should treat this as the canonical install path:
+
+```bash
+pip install "agent-control-evaluators[<name>]"
+```
+
+Direct wheel installs such as `pip install agent-control-evaluator-<name>` can still be
+documented, but they are secondary to the extra on `agent-control-evaluators`.
+
+## Expected repo wiring
+
+After the new package exists as a real contrib package, wire it into the repo contract:
+
+1. Add the extra to `evaluators/builtin/pyproject.toml`:
+
+   ```toml
+   [project.optional-dependencies]
+   <name> = ["agent-control-evaluator-<name>>=<minimum-compatible-version>"]
+   ```
+
+   Keep this extra on the current monorepo release line. The release build rewrites builtin
+   dependency floors to the active release version before publishing
+   `agent-control-evaluators`, so a lower source floor here would not survive into the
+   published extra metadata.
+
+2. Add the workspace source pin to `evaluators/builtin/pyproject.toml`:
+
+   ```toml
+   [tool.uv.sources]
+   agent-control-evaluator-<name> = { path = "../contrib/<name>", editable = true }
+   ```
+
+3. Add the package to `tool.semantic_release.version_toml` in the root `pyproject.toml`:
+
+   ```toml
+   "evaluators/contrib/<name>/pyproject.toml:project.version",
+   ```
+
+   The repo's release automation discovers real contrib packages automatically via
+   `scripts/contrib_packages.py`, so once the package has a real `pyproject.toml` and the
+   builtin extra / uv source wiring above is in place, `scripts/build.py` and
+   `.github/workflows/release.yaml` will pick it up without additional manual edits.
+
+Until those steps are done, the package is still scaffolding rather than a real contrib package.
 
 Docs: https://docs.agentcontrol.dev/concepts/evaluators/contributing-evaluator

evaluators/contrib/template/pyproject.toml.template

@@ -1,7 +1,8 @@
 [project]
-name = "agent-control-evaluator-{{ORG}}"
-version = "3.0.0"
-description = "{{ORG}} evaluators for agent-control"
+name = "agent-control-evaluator-{{NAME}}"
+version = "0.1.0"
+description = "{{NAME}} evaluators for agent-control"
+readme = "README.md"
 requires-python = ">=3.12"
 license = { text = "Apache-2.0" }
 authors = [{ name = "{{AUTHOR}}" }]
@@ -15,20 +16,22 @@ dependencies = [
 dev = [
     "pytest>=8.0.0",
     "pytest-asyncio>=0.23.0",
+    "pytest-cov>=4.0.0",
     "ruff>=0.1.0",
     "mypy>=1.8.0",
 ]
 
 [project.entry-points."agent_control.evaluators"]
-# Format: "org.evaluator_name" = "package.module:Class"
-"{{ORG}}.{{EVALUATOR}}" = "agent_control_evaluator_{{ORG}}.{{EVALUATOR}}:{{CLASS}}"
+# Keep this aligned with EvaluatorMetadata.name (for example "budget" or
+# "cisco.ai_defense").
+"{{ENTRY_POINT}}" = "agent_control_evaluator_{{NAME}}.{{EVALUATOR}}:{{CLASS}}"
 
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"
 
 [tool.hatch.build.targets.wheel]
-packages = ["src/agent_control_evaluator_{{ORG}}"]
+packages = ["src/agent_control_evaluator_{{NAME}}"]
 
 [tool.uv.sources]
 agent-control-evaluators = { path = "../../builtin", editable = true }