Skip to content

Mark grader content-filter refusals as unscored #10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
iphan merged 2 commits into from
Jun 8, 2026
Merged

Mark grader content-filter refusals as unscored #10

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9fb4d5f
Lazily import EdisonScientific/labbench2 deps
iphan Jun 5, 2026
49346d7
Mark grader content-filter refusals as unscored instead of INCORRECT
iphan Jun 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions src/lab_bench_2/attachment_builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,6 @@
import hashlib
from pathlib import Path

# MEDIA_TYPES is reused from EdisonScientific/labbench2 evals/utils.py.
# Its unusual entries (".json" / ".csv" -> "text/plain") are deliberate
# provider-compatibility workarounds: Vertex AI rejects application/json and
# Anthropic's Messages document API rejects text/csv.
from evals.utils import MEDIA_TYPES
from inspect_ai.model import Content, ContentDocument, ContentImage

IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg"}
Expand All @@ -34,6 +29,12 @@ def build(files: list[Path], resize_oversized_images: bool = False) -> list[Cont


def mime_type(file_path: Path) -> str:
# MEDIA_TYPES is reused from EdisonScientific/labbench2 evals/utils.py.
# Its unusual entries (".json" / ".csv" -> "text/plain") are deliberate
# provider-compatibility workarounds: Vertex AI rejects application/json and
# Anthropic's Messages document API rejects text/csv.
from evals.utils import MEDIA_TYPES

return MEDIA_TYPES.get(file_path.suffix.lower(), "application/octet-stream")


Expand Down
13 changes: 11 additions & 2 deletions src/lab_bench_2/dataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,21 @@
import logging
from collections.abc import Sequence
from pathlib import Path
from typing import Any, cast
from typing import TYPE_CHECKING, Any, cast

from evals.models import LabBenchQuestion
from inspect_ai.dataset import Dataset, MemoryDataset, Sample, hf_dataset
from inspect_ai.model import ChatMessage, ChatMessageUser, Content, ContentText

from lab_bench_2 import attachment_builder, file_downloader, prompt_composer
from lab_bench_2.prompt_composer import Mode
from utils.sample_ids import create_stable_id

if TYPE_CHECKING:
# ``evals`` ships with the optional ``labbench2`` dependency; import it only
# for type checking so module load doesn't require the extra (which would
# break Inspect's entry-point task discovery). Runtime uses import it locally.
from evals.models import LabBenchQuestion

LAB_BENCH_2_DATASET_PATH = "EdisonScientific/labbench2"
LAB_BENCH_2_DATASET_REVISION = "27d12d72af24e3f70db8a99df63e567366cbdb80"
LAB_BENCH_2_DATASET_SPLIT = "train"
Expand All @@ -40,6 +45,8 @@ def record_to_sample(record: dict[str, Any], mode: Mode = "inject") -> Sample:
Precondition: the record's question supports ``mode``. The loader filters
unsupported records via ``_question_supports_mode`` before calling this.
"""
from evals.models import LabBenchQuestion

question = LabBenchQuestion.model_validate(record)

metadata: dict[str, Any] = {
Expand Down Expand Up @@ -102,6 +109,8 @@ def load_lab_bench_2_dataset(
"""

def to_samples(record: dict[str, Any]) -> list[Sample]:
from evals.models import LabBenchQuestion

question = LabBenchQuestion.model_validate(record)
if not _question_supports_mode(question, mode):
logger.info(
Expand Down
11 changes: 7 additions & 4 deletions src/lab_bench_2/file_downloader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,17 @@
from pathlib import Path
from typing import cast

from evals.utils import GCS_BUCKET, download_question_files


def fetch(gcs_prefix: str, bucket_name: str = GCS_BUCKET) -> Path:
def fetch(gcs_prefix: str, bucket_name: str | None = None) -> Path:
"""Download files under ``gcs_prefix`` and return the local directory.

Raises ``RuntimeError`` if the prefix yields no files.
``bucket_name`` defaults to the reference implementation's ``GCS_BUCKET``
when omitted. Raises ``RuntimeError`` if the prefix yields no files.
"""
from evals.utils import GCS_BUCKET, download_question_files

if bucket_name is None:
bucket_name = GCS_BUCKET
files_path = cast(
Path,
download_question_files(bucket_name=bucket_name, gcs_prefix=gcs_prefix),
Expand Down
4 changes: 2 additions & 2 deletions src/lab_bench_2/prompt_composer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@
from pathlib import Path
from typing import Literal

from evals.utils import is_text_injectable_format

Mode = Literal["file", "inject", "retrieve"]

# Copied verbatim from EdisonScientific/labbench2
Expand Down Expand Up @@ -44,6 +42,8 @@ def compose(


def _injected_files_text(files: list[Path]) -> str:
from evals.utils import is_text_injectable_format

chunks = [
f"## {f.name}\n\n{f.read_text()}" for f in files if is_text_injectable_format(f)
]
Expand Down
59 changes: 52 additions & 7 deletions src/lab_bench_2/scorers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,12 @@
from pathlib import Path
from typing import Any, cast

from inspect_ai.model import GenerateConfig, ResponseSchema, get_model
from inspect_ai.model import (
GenerateConfig,
ModelOutput,
ResponseSchema,
get_model,
)
from inspect_ai.scorer import (
CORRECT,
INCORRECT,
Expand All @@ -27,6 +32,10 @@
DEFAULT_GRADER_MODEL = "anthropic/claude-sonnet-4-5"
GRADER_ROLE = "grader"

# Content moderation can block a grader response non-deterministically, so a
# blocked call might clear on a retry. Cap the attempts before giving up.
MAX_GRADER_ATTEMPTS = 3

JUDGE_VERDICT_CORRECT = "correct"
JUDGE_VERDICT_INCORRECT = "incorrect"
JUDGE_VERDICT_UNSURE = "unsure"
Expand Down Expand Up @@ -59,24 +68,49 @@ def _judge_score(prompt_template: str) -> Scorer:
name="evaluation_result", json_schema=json_schema(EvaluationResult)
)

async def call_grader_with_retry(prompt: str) -> ModelOutput | None:

@ItsTania ItsTania Jun 8, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the first time I've seen the Score.unscored option - v cool
I like that it enables future scoring and really highlights that the issue is with the scoring (and make re-scoring easier https://inspect.aisi.org.uk/scoring-workflow.html)

The pattern I previously knew of is to raise an error and rely on --retry-on-error (https://inspect.aisi.org.uk/options.html#errors-and-limits). The benefit is the retry is built in, but the ^ re-scoring options aren't (get logged as errors after x retries))

I wonder if there is a neat fix combining the two in inspect ai! Esp as LLM judges are getting more popular as scoring options.

result: ModelOutput | None = None
grader = get_model(role=GRADER_ROLE, default=DEFAULT_GRADER_MODEL)
attempts = 0
while attempts < MAX_GRADER_ATTEMPTS:
attempts += 1
result = await grader.generate(
prompt,
config=GenerateConfig(temperature=0.0, response_schema=response_schema),
)
if not _grader_refused(result):
break
return result

async def score(state: TaskState, target: Target) -> Score:
answer = state.output.completion.strip()
if not answer:
return Score(
value=INCORRECT, answer="", explanation="No answer was produced."
)

grader = get_model(role=GRADER_ROLE, default=DEFAULT_GRADER_MODEL)

prompt = prompt_template.format(
question=state.input_text,
correct_answer=target.text,
answer=answer,
)
result = await grader.generate(
prompt,
config=GenerateConfig(temperature=0.0, response_schema=response_schema),
)

result = await call_grader_with_retry(prompt)

if result is None or _grader_refused(result):
stop_reason = getattr(result, "stop_reason", None)
return Score.unscored(
answer=answer,
explanation=(
f"Grader returned no gradeable response "
f"attempt(s) (stop_reason={stop_reason}); sample left unscored."
),
metadata={
"verdict": None,
"verdict_source": "refusal",
"grader_stop_reason": stop_reason,
},
)

try:
evaluation = EvaluationResult.model_validate_json(result.completion)
Expand All @@ -99,6 +133,17 @@ async def score(state: TaskState, target: Target) -> Score:
return score


def _grader_refused(result: ModelOutput) -> bool:
"""Return True when the grader produced no gradeable response.

A ``content_filter`` stop reason or an empty completion means moderation
blocked the grader (or it returned nothing), not that the answer is wrong.
"""
if result.stop_reason == "content_filter":

@iphan iphan Jun 6, 2026

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ItsTania I was able to repro the issue that you described here with Sonnet as the grader. It turned out this was due to content filtering.
I chose to exclude the sample from the results in this case where the grader model refuses to answer.

thanks again for pointing out this issue

@ItsTania ItsTania Jun 8, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thats so interesting that the content filter blocked the grading stage but not the actual question answering stage ahah

@iphan iphan Jun 8, 2026

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess the different provider have different content filters: the model under test is gpt 5.2, whereas the judge is Sonnet 4.5

return True
return not (result.completion or "").strip()


@scorer(metrics=[accuracy(), stderr()])
def semantic_judge_scorer() -> Scorer:
"""Grade an open-ended answer against the reference using a judge model."""
Expand Down
10 changes: 3 additions & 7 deletions tests/lab_bench_2/test_file_downloader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,7 @@ def test_raises_when_no_files_downloaded(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
# given a download that returns an empty directory
monkeypatch.setattr(
file_downloader, "download_question_files", lambda **_: tmp_path
)
monkeypatch.setattr("evals.utils.download_question_files", lambda **_: tmp_path)

# when / then
with pytest.raises(RuntimeError, match="none were downloaded"):
Expand All @@ -49,9 +47,7 @@ def test_returns_directory_when_populated(
) -> None:
# given a populated download
(tmp_path / "ok.txt").write_text("ok")
monkeypatch.setattr(
file_downloader, "download_question_files", lambda **_: tmp_path
)
monkeypatch.setattr("evals.utils.download_question_files", lambda **_: tmp_path)

# when
result = file_downloader.fetch("some/prefix")
Expand All @@ -70,7 +66,7 @@ def stub(**kwargs: str) -> Path:
captured.update(kwargs)
return tmp_path

monkeypatch.setattr(file_downloader, "download_question_files", stub)
monkeypatch.setattr("evals.utils.download_question_files", stub)

# when
file_downloader.fetch("some/prefix", bucket_name="custom-bucket")
Expand Down
42 changes: 42 additions & 0 deletions tests/lab_bench_2/test_import_hygiene.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
import subprocess
import sys
import textwrap


def test_package_imports_without_optional_labbench2_dependency() -> None:

@ItsTania ItsTania Jun 8, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not relevant to this PR - but I wonder if this optional dependency is a weakness of the template... it seems like it added more confusion than necessary for most people who would only implement one eval per repo...

@iphan iphan Jun 8, 2026

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If there is only a single eval in the repo, users can make the deps required. Just add them in the main dependencies list here.
I put LB2's deps as optional to test the template setup for the multi-eval per repo case

# given a fresh interpreter where the optional ``labbench2`` extra (which
# provides the ``evals`` / ``labbench2`` modules) cannot be imported
script = textwrap.dedent(
"""
import sys

class _Blocker:
def find_spec(self, name, path=None, target=None):
if name.split(".")[0] in {"evals", "labbench2"}:
raise ModuleNotFoundError(name)
return None

sys.meta_path.insert(0, _Blocker())

# Importing the package is what Inspect does for entry-point task
# discovery; it must not pull in the optional dependency.
import lab_bench_2
from lab_bench_2 import lab_bench_2 as task, SUPPORTED_TAGS

assert callable(task)
assert "litqa3" in SUPPORTED_TAGS
assert "evals" not in sys.modules
assert "labbench2" not in sys.modules
"""
)

# when the package is imported in that interpreter
result = subprocess.run(
[sys.executable, "-c", script],
capture_output=True,
text=True,
check=False,
)

# then discovery succeeds without the optional dependency installed
assert result.returncode == 0, result.stderr
Loading
Loading