Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 27 additions & 1 deletion llm-worker/contracts.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,22 @@ def _clean_mitre(values: list[str]) -> list[str]:
return list(dict.fromkeys(values))


# The largest `maxLength` any annotation field may declare.
#
# Every one of these schemas is handed to Ollama as a structured-output
# `format`, and Ollama compiles it into a GBNF grammar where a bounded string
# becomes a repetition rule. Past roughly 1200 that grammar stops compiling
# and the request fails with `400 Failed to initialize samplers: failed to
# parse grammar` -- rejecting the schema entirely, not truncating anything.
#
# #1748: DailyReport shipped with 2000 and its pipeline never produced a
# single real output. Nothing caught it, because the failure is invisible
# until that annotation type actually runs against a live model. test_
# contracts.py asserts this bound instead, so a future field cannot quietly
# disable a pipeline the same way.
MAX_ANNOTATION_STRING = 1200


class SessionAnalysis(StrictAnnotation):
summary: str = Field(min_length=1, max_length=1200)
intent: Literal[
Expand Down Expand Up @@ -174,7 +190,17 @@ def validate_iocs(cls, values: list[str]) -> list[str]:


class DailyReport(StrictAnnotation):
summary: str = Field(min_length=1, max_length=2000)
# 1200, not 2000: Ollama's grammar compiler rejects the whole schema
# outright above roughly this bound, with
# `400 Failed to initialize samplers: failed to parse grammar` (#1748).
# A bounded string becomes a repetition rule in the generated GBNF, and
# somewhere between 1200 and 2000 that stops compiling. Bisected down to
# this single field: `summary` alone reproduces it, the three arrays
# together do not, and restoring 1200 fixes the full schema.
#
# This is a ceiling on what may be *asked of the model*, not on what is
# accepted -- see MAX_ANNOTATION_STRING and the validator below.
summary: str = Field(min_length=1, max_length=MAX_ANNOTATION_STRING)
highlights: list[str] = Field(max_length=20)
trends: list[str] = Field(max_length=20)
recommended_checks: list[str] = Field(max_length=20)
Expand Down
56 changes: 56 additions & 0 deletions llm-worker/tests/test_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

import contracts
import worker # noqa: E402
from contracts import SessionAnalysis # noqa: E402

Expand Down Expand Up @@ -257,6 +258,61 @@ def test_canary_scans_bounded_cycles_and_stops_after_one_session(self):
self.assertEqual(result["reports"], 0)


class AnnotationSchemaGrammarBoundTests(unittest.TestCase):
"""#1748: no annotation field may declare a maxLength Ollama cannot compile.

Each of these schemas is sent to Ollama as a structured-output `format`,
and a bounded string becomes a repetition rule in the generated GBNF. Past
roughly 1200 that grammar stops compiling and the request is rejected
outright:

400 Failed to initialize samplers: failed to parse grammar

DailyReport shipped with summary maxLength 2000 and its pipeline never
produced a single real output. Nothing caught it, because the failure only
appears when that annotation type runs against a live model -- and it was
only enabled long after it was written.

Bisected to that one field: `summary` alone reproduces it, all three
arrays together do not, and lowering it to 1200 fixes the full schema.
"""

ANNOTATIONS = (contracts.SessionAnalysis, contracts.PayloadAnalysis, contracts.DailyReport)

def _string_bounds(self, schema, path=""):
"""Every maxLength in a JSON schema, including inside arrays and $defs."""
found = []
if isinstance(schema, dict):
if schema.get("type") == "string" and "maxLength" in schema:
found.append((path or "<root>", schema["maxLength"]))
for key, value in schema.items():
found += self._string_bounds(value, f"{path}.{key}" if path else key)
elif isinstance(schema, list):
for i, value in enumerate(schema):
found += self._string_bounds(value, f"{path}[{i}]")
return found

def test_no_field_exceeds_the_compilable_bound(self):
for annotation in self.ANNOTATIONS:
for field, bound in self._string_bounds(annotation.model_json_schema()):
self.assertLessEqual(
bound, contracts.MAX_ANNOTATION_STRING,
f"{annotation.__name__} {field} declares maxLength={bound}; Ollama's "
f"grammar compiler rejects the whole schema above "
f"{contracts.MAX_ANNOTATION_STRING} (#1748)",
)

def test_the_bound_is_one_shared_constant(self):
# The point of a single constant is that raising the ceiling is a
# deliberate, reviewable act rather than a number someone copies.
self.assertEqual(contracts.MAX_ANNOTATION_STRING, 1200)

def test_daily_report_specifically(self):
# The regression itself, named, so a revert is unambiguous.
summary = contracts.DailyReport.model_json_schema()["properties"]["summary"]
self.assertEqual(summary["maxLength"], 1200)


class CycleStageIsolationTests(unittest.TestCase):
"""#1748: one failing stage must not discard the others' work.

Expand Down
Loading