feat(nlp2dsl): constrain live conversation fields

.env.example

@@ -1,5 +1,6 @@
 # OpenRouter identifies requests in its App column through X-Title.
 OPENROUTER_API_KEY=
 OPENROUTER_APP_NAME=testql
-OPENROUTER_SITE_URL=
+OPENROUTER_APP_URL=
 LLM_MODEL=openrouter/z-ai/glm-5.2
+TESTQL_LIVE_LLM_MODEL=openrouter/z-ai/glm-5.2

README.md

@@ -128,6 +128,10 @@ Set `OPENROUTER_APP_NAME` to identify TestQL in OpenRouter logs. If it is not
 set, TestQL uses the current project folder name. See `.env.example` for the
 GLM 5.2 defaults.
 
+The optional live nlp2dsl conversation provider similarly uses
+`ConversationFields 1.0.0`. It returns only requested missing fields, while
+preserving the existing plain field mapping passed into `llmContext`.
+
 
 ## Artifact Discovery, Topology, and Web Inspection
 

pyproject.toml

@@ -84,6 +84,9 @@ testql = [
     "contracts/nlp2env/v1/*.gbnf",
     "contracts/nlp2env/v1/*.json",
     "contracts/nlp2env/v1/*.proto",
+    "contracts/nlp2dsl_conversation/v1/*.gbnf",
+    "contracts/nlp2dsl_conversation/v1/*.json",
+    "contracts/nlp2dsl_conversation/v1/*.proto",
     "data/*.json",
     "interpreter/*.cjs",
 ]

testql/adapters/nlp2dsl/live_llm.py

@@ -9,26 +9,38 @@
 
 import httpx
 
+from testql.contracts.nlp2dsl_conversation import response_format, validate_payload
+
 
 @dataclass
 class LiveLLMProvider:
     """Call an OpenAI-compatible chat API to fill missing dialog fields."""
 
     api_key: str
-    model: str = "openrouter/qwen/qwen3-coder-next"
+    model: str = "openrouter/z-ai/glm-5.2"
     base_url: str = "https://openrouter.ai/api/v1"
     timeout_s: float = 60.0
     extra_headers: dict[str, str] = field(default_factory=dict)
 
     @classmethod
-    def from_env(cls) -> "LiveLLMProvider":
-        api_key = os.environ.get("OPENROUTER_API_KEY") or os.environ.get("LLM_API_KEY") or ""
+    def from_env(cls) -> LiveLLMProvider:
+        api_key = (
+            os.environ.get("OPENROUTER_API_KEY") or os.environ.get("LLM_API_KEY") or ""
+        )
         if not api_key:
-            raise RuntimeError("TESTQL_LIVE_LLM=1 requires OPENROUTER_API_KEY or LLM_API_KEY")
+            raise RuntimeError(
+                "TESTQL_LIVE_LLM=1 requires OPENROUTER_API_KEY or LLM_API_KEY"
+            )
         return cls(
             api_key=api_key,
-            model=os.environ.get("TESTQL_LIVE_LLM_MODEL", os.environ.get("LLM_MODEL", "openrouter/qwen/qwen3-coder-next")),
-            base_url=os.environ.get("TESTQL_LIVE_LLM_BASE_URL", os.environ.get("LLM_BASE_URL", "https://openrouter.ai/api/v1")).rstrip("/"),
+            model=os.environ.get(
+                "TESTQL_LIVE_LLM_MODEL",
+                os.environ.get("LLM_MODEL", "openrouter/z-ai/glm-5.2"),
+            ),
+            base_url=os.environ.get(
+                "TESTQL_LIVE_LLM_BASE_URL",
+                os.environ.get("LLM_BASE_URL", "https://openrouter.ai/api/v1"),
+            ).rstrip("/"),
         )
 
     def reply_for(
@@ -38,17 +50,28 @@ def reply_for(
         missing: list[str] | None = None,
         context: dict[str, Any] | None = None,
     ) -> dict[str, Any]:
-        prompt = self._build_prompt(conversation_id, missing=missing or [], context=context or {})
+        prompt = self._build_prompt(
+            conversation_id, missing=missing or [], context=context or {}
+        )
         content = self._chat(prompt)
-        return self._parse_json_object(content)
+        payload = self._parse_json_object(content)
+        fields = payload["fields"]
+        unexpected = set(fields) - set(missing or [])
+        if missing and unexpected:
+            names = ", ".join(sorted(unexpected))
+            raise ValueError(f"live LLM returned fields outside missing set: {names}")
+        return fields
 
-    def _build_prompt(self, conversation_id: str, *, missing: list[str], context: dict[str, Any]) -> str:
+    def _build_prompt(
+        self, conversation_id: str, *, missing: list[str], context: dict[str, Any]
+    ) -> str:
         return (
             "You are completing missing fields for an automated integration test.\n"
             f"conversationId: {conversation_id}\n"
             f"missing fields: {missing}\n"
             f"context: {json.dumps(context, ensure_ascii=False)}\n"
-            "Respond with a single JSON object only — keys should address the missing fields "
+            "Respond with a ConversationFields 1.0.0 JSON object only. "
+            "Put values under fields; keys must address the missing fields "
             "(e.g. attachmentPath, recipient). No markdown."
         )
 
@@ -57,12 +80,22 @@ def _chat(self, prompt: str) -> str:
         headers = {
             "Authorization": f"Bearer {self.api_key}",
             "Content-Type": "application/json",
+            "X-Title": os.environ.get("OPENROUTER_APP_NAME", "").strip()
+            or os.path.basename(os.getcwd())
+            or "testql",
             **self.extra_headers,
         }
+        app_url = (
+            os.environ.get("OPENROUTER_APP_URL", "").strip()
+            or os.environ.get("OPENROUTER_SITE_URL", "").strip()
+        )
+        if app_url and "HTTP-Referer" not in headers:
+            headers["HTTP-Referer"] = app_url
         payload = {
-            "model": self.model,
+            "model": self.model.removeprefix("openrouter/"),
             "messages": [{"role": "user", "content": prompt}],
             "temperature": 0,
+            "response_format": response_format(),
         }
         with httpx.Client(timeout=self.timeout_s) as client:
             response = client.post(url, headers=headers, json=payload)
@@ -79,12 +112,11 @@ def _chat(self, prompt: str) -> str:
 
     @staticmethod
     def _parse_json_object(text: str) -> dict[str, Any]:
-        stripped = text.strip()
-        if stripped.startswith("```"):
-            stripped = stripped.strip("`")
-            if stripped.lower().startswith("json"):
-                stripped = stripped[4:].strip()
-        parsed = json.loads(stripped)
+        try:
+            parsed = json.loads(text)
+        except json.JSONDecodeError as exc:
+            raise ValueError("live LLM response must be a single JSON object") from exc
         if not isinstance(parsed, dict):
-            raise ValueError("live LLM response must be a JSON object")
+            raise TypeError("live LLM response must be a JSON object")
+        validate_payload(parsed)
         return parsed

testql/contracts/nlp2dsl_conversation/__init__.py

@@ -0,0 +1,46 @@
+"""Runtime binding for the nlp2dsl ConversationFields response contract."""
+
+from __future__ import annotations
+
+import json
+from importlib.resources import files
+from typing import Any
+
+from jsonschema import Draft202012Validator
+
+CONTRACT_VERSION = "1.0.0"
+
+
+def _contract_file(name: str):
+    return files(__package__).joinpath("v1", name)
+
+
+def load_schema() -> dict[str, Any]:
+    return json.loads(
+        _contract_file("conversation-fields.schema.json").read_text(encoding="utf-8")
+    )
+
+
+def validate_payload(payload: object) -> None:
+    validator = Draft202012Validator(load_schema())
+    errors = sorted(validator.iter_errors(payload), key=lambda error: list(error.path))
+    if errors:
+        first = errors[0]
+        location = ".".join(str(part) for part in first.absolute_path) or "$"
+        raise ValueError(
+            f"live LLM response violates ConversationFields v1 at {location}: {first.message}"
+        )
+
+
+def response_format() -> dict[str, Any]:
+    return {
+        "type": "json_schema",
+        "json_schema": {
+            "name": "testql_conversation_fields_v1",
+            "strict": True,
+            "schema": load_schema(),
+        },
+    }
+
+
+__all__ = ["CONTRACT_VERSION", "load_schema", "response_format", "validate_payload"]

testql/contracts/nlp2dsl_conversation/v1/conversation-fields.gbnf

@@ -0,0 +1,9 @@
+root ::= ws "{" ws version ws "," ws fields ws "}" ws
+version ::= "\"contractVersion\"" ws ":" ws "\"1.0.0\""
+fields ::= "\"fields\"" ws ":" ws "{" ws pair (ws "," ws pair)* ws "}"
+pair ::= string ws ":" ws string
+
+string ::= "\"" char* "\""
+char ::= [^"\\\x00-\x1f] | "\\" (["\\/bfnrt] | "u" hex hex hex hex)
+hex ::= [0-9a-fA-F]
+ws ::= [ \t\n\r]*

testql/contracts/nlp2dsl_conversation/v1/conversation-fields.proto

@@ -0,0 +1,8 @@
+syntax = "proto3";
+
+package testql.contracts.nlp2dsl_conversation.v1;
+
+message ConversationFields {
+  string contract_version = 1 [json_name = "contractVersion"];
+  map<string, string> fields = 2;
+}

testql/contracts/nlp2dsl_conversation/v1/conversation-fields.schema.json

@@ -0,0 +1,23 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://testql.dev/contracts/nlp2dsl-conversation/v1/conversation-fields.schema.json",
+  "title": "TestQL ConversationFields v1",
+  "type": "object",
+  "required": ["contractVersion", "fields"],
+  "properties": {
+    "contractVersion": { "const": "1.0.0" },
+    "fields": {
+      "type": "object",
+      "minProperties": 1,
+      "maxProperties": 32,
+      "propertyNames": {
+        "pattern": "^[A-Za-z][A-Za-z0-9_.-]{0,127}$"
+      },
+      "additionalProperties": {
+        "type": "string",
+        "maxLength": 8000
+      }
+    }
+  },
+  "additionalProperties": false
+}

testql/contracts/nlp2dsl_conversation/v1/manifest.json

@@ -0,0 +1,16 @@
+{
+  "contract": "testql.nlp2dsl.ConversationFields",
+  "version": "1.0.0",
+  "boundary": "testql.adapters.nlp2dsl.live_llm.LiveLLMProvider.reply_for",
+  "mediaType": "application/json",
+  "artifacts": {
+    "grammar": "conversation-fields.gbnf",
+    "protobuf": "conversation-fields.proto",
+    "schema": "conversation-fields.schema.json"
+  },
+  "provider": "response_format.json_schema",
+  "runtime": {
+    "parser": "testql.adapters.nlp2dsl.live_llm.LiveLLMProvider._parse_json_object",
+    "validator": "testql.contracts.nlp2dsl_conversation.validate_payload"
+  }
+}

testql/nlp2env/llm.py

@@ -72,7 +72,7 @@ def _extract_json_object(text: str) -> dict[str, Any]:
     except json.JSONDecodeError as exc:
         raise ValueError("LLM response must be a single JSON object") from exc
     if not isinstance(parsed, dict):
-        raise ValueError("LLM response must be a JSON object")
+        raise TypeError("LLM response must be a JSON object")
     validate_tool_call(parsed)
     return parsed
 
@@ -85,9 +85,12 @@ def _openrouter_headers(api_key: str) -> dict[str, str]:
         "Authorization": f"Bearer {api_key}",
         "X-Title": app_name,
     }
-    site_url = os.getenv("OPENROUTER_SITE_URL", "").strip()
-    if site_url:
-        headers["HTTP-Referer"] = site_url
+    app_url = (
+        os.getenv("OPENROUTER_APP_URL", "").strip()
+        or os.getenv("OPENROUTER_SITE_URL", "").strip()
+    )
+    if app_url:
+        headers["HTTP-Referer"] = app_url
     return headers
 
 

tests/fixtures/contracts/nlp2dsl-conversation/v1/invalid-conversation-fields.json

@@ -0,0 +1,6 @@
+{
+  "contractVersion": "1.0.0",
+  "fields": {
+    "recipient": { "address": "test@example.com" }
+  }
+}

tests/fixtures/contracts/nlp2dsl-conversation/v1/valid-conversation-fields.json

@@ -0,0 +1,7 @@
+{
+  "contractVersion": "1.0.0",
+  "fields": {
+    "attachmentPath": "/tmp/invoice.pdf",
+    "recipient": "test@example.com"
+  }
+}

tests/test_conversation_live_llm.py

@@ -7,7 +7,11 @@
 import httpx
 import pytest
 
-from testql.adapters.nlp2dsl import LiveLLMProvider, live_llm_enabled, resolve_llm_provider
+from testql.adapters.nlp2dsl import (
+    LiveLLMProvider,
+    live_llm_enabled,
+    resolve_llm_provider,
+)
 from testql.adapters.nlp2dsl.mock_llm import MockLLMProvider
 from testql.conversation import ConversationRunner
 
@@ -34,10 +38,10 @@ def test_live_without_key_raises(self, monkeypatch):
 
 
 class TestLiveLLMParsing:
-    def test_parse_json_object_strips_fence(self):
+    def test_parse_json_object_rejects_fence(self):
         raw = '```json\n{"attachmentPath": "/tmp/x.pdf"}\n```'
-        parsed = LiveLLMProvider._parse_json_object(raw)
-        assert parsed["attachmentPath"] == "/tmp/x.pdf"
+        with pytest.raises(ValueError, match="single JSON object"):
+            LiveLLMProvider._parse_json_object(raw)
 
 
 @pytest.mark.live_llm
@@ -67,16 +71,22 @@ def test_conversation_runner_with_live_llm_smoke():
 
     from testql.ir import Capture, Nlp2DslStep, TestPlan
 
-    plan = TestPlan(steps=[
-        Nlp2DslStep(endpoint="chatstart", payload={"userId": "live-test"}, captures=[
-            Capture(var_name="conversationId", from_path="conversationId"),
-        ]),
-        Nlp2DslStep(
-            endpoint="chatmessage",
-            payload={"conversationId": "${conversationId}", "text": "ping"},
-            mock_llm={},
-        ),
-    ])
+    plan = TestPlan(
+        steps=[
+            Nlp2DslStep(
+                endpoint="chatstart",
+                payload={"userId": "live-test"},
+                captures=[
+                    Capture(var_name="conversationId", from_path="conversationId"),
+                ],
+            ),
+            Nlp2DslStep(
+                endpoint="chatmessage",
+                payload={"conversationId": "${conversationId}", "text": "ping"},
+                mock_llm={},
+            ),
+        ]
+    )
     runner = ConversationRunner(api_url=nlp2dsl_url, live_llm=True)
     result = runner.run(plan)
     assert any(t.kind == "nlp2dsl" for t in result.turns)