Add SubmitTestPlan action to submit to Test plans to TestRails

agents/test-plan-generator/hackbot_agents/test_plan_generator/__main__.py

@@ -1,4 +1,5 @@
 from hackbot_runtime import HackbotContext, run_async
+from hackbot_runtime.actions.testrail import record_test_plan
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 from .agent import TestPlanGeneratorResult, run_test_plan_generator
@@ -21,7 +22,7 @@ async def main(ctx: HackbotContext) -> TestPlanGeneratorResult:
 
     firefox_path = str(install_firefox_nightly())
 
-    return await run_test_plan_generator(
+    result = await run_test_plan_generator(
         feature_name=inputs.feature_name,
         feature_description=inputs.feature_description,
         test_scope=inputs.test_scope,
@@ -32,6 +33,8 @@ async def main(ctx: HackbotContext) -> TestPlanGeneratorResult:
         log=ctx.log_path,
         verbose=True,
     )
+    record_test_plan(ctx.actions, result.result.model_dump(mode="json"))
+    return result
 
 
 if __name__ == "__main__":

libs/hackbot-runtime/hackbot_runtime/actions/__init__.py

@@ -7,9 +7,15 @@
 claude-sdk adapter is ``hackbot_runtime.actions.claude_sdk.actions_server_for``.
 """
 
-from hackbot_runtime.actions import bugzilla, phabricator
+from hackbot_runtime.actions import bugzilla, phabricator, testrail
 from hackbot_runtime.actions.recorder import ActionsRecorder
 
 ACTIONS_SERVER_NAME = "actions"
 
-__all__ = ["ACTIONS_SERVER_NAME", "ActionsRecorder", "bugzilla", "phabricator"]
+__all__ = [
+    "ACTIONS_SERVER_NAME",
+    "ActionsRecorder",
+    "bugzilla",
+    "phabricator",
+    "testrail",
+]

libs/hackbot-runtime/hackbot_runtime/actions/claude_sdk.py

@@ -15,6 +15,7 @@
 from hackbot_runtime.actions import ACTIONS_SERVER_NAME
 from hackbot_runtime.actions import bugzilla as _bugzilla
 from hackbot_runtime.actions import phabricator as _phabricator
+from hackbot_runtime.actions import testrail as _testrail
 from hackbot_runtime.actions.recorder import ActionsRecorder
 
 
@@ -33,7 +34,7 @@ def actions_server_for(
     """
     if recorder is None:
         recorder = ActionsRecorder(artifacts_dir=fallback_artifacts_dir)
-    tools = _bugzilla.TOOLS + _phabricator.TOOLS
+    tools = _bugzilla.TOOLS + _phabricator.TOOLS + _testrail.TOOLS
     if types is not None:
         wanted = set(types)
         tools = [t for t in tools if t.dotted in wanted]

libs/hackbot-runtime/hackbot_runtime/actions/handlers/registry.py

@@ -12,6 +12,7 @@
     SubmitPatchHandler,
     UpdatePatchHandler,
 )
+from hackbot_runtime.actions.handlers.testrail_handler import SubmitTestPlanHandler
 
 # Maps a recorded action's dotted `type` to the handler that applies it.
 # Adding a new action type later is a one-line addition here — the dispatch
@@ -24,6 +25,7 @@
     "phabricator.submit_patch": SubmitPatchHandler(),
     "phabricator.update_patch": UpdatePatchHandler(),
     "phabricator.add_comment": PhabricatorAddCommentHandler(),
+    "testrail.submit_test_plan": SubmitTestPlanHandler(),
 }
 
 

libs/hackbot-runtime/hackbot_runtime/actions/handlers/testrail_handler.py

@@ -0,0 +1,139 @@
+"""Apply-side TestRail action for generated test cases."""
+
+from __future__ import annotations
+
+import logging
+from functools import lru_cache
+from typing import Any
+
+from testrail_client import TestRailClient
+
+from hackbot_runtime.actions.handlers.base import ActionResult, ApplyContext
+
+log = logging.getLogger(__name__)
+
+_CASE_TYPE_NAME = "Functional"
+_CASE_TEMPLATE_NAME = "Test Case (Steps)"
+_CASE_LABEL = "AI Generated"
+_SECTION_NAME = "Test Cases"
+
+
+@lru_cache(maxsize=1)
+def _client() -> TestRailClient:
+    return TestRailClient()
+
+
+def _require_id(response: object, object_name: str) -> int:
+    if not isinstance(response, dict):
+        raise RuntimeError(f"TestRail did not return a {object_name} object")
+    value = response.get("id")
+    try:
+        return int(value)
+    except (TypeError, ValueError) as exc:
+        raise RuntimeError(
+            f"TestRail did not return an id for the created {object_name}"
+        ) from exc
+
+
+async def _resolve_case_type_id(client: TestRailClient) -> int:
+    response = await client.get_case_types()
+    case_types = (
+        response.get("case_types", []) if isinstance(response, dict) else response
+    )
+    for case_type in case_types:
+        if (
+            isinstance(case_type, dict)
+            and str(case_type.get("name", "")).strip().casefold()
+            == _CASE_TYPE_NAME.casefold()
+        ):
+            return _require_id(case_type, "case type")
+    raise RuntimeError(f'TestRail has no case type named "{_CASE_TYPE_NAME}"')
+
+
+async def _resolve_template_id(client: TestRailClient) -> int:
+    response = await client.get_templates()
+    templates = (
+        response.get("templates", []) if isinstance(response, dict) else response
+    )
+    for template in templates:
+        if (
+            isinstance(template, dict)
+            and str(template.get("name", "")).strip().casefold()
+            == _CASE_TEMPLATE_NAME.casefold()
+        ):
+            return _require_id(template, "template")
+    raise RuntimeError(f'TestRail has no template named "{_CASE_TEMPLATE_NAME}"')
+
+
+def _case_payload(
+    test_case: dict[str, Any],
+    case_type_id: int,
+    template_id: int,
+) -> dict[str, Any]:
+    steps = [str(step) for step in test_case.get("steps", [])]
+    payload: dict[str, Any] = {
+        "title": test_case["title"],
+        "type_id": case_type_id,
+        "template_id": template_id,
+        "labels": [_CASE_LABEL],
+        "custom_steps_separated": [{"content": step, "expected": ""} for step in steps],
+    }
+    if test_case.get("preconditions"):
+        payload["custom_preconds"] = test_case["preconditions"]
+    return payload
+
+
+class SubmitTestPlanHandler:
+    async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult:
+        feature = str(params.get("feature") or "").strip()
+        test_cases = params.get("generated_test_cases") or []
+
+        if not feature:
+            return ActionResult.failed("TestRail submission requires a feature name")
+        if not test_cases:
+            return ActionResult.failed("TestRail submission requires test cases")
+
+        try:
+            client = _client()
+            case_type_id = await _resolve_case_type_id(client)
+            template_id = await _resolve_template_id(client)
+            suite_name = f"[Hackbot] - {feature}"
+            suite_id = _require_id(
+                await client.add_suite(suite_name),
+                "suite",
+            )
+            section_id = _require_id(
+                await client.add_section(suite_id, _SECTION_NAME),
+                "section",
+            )
+
+            created_case_ids: dict[int, int] = {}
+            for test_case in test_cases:
+                generated_id = int(test_case["id"])
+                case_id = _require_id(
+                    await client.add_case(
+                        section_id,
+                        _case_payload(
+                            test_case,
+                            case_type_id,
+                            template_id,
+                        ),
+                    ),
+                    "case",
+                )
+                created_case_ids[generated_id] = case_id
+
+        except Exception as exc:
+            log.exception(
+                "Failed to submit test plan to TestRail for run %s", ctx.run_id
+            )
+            return ActionResult.failed(str(exc))
+
+        return ActionResult.ok(
+            {
+                "suite_id": suite_id,
+                "url": client.suite_url(suite_id),
+                "section_id": section_id,
+                "case_ids": list(created_case_ids.values()),
+            }
+        )

libs/hackbot-runtime/hackbot_runtime/actions/testrail.py

@@ -0,0 +1,122 @@
+"""TestRail-domain recordable actions.
+
+The test plan generator records this action deterministically after its
+structured result has been validated. The external TestRail mutation still
+happens only in the apply side handler.
+"""
+
+from __future__ import annotations
+
+from typing import Annotated, Any
+
+from agent_tools.registry import ToolError, tool, tools_in
+from pydantic import BaseModel, Field, ValidationError, field_validator
+
+from hackbot_runtime.actions.recorder import ActionsRecorder
+
+ACTION_TYPE = "testrail.submit_test_plan"
+
+
+class TestRailCaseInput(BaseModel):
+    id: int
+    title: str = Field(description="TestRail test case title.")
+    context: str | None = Field(
+        default=None, description="Optional context for where this case applies."
+    )
+    preconditions: str | None = Field(
+        default=None, description="Optional setup required before running this case."
+    )
+    steps: list[str] = Field(
+        min_length=1, description="Ordered steps a QA engineer should follow."
+    )
+
+    @field_validator("title")
+    @classmethod
+    def title_must_not_be_blank(cls, value: str) -> str:
+        value = value.strip()
+        if not value:
+            raise ValueError("title must not be blank")
+        return value
+
+    @field_validator("steps")
+    @classmethod
+    def steps_must_not_be_blank(cls, value: list[str]) -> list[str]:
+        steps = [step.strip() for step in value]
+        if any(not step for step in steps):
+            raise ValueError("steps must not contain blank items")
+        return steps
+
+
+class SubmitTestPlanInput(BaseModel):
+    feature: str = Field(description="Feature covered by the generated test cases.")
+    generated_test_cases: list[TestRailCaseInput] = Field(
+        min_length=1, description="Generated test cases to upload to TestRail."
+    )
+
+    @field_validator("feature")
+    @classmethod
+    def feature_must_not_be_blank(cls, value: str) -> str:
+        value = value.strip()
+        if not value:
+            raise ValueError("feature must not be blank")
+        return value
+
+
+def _confirm(recorder: ActionsRecorder, action_type: str) -> str:
+    return f"Recorded {action_type} (#{len(recorder.actions) - 1})."
+
+
+def _validated_params(feature: str, generated_test_cases: list[Any]) -> dict[str, Any]:
+    try:
+        validated = SubmitTestPlanInput.model_validate(
+            {"feature": feature, "generated_test_cases": generated_test_cases}
+        )
+    except ValidationError as exc:
+        raise ToolError(
+            "invalid TestRail submission",
+            payload={"error": "invalid TestRail submission", "details": exc.errors()},
+        ) from exc
+    return validated.model_dump(mode="json")
+
+
+@tool
+async def submit_test_plan(
+    recorder: ActionsRecorder,
+    feature: Annotated[
+        str,
+        Field(
+            description=(
+                "Feature name for the new TestRail suite. The apply step always "
+                "creates a new suite for this feature."
+            )
+        ),
+    ],
+    generated_test_cases: Annotated[
+        list[TestRailCaseInput],
+        Field(description="Generated test cases to upload together to TestRail."),
+    ],
+) -> str:
+    """Record a generated test plan for deferred TestRail submission.
+
+    This records one reviewed action. The apply step creates a new TestRail
+    suite, creates a section in it, and uploads all supplied test cases.
+    Nothing is sent to TestRail during the agent run.
+    """
+    params = _validated_params(feature, generated_test_cases)
+    recorder.record(ACTION_TYPE, params)
+    return _confirm(recorder, ACTION_TYPE)
+
+
+def record_test_plan(
+    recorder: ActionsRecorder,
+    test_plan: dict[str, Any],
+) -> dict:
+    """Record validated generated cases for deferred TestRail submission."""
+    params = _validated_params(
+        str(test_plan.get("feature") or ""),
+        list(test_plan.get("generated_test_cases") or []),
+    )
+    return recorder.record(ACTION_TYPE, params)
+
+
+TOOLS = tools_in(__name__)

libs/hackbot-runtime/pyproject.toml

@@ -11,6 +11,7 @@ dependencies = [
     "async-lru>=2.0.0",
     "agent-tools",
     "phabricator-client",
+    "testrail-client",
     "weave>=0.53.1"
 ]
 
@@ -24,6 +25,7 @@ phabricator = ["MozPhab==2.15.3"]
 [tool.uv.sources]
 agent-tools = { workspace = true }
 phabricator-client = { workspace = true }
+testrail-client = { workspace = true }
 
 [build-system]
 requires = ["hatchling"]

libs/hackbot-runtime/tests/test_claude_sdk.py

@@ -10,6 +10,7 @@
     "bugzilla.add_comment",
     "bugzilla.add_attachment",
     "bugzilla.create_bug",
+    "testrail.submit_test_plan",
 ]
 
 
@@ -53,6 +54,7 @@ async def test_lists_expected_tools_without_recorder():
         "bugzilla_add_comment",
         "bugzilla_add_attachment",
         "bugzilla_create_bug",
+        "testrail_submit_test_plan",
     }
     for t in tools:
         assert "recorder" not in t.inputSchema.get("properties", {})
@@ -105,3 +107,11 @@ async def test_actions_server_for_exposes_selected_tools():
     _, config = actions_server_for(ActionsRecorder(), types=["bugzilla.update_bug"])
     tools = await _list(config["instance"])
     assert {t.name for t in tools} == {"bugzilla_update_bug"}
+
+
+async def test_actions_server_exposes_selected_testrail_tool():
+    _, config = actions_server_for(
+        ActionsRecorder(), types=["testrail.submit_test_plan"]
+    )
+    tools = await _list(config["instance"])
+    assert {t.name for t in tools} == {"testrail_submit_test_plan"}