diff --git a/agents/test-plan-generator/hackbot_agents/test_plan_generator/__main__.py b/agents/test-plan-generator/hackbot_agents/test_plan_generator/__main__.py index af67eac60b..aaae133ca7 100644 --- a/agents/test-plan-generator/hackbot_agents/test_plan_generator/__main__.py +++ b/agents/test-plan-generator/hackbot_agents/test_plan_generator/__main__.py @@ -1,4 +1,5 @@ from hackbot_runtime import HackbotContext, run_async +from hackbot_runtime.actions.testrail import record_test_plan from pydantic_settings import BaseSettings, SettingsConfigDict from .agent import TestPlanGeneratorResult, run_test_plan_generator @@ -21,7 +22,7 @@ async def main(ctx: HackbotContext) -> TestPlanGeneratorResult: firefox_path = str(install_firefox_nightly()) - return await run_test_plan_generator( + result = await run_test_plan_generator( feature_name=inputs.feature_name, feature_description=inputs.feature_description, test_scope=inputs.test_scope, @@ -32,6 +33,8 @@ async def main(ctx: HackbotContext) -> TestPlanGeneratorResult: log=ctx.log_path, verbose=True, ) + record_test_plan(ctx.actions, result.result.model_dump(mode="json")) + return result if __name__ == "__main__": diff --git a/libs/hackbot-runtime/hackbot_runtime/actions/__init__.py b/libs/hackbot-runtime/hackbot_runtime/actions/__init__.py index e38385139b..a620c7b37c 100644 --- a/libs/hackbot-runtime/hackbot_runtime/actions/__init__.py +++ b/libs/hackbot-runtime/hackbot_runtime/actions/__init__.py @@ -7,9 +7,15 @@ claude-sdk adapter is ``hackbot_runtime.actions.claude_sdk.actions_server_for``. """ -from hackbot_runtime.actions import bugzilla, phabricator +from hackbot_runtime.actions import bugzilla, phabricator, testrail from hackbot_runtime.actions.recorder import ActionsRecorder ACTIONS_SERVER_NAME = "actions" -__all__ = ["ACTIONS_SERVER_NAME", "ActionsRecorder", "bugzilla", "phabricator"] +__all__ = [ + "ACTIONS_SERVER_NAME", + "ActionsRecorder", + "bugzilla", + "phabricator", + "testrail", +] diff --git a/libs/hackbot-runtime/hackbot_runtime/actions/claude_sdk.py b/libs/hackbot-runtime/hackbot_runtime/actions/claude_sdk.py index d52c73e778..a617ac6147 100644 --- a/libs/hackbot-runtime/hackbot_runtime/actions/claude_sdk.py +++ b/libs/hackbot-runtime/hackbot_runtime/actions/claude_sdk.py @@ -15,6 +15,7 @@ from hackbot_runtime.actions import ACTIONS_SERVER_NAME from hackbot_runtime.actions import bugzilla as _bugzilla from hackbot_runtime.actions import phabricator as _phabricator +from hackbot_runtime.actions import testrail as _testrail from hackbot_runtime.actions.recorder import ActionsRecorder @@ -33,7 +34,7 @@ def actions_server_for( """ if recorder is None: recorder = ActionsRecorder(artifacts_dir=fallback_artifacts_dir) - tools = _bugzilla.TOOLS + _phabricator.TOOLS + tools = _bugzilla.TOOLS + _phabricator.TOOLS + _testrail.TOOLS if types is not None: wanted = set(types) tools = [t for t in tools if t.dotted in wanted] diff --git a/libs/hackbot-runtime/hackbot_runtime/actions/handlers/registry.py b/libs/hackbot-runtime/hackbot_runtime/actions/handlers/registry.py index 6eee8c5e4f..f9cd20985f 100644 --- a/libs/hackbot-runtime/hackbot_runtime/actions/handlers/registry.py +++ b/libs/hackbot-runtime/hackbot_runtime/actions/handlers/registry.py @@ -12,6 +12,7 @@ SubmitPatchHandler, UpdatePatchHandler, ) +from hackbot_runtime.actions.handlers.testrail_handler import SubmitTestPlanHandler # Maps a recorded action's dotted `type` to the handler that applies it. # Adding a new action type later is a one-line addition here — the dispatch @@ -24,6 +25,7 @@ "phabricator.submit_patch": SubmitPatchHandler(), "phabricator.update_patch": UpdatePatchHandler(), "phabricator.add_comment": PhabricatorAddCommentHandler(), + "testrail.submit_test_plan": SubmitTestPlanHandler(), } diff --git a/libs/hackbot-runtime/hackbot_runtime/actions/handlers/testrail_handler.py b/libs/hackbot-runtime/hackbot_runtime/actions/handlers/testrail_handler.py new file mode 100644 index 0000000000..bdddf00382 --- /dev/null +++ b/libs/hackbot-runtime/hackbot_runtime/actions/handlers/testrail_handler.py @@ -0,0 +1,139 @@ +"""Apply-side TestRail action for generated test cases.""" + +from __future__ import annotations + +import logging +from functools import lru_cache +from typing import Any + +from testrail_client import TestRailClient + +from hackbot_runtime.actions.handlers.base import ActionResult, ApplyContext + +log = logging.getLogger(__name__) + +_CASE_TYPE_NAME = "Functional" +_CASE_TEMPLATE_NAME = "Test Case (Steps)" +_CASE_LABEL = "AI Generated" +_SECTION_NAME = "Test Cases" + + +@lru_cache(maxsize=1) +def _client() -> TestRailClient: + return TestRailClient() + + +def _require_id(response: object, object_name: str) -> int: + if not isinstance(response, dict): + raise RuntimeError(f"TestRail did not return a {object_name} object") + value = response.get("id") + try: + return int(value) + except (TypeError, ValueError) as exc: + raise RuntimeError( + f"TestRail did not return an id for the created {object_name}" + ) from exc + + +async def _resolve_case_type_id(client: TestRailClient) -> int: + response = await client.get_case_types() + case_types = ( + response.get("case_types", []) if isinstance(response, dict) else response + ) + for case_type in case_types: + if ( + isinstance(case_type, dict) + and str(case_type.get("name", "")).strip().casefold() + == _CASE_TYPE_NAME.casefold() + ): + return _require_id(case_type, "case type") + raise RuntimeError(f'TestRail has no case type named "{_CASE_TYPE_NAME}"') + + +async def _resolve_template_id(client: TestRailClient) -> int: + response = await client.get_templates() + templates = ( + response.get("templates", []) if isinstance(response, dict) else response + ) + for template in templates: + if ( + isinstance(template, dict) + and str(template.get("name", "")).strip().casefold() + == _CASE_TEMPLATE_NAME.casefold() + ): + return _require_id(template, "template") + raise RuntimeError(f'TestRail has no template named "{_CASE_TEMPLATE_NAME}"') + + +def _case_payload( + test_case: dict[str, Any], + case_type_id: int, + template_id: int, +) -> dict[str, Any]: + steps = [str(step) for step in test_case.get("steps", [])] + payload: dict[str, Any] = { + "title": test_case["title"], + "type_id": case_type_id, + "template_id": template_id, + "labels": [_CASE_LABEL], + "custom_steps_separated": [{"content": step, "expected": ""} for step in steps], + } + if test_case.get("preconditions"): + payload["custom_preconds"] = test_case["preconditions"] + return payload + + +class SubmitTestPlanHandler: + async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult: + feature = str(params.get("feature") or "").strip() + test_cases = params.get("generated_test_cases") or [] + + if not feature: + return ActionResult.failed("TestRail submission requires a feature name") + if not test_cases: + return ActionResult.failed("TestRail submission requires test cases") + + try: + client = _client() + case_type_id = await _resolve_case_type_id(client) + template_id = await _resolve_template_id(client) + suite_name = f"[Hackbot] - {feature}" + suite_id = _require_id( + await client.add_suite(suite_name), + "suite", + ) + section_id = _require_id( + await client.add_section(suite_id, _SECTION_NAME), + "section", + ) + + created_case_ids: dict[int, int] = {} + for test_case in test_cases: + generated_id = int(test_case["id"]) + case_id = _require_id( + await client.add_case( + section_id, + _case_payload( + test_case, + case_type_id, + template_id, + ), + ), + "case", + ) + created_case_ids[generated_id] = case_id + + except Exception as exc: + log.exception( + "Failed to submit test plan to TestRail for run %s", ctx.run_id + ) + return ActionResult.failed(str(exc)) + + return ActionResult.ok( + { + "suite_id": suite_id, + "url": client.suite_url(suite_id), + "section_id": section_id, + "case_ids": list(created_case_ids.values()), + } + ) diff --git a/libs/hackbot-runtime/hackbot_runtime/actions/testrail.py b/libs/hackbot-runtime/hackbot_runtime/actions/testrail.py new file mode 100644 index 0000000000..0941b1cc04 --- /dev/null +++ b/libs/hackbot-runtime/hackbot_runtime/actions/testrail.py @@ -0,0 +1,122 @@ +"""TestRail-domain recordable actions. + +The test plan generator records this action deterministically after its +structured result has been validated. The external TestRail mutation still +happens only in the apply side handler. +""" + +from __future__ import annotations + +from typing import Annotated, Any + +from agent_tools.registry import ToolError, tool, tools_in +from pydantic import BaseModel, Field, ValidationError, field_validator + +from hackbot_runtime.actions.recorder import ActionsRecorder + +ACTION_TYPE = "testrail.submit_test_plan" + + +class TestRailCaseInput(BaseModel): + id: int + title: str = Field(description="TestRail test case title.") + context: str | None = Field( + default=None, description="Optional context for where this case applies." + ) + preconditions: str | None = Field( + default=None, description="Optional setup required before running this case." + ) + steps: list[str] = Field( + min_length=1, description="Ordered steps a QA engineer should follow." + ) + + @field_validator("title") + @classmethod + def title_must_not_be_blank(cls, value: str) -> str: + value = value.strip() + if not value: + raise ValueError("title must not be blank") + return value + + @field_validator("steps") + @classmethod + def steps_must_not_be_blank(cls, value: list[str]) -> list[str]: + steps = [step.strip() for step in value] + if any(not step for step in steps): + raise ValueError("steps must not contain blank items") + return steps + + +class SubmitTestPlanInput(BaseModel): + feature: str = Field(description="Feature covered by the generated test cases.") + generated_test_cases: list[TestRailCaseInput] = Field( + min_length=1, description="Generated test cases to upload to TestRail." + ) + + @field_validator("feature") + @classmethod + def feature_must_not_be_blank(cls, value: str) -> str: + value = value.strip() + if not value: + raise ValueError("feature must not be blank") + return value + + +def _confirm(recorder: ActionsRecorder, action_type: str) -> str: + return f"Recorded {action_type} (#{len(recorder.actions) - 1})." + + +def _validated_params(feature: str, generated_test_cases: list[Any]) -> dict[str, Any]: + try: + validated = SubmitTestPlanInput.model_validate( + {"feature": feature, "generated_test_cases": generated_test_cases} + ) + except ValidationError as exc: + raise ToolError( + "invalid TestRail submission", + payload={"error": "invalid TestRail submission", "details": exc.errors()}, + ) from exc + return validated.model_dump(mode="json") + + +@tool +async def submit_test_plan( + recorder: ActionsRecorder, + feature: Annotated[ + str, + Field( + description=( + "Feature name for the new TestRail suite. The apply step always " + "creates a new suite for this feature." + ) + ), + ], + generated_test_cases: Annotated[ + list[TestRailCaseInput], + Field(description="Generated test cases to upload together to TestRail."), + ], +) -> str: + """Record a generated test plan for deferred TestRail submission. + + This records one reviewed action. The apply step creates a new TestRail + suite, creates a section in it, and uploads all supplied test cases. + Nothing is sent to TestRail during the agent run. + """ + params = _validated_params(feature, generated_test_cases) + recorder.record(ACTION_TYPE, params) + return _confirm(recorder, ACTION_TYPE) + + +def record_test_plan( + recorder: ActionsRecorder, + test_plan: dict[str, Any], +) -> dict: + """Record validated generated cases for deferred TestRail submission.""" + params = _validated_params( + str(test_plan.get("feature") or ""), + list(test_plan.get("generated_test_cases") or []), + ) + return recorder.record(ACTION_TYPE, params) + + +TOOLS = tools_in(__name__) diff --git a/libs/hackbot-runtime/pyproject.toml b/libs/hackbot-runtime/pyproject.toml index 1007a10173..f6aac5835d 100644 --- a/libs/hackbot-runtime/pyproject.toml +++ b/libs/hackbot-runtime/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "async-lru>=2.0.0", "agent-tools", "phabricator-client", + "testrail-client", "weave>=0.53.1" ] @@ -24,6 +25,7 @@ phabricator = ["MozPhab==2.15.3"] [tool.uv.sources] agent-tools = { workspace = true } phabricator-client = { workspace = true } +testrail-client = { workspace = true } [build-system] requires = ["hatchling"] diff --git a/libs/hackbot-runtime/tests/test_claude_sdk.py b/libs/hackbot-runtime/tests/test_claude_sdk.py index 7f665ca436..2c59444588 100644 --- a/libs/hackbot-runtime/tests/test_claude_sdk.py +++ b/libs/hackbot-runtime/tests/test_claude_sdk.py @@ -10,6 +10,7 @@ "bugzilla.add_comment", "bugzilla.add_attachment", "bugzilla.create_bug", + "testrail.submit_test_plan", ] @@ -53,6 +54,7 @@ async def test_lists_expected_tools_without_recorder(): "bugzilla_add_comment", "bugzilla_add_attachment", "bugzilla_create_bug", + "testrail_submit_test_plan", } for t in tools: assert "recorder" not in t.inputSchema.get("properties", {}) @@ -105,3 +107,11 @@ async def test_actions_server_for_exposes_selected_tools(): _, config = actions_server_for(ActionsRecorder(), types=["bugzilla.update_bug"]) tools = await _list(config["instance"]) assert {t.name for t in tools} == {"bugzilla_update_bug"} + + +async def test_actions_server_exposes_selected_testrail_tool(): + _, config = actions_server_for( + ActionsRecorder(), types=["testrail.submit_test_plan"] + ) + tools = await _list(config["instance"]) + assert {t.name for t in tools} == {"testrail_submit_test_plan"} diff --git a/libs/hackbot-runtime/tests/test_testrail_action.py b/libs/hackbot-runtime/tests/test_testrail_action.py new file mode 100644 index 0000000000..d059911863 --- /dev/null +++ b/libs/hackbot-runtime/tests/test_testrail_action.py @@ -0,0 +1,51 @@ +import pytest +from agent_tools.registry import ToolError +from hackbot_runtime.actions import ActionsRecorder, testrail +from hackbot_runtime.actions.handlers import get_handler +from hackbot_runtime.actions.handlers.testrail_handler import SubmitTestPlanHandler +from hackbot_runtime.actions.testrail import ACTION_TYPE + + +def _cases(): + return [ + { + "id": 1, + "title": "The PDF opens", + "context": "content", + "preconditions": "A PDF is available.", + "steps": ["Open the PDF"], + } + ] + + +async def test_submit_test_plan_tool_records_deferred_action(): + recorder = ActionsRecorder() + + message = await testrail.submit_test_plan( + recorder, feature="Feature", generated_test_cases=_cases() + ) + + assert message == "Recorded testrail.submit_test_plan (#0)." + assert recorder.actions[0]["type"] == ACTION_TYPE + assert recorder.actions[0]["params"] == { + "feature": "Feature", + "generated_test_cases": _cases(), + } + + +async def test_submit_test_plan_tool_rejects_invalid_input(): + recorder = ActionsRecorder() + + with pytest.raises(ToolError) as exc: + await testrail.submit_test_plan( + recorder, + feature=" ", + generated_test_cases=[{"id": 1, "title": "Case", "steps": []}], + ) + + assert "invalid TestRail submission" in str(exc.value) + assert recorder.actions == [] + + +def test_submit_test_plan_handler_is_registered(): + assert isinstance(get_handler(ACTION_TYPE), SubmitTestPlanHandler) diff --git a/libs/hackbot-runtime/tests/test_testrail_handler.py b/libs/hackbot-runtime/tests/test_testrail_handler.py new file mode 100644 index 0000000000..fc2599e3ce --- /dev/null +++ b/libs/hackbot-runtime/tests/test_testrail_handler.py @@ -0,0 +1,232 @@ +"""Tests for the apply-side TestRail action handler.""" + +from hackbot_runtime.actions.handlers import ApplyContext, testrail_handler + + +def _ctx(): + async def download(_key): + raise AssertionError("TestRail submissions do not use artifacts") + + return ApplyContext(run_id="run-1", download_artifact=download) + + +def _plan(): + return { + "feature": "PDF Improvements", + "generated_test_cases": [ + { + "id": 1, + "title": "The PDF opens", + "context": "content", + "preconditions": "A PDF is available.", + "steps": ["Open the PDF", "Select some text"], + }, + { + "id": 2, + "title": "The toolbar remains available", + "context": "chrome", + "preconditions": None, + "steps": ["Open the toolbar"], + }, + ], + "results": [ + { + "id": 1, + "status": "passed", + "summary": "The PDF behaved as expected.", + "failure_reason": None, + "step_results": [ + { + "step_number": 1, + "status": "passed", + "observation": "The PDF opened.", + "failure_reason": None, + }, + { + "step_number": 2, + "status": "passed", + "observation": "Text was selected.", + "failure_reason": None, + }, + ], + }, + { + "id": 2, + "status": "unsuitable", + "summary": "The toolbar could not be inspected.", + "failure_reason": "No available tool can inspect it.", + "step_results": [ + { + "step_number": 1, + "status": "not_run", + "observation": "Not run.", + "failure_reason": None, + } + ], + }, + ], + "summary": "One passed and one was unsuitable.", + } + + +class _FakeClient: + def __init__(self, responses=None): + self.calls = [] + self._responses = list( + responses + or [ + [{"id": 6, "name": "Functional"}], + [{"id": 2, "name": "Test Case (Steps)"}], + {"id": 10}, + {"id": 20}, + {"id": 101}, + {"id": 102}, + ] + ) + + def _next(self): + value = self._responses.pop(0) + if isinstance(value, Exception): + raise value + return value + + async def get_case_types(self): + self.calls.append(("get_case_types",)) + return self._next() + + async def get_templates(self): + self.calls.append(("get_templates",)) + return self._next() + + async def add_suite(self, name): + self.calls.append(("add_suite", name)) + return self._next() + + async def add_section(self, suite_id, name): + self.calls.append(("add_section", suite_id, name)) + return self._next() + + async def add_case(self, section_id, payload): + self.calls.append(("add_case", section_id, payload)) + return self._next() + + def suite_url(self, suite_id): + return f"https://testrail.example/index.php?/suites/view/{suite_id}" + + +async def test_submit_test_plan_creates_suite_section_and_cases(monkeypatch): + client = _FakeClient() + monkeypatch.setattr(testrail_handler, "_client", lambda: client) + + result = await testrail_handler.SubmitTestPlanHandler().apply(_plan(), _ctx()) + + assert result.status == "applied" + assert result.result == { + "suite_id": 10, + "url": "https://testrail.example/index.php?/suites/view/10", + "section_id": 20, + "case_ids": [101, 102], + } + + assert client.calls[0] == ("get_case_types",) + assert client.calls[1] == ("get_templates",) + assert client.calls[2] == ("add_suite", "[Hackbot] - PDF Improvements") + assert client.calls[3] == ("add_section", 10, "Test Cases") + + first_case = client.calls[4] + assert first_case[0:2] == ("add_case", 20) + assert first_case[2] == { + "title": "The PDF opens", + "type_id": 6, + "template_id": 2, + "labels": ["AI Generated"], + "custom_preconds": "A PDF is available.", + "custom_steps_separated": [ + {"content": "Open the PDF", "expected": ""}, + {"content": "Select some text", "expected": ""}, + ], + } + + +async def test_submit_test_plan_reports_api_failure(monkeypatch): + client = _FakeClient(responses=[RuntimeError("TestRail is unavailable")]) + monkeypatch.setattr(testrail_handler, "_client", lambda: client) + + result = await testrail_handler.SubmitTestPlanHandler().apply(_plan(), _ctx()) + + assert result.status == "failed" + assert result.error == "TestRail is unavailable" + + +async def test_submit_test_plan_requires_feature(monkeypatch): + client = _FakeClient() + monkeypatch.setattr(testrail_handler, "_client", lambda: client) + plan = _plan() + plan["feature"] = " " + + result = await testrail_handler.SubmitTestPlanHandler().apply(plan, _ctx()) + + assert result.status == "failed" + assert result.error == "TestRail submission requires a feature name" + assert client.calls == [] + + +async def test_submit_test_plan_requires_cases(monkeypatch): + client = _FakeClient() + monkeypatch.setattr(testrail_handler, "_client", lambda: client) + plan = _plan() + plan["generated_test_cases"] = [] + + result = await testrail_handler.SubmitTestPlanHandler().apply(plan, _ctx()) + + assert result.status == "failed" + assert result.error == "TestRail submission requires test cases" + assert client.calls == [] + + +async def test_resolve_case_type_id_by_name(): + client = _FakeClient( + responses=[ + [ + {"id": 3, "name": "Automated"}, + {"id": 12, "name": " functional "}, + ] + ] + ) + + assert await testrail_handler._resolve_case_type_id(client) == 12 + + +async def test_resolve_case_type_id_requires_functional_type(): + client = _FakeClient(responses=[[{"id": 3, "name": "Automated"}]]) + + try: + await testrail_handler._resolve_case_type_id(client) + except RuntimeError as exc: + assert str(exc) == 'TestRail has no case type named "Functional"' + else: + raise AssertionError("missing Functional case type did not fail") + + +async def test_resolve_template_id_by_name(): + client = _FakeClient( + responses=[ + [ + {"id": 1, "name": "Test Case (Text)"}, + {"id": 2, "name": " test case (steps) "}, + ] + ] + ) + + assert await testrail_handler._resolve_template_id(client) == 2 + + +async def test_resolve_template_id_requires_steps_template(): + client = _FakeClient(responses=[[{"id": 1, "name": "Test Case (Text)"}]]) + + try: + await testrail_handler._resolve_template_id(client) + except RuntimeError as exc: + assert str(exc) == 'TestRail has no template named "Test Case (Steps)"' + else: + raise AssertionError("missing Test Case (Steps) template did not fail") diff --git a/libs/testrail-client/pyproject.toml b/libs/testrail-client/pyproject.toml new file mode 100644 index 0000000000..6a773eeae4 --- /dev/null +++ b/libs/testrail-client/pyproject.toml @@ -0,0 +1,23 @@ +[project] +name = "testrail-client" +version = "0.1.0" +description = "Small shared TestRail API client (httpx-based)" +requires-python = ">=3.12" +dependencies = [ + "httpx>=0.26.0", + "pydantic-settings>=2.1.0", +] + +[project.optional-dependencies] +dev = ["pytest>=8.0.0", "pytest-asyncio>=0.23.0"] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["testrail_client"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] diff --git a/libs/testrail-client/testrail_client/__init__.py b/libs/testrail-client/testrail_client/__init__.py new file mode 100644 index 0000000000..d4b4d8b463 --- /dev/null +++ b/libs/testrail-client/testrail_client/__init__.py @@ -0,0 +1,4 @@ +from testrail_client.client import TestRailClient +from testrail_client.config import TestRailSettings + +__all__ = ["TestRailClient", "TestRailSettings"] diff --git a/libs/testrail-client/testrail_client/client.py b/libs/testrail-client/testrail_client/client.py new file mode 100644 index 0000000000..c8634d8cbb --- /dev/null +++ b/libs/testrail-client/testrail_client/client.py @@ -0,0 +1,68 @@ +"""Small shared TestRail API client.""" + +from __future__ import annotations + +from typing import Any + +import httpx + +from testrail_client.config import TestRailSettings + + +class TestRailClient: + def __init__(self, settings: TestRailSettings | None = None) -> None: + self.settings = settings or TestRailSettings.from_env() + + @property + def base_url(self) -> str: + return self.settings.url.rstrip("/") + + def suite_url(self, suite_id: int) -> str: + return f"{self.base_url}/index.php?/suites/view/{suite_id}" + + def _api_url(self, endpoint: str) -> str: + return f"{self.base_url}/index.php?/api/v2/{endpoint.lstrip('/')}" + + async def request( + self, method: str, endpoint: str, data: dict[str, Any] | None = None + ) -> dict[str, Any] | list[Any]: + async with httpx.AsyncClient(timeout=self.settings.timeout_seconds) as client: + response = await client.request( + method, + self._api_url(endpoint), + auth=(self.settings.username, self.settings.api_key), + json=data, + headers={"Content-Type": "application/json"}, + ) + response.raise_for_status() + if not response.content: + return {} + result = response.json() + if not isinstance(result, (dict, list)): + raise RuntimeError("TestRail returned an unexpected response") + return result + + async def get_case_types(self) -> dict[str, Any] | list[Any]: + return await self.request("GET", "get_case_types") + + async def get_templates(self) -> dict[str, Any] | list[Any]: + return await self.request("GET", f"get_templates/{self.settings.project_id}") + + async def add_suite(self, name: str) -> dict[str, Any] | list[Any]: + return await self.request( + "POST", + f"add_suite/{self.settings.project_id}", + {"name": name}, + ) + + async def add_section(self, suite_id: int, name: str) -> dict[str, Any] | list[Any]: + return await self.request( + "POST", + f"add_section/{self.settings.project_id}", + {"suite_id": suite_id, "name": name}, + ) + + async def add_case( + self, section_id: int, payload: dict[str, Any] + ) -> dict[str, Any] | list[Any]: + return await self.request("POST", f"add_case/{section_id}", payload) diff --git a/libs/testrail-client/testrail_client/config.py b/libs/testrail-client/testrail_client/config.py new file mode 100644 index 0000000000..fa534d7940 --- /dev/null +++ b/libs/testrail-client/testrail_client/config.py @@ -0,0 +1,22 @@ +"""Configuration for :class:`TestRailClient`.""" + +from __future__ import annotations + +from pydantic import BaseModel, Field +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class TestRailSettings(BaseModel): + username: str = Field(min_length=1) + api_key: str = Field(min_length=1) + project_id: int + url: str = "https://mozilla.testrail.io" + timeout_seconds: int = 30 + + @classmethod + def from_env(cls) -> TestRailSettings: + return _TestRailEnvSettings() + + +class _TestRailEnvSettings(TestRailSettings, BaseSettings): + model_config = SettingsConfigDict(env_prefix="TESTRAIL_", extra="ignore") diff --git a/libs/testrail-client/tests/test_client.py b/libs/testrail-client/tests/test_client.py new file mode 100644 index 0000000000..850cac30fd --- /dev/null +++ b/libs/testrail-client/tests/test_client.py @@ -0,0 +1,191 @@ +"""Tests for the shared TestRail client.""" + +import httpx +import pytest +from pydantic import ValidationError +from testrail_client import TestRailClient as Client +from testrail_client import TestRailSettings as Settings +from testrail_client import client as client_module + + +class _FakeResponse: + def __init__(self, payload=None, content: bytes | None = None): + self._payload = payload + self.content = b"{}" if content is None else content + + def raise_for_status(self) -> None: + pass + + def json(self): + return self._payload + + +def _settings(**kwargs) -> Settings: + defaults = { + "username": "qa@example.com", + "api_key": "secret", + "project_id": 73, + } + defaults.update(kwargs) + return Settings(**defaults) + + +def _client(**kwargs) -> Client: + return Client(_settings(**kwargs)) + + +def _capture_request(monkeypatch, payload) -> dict: + captured: dict = {} + + class _FakeAsyncClient: + def __init__(self, timeout=None): + captured["timeout"] = timeout + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + async def request(self, method, url, **kwargs): + captured.update(method=method, url=url, **kwargs) + return _FakeResponse(payload) + + monkeypatch.setattr(client_module.httpx, "AsyncClient", _FakeAsyncClient) + return captured + + +async def test_request_uses_basic_authentication(monkeypatch): + captured = _capture_request(monkeypatch, {"id": 10}) + + result = await _client(url="https://testrail.example/").request( + "POST", "add_suite/73", {"name": "Suite"} + ) + + assert result == {"id": 10} + assert captured["url"] == ( + "https://testrail.example/index.php?/api/v2/add_suite/73" + ) + assert captured["auth"] == ("qa@example.com", "secret") + assert captured["json"] == {"name": "Suite"} + assert captured["timeout"] == 30 + + +async def test_request_allows_empty_response(monkeypatch): + captured: dict = {} + + class _FakeAsyncClient: + def __init__(self, timeout=None): + captured["timeout"] = timeout + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + async def request(self, method, url, **kwargs): + return _FakeResponse(content=b"") + + monkeypatch.setattr(client_module.httpx, "AsyncClient", _FakeAsyncClient) + + assert await _client().request("POST", "add_suite/73") == {} + + +async def test_request_requires_json_object_or_array(monkeypatch): + _capture_request(monkeypatch, "not-json-object") + + with pytest.raises(RuntimeError, match="unexpected response"): + await _client().request("GET", "get_projects") + + +async def test_endpoint_wrappers(monkeypatch): + captured = _capture_request(monkeypatch, {"id": 10}) + client = _client() + + await client.get_case_types() + assert captured["method"] == "GET" + assert captured["url"].endswith("/get_case_types") + + await client.get_templates() + assert captured["url"].endswith("/get_templates/73") + + await client.add_suite("Suite") + assert captured["url"].endswith("/add_suite/73") + assert captured["json"] == {"name": "Suite"} + + await client.add_section(10, "Cases") + assert captured["url"].endswith("/add_section/73") + assert captured["json"] == {"suite_id": 10, "name": "Cases"} + + await client.add_case(20, {"title": "Case"}) + assert captured["url"].endswith("/add_case/20") + assert captured["json"] == {"title": "Case"} + + +def test_suite_url_default_base(): + assert ( + _client().suite_url(42) + == "https://mozilla.testrail.io/index.php?/suites/view/42" + ) + + +def test_suite_url_injected_base(): + assert ( + _client(url="https://testrail.example/").suite_url(42) + == "https://testrail.example/index.php?/suites/view/42" + ) + + +def test_from_env_reads_environment(monkeypatch): + monkeypatch.setenv("TESTRAIL_USERNAME", "qa@example.com") + monkeypatch.setenv("TESTRAIL_API_KEY", "secret") + monkeypatch.setenv("TESTRAIL_PROJECT_ID", "73") + monkeypatch.setenv("TESTRAIL_URL", "https://testrail.env.example") + + settings = Settings.from_env() + + assert isinstance(settings, Settings) + assert settings.username == "qa@example.com" + assert settings.api_key == "secret" + assert settings.project_id == 73 + assert settings.url == "https://testrail.env.example" + + +def test_missing_required_env_rejected(monkeypatch): + monkeypatch.delenv("TESTRAIL_USERNAME", raising=False) + monkeypatch.delenv("TESTRAIL_API_KEY", raising=False) + monkeypatch.delenv("TESTRAIL_PROJECT_ID", raising=False) + + with pytest.raises(ValidationError): + Settings.from_env() + + +def test_invalid_project_id_rejected(): + with pytest.raises(ValidationError): + Settings( + username="qa@example.com", + api_key="secret", + project_id="grave-yard", + ) + + +async def test_custom_timeout_is_passed(monkeypatch): + captured = _capture_request(monkeypatch, {}) + await _client(timeout_seconds=5).request("GET", "get_projects") + assert captured["timeout"] == 5 + + +def test_defaults_from_env_when_no_settings(monkeypatch): + monkeypatch.setenv("TESTRAIL_USERNAME", "qa@example.com") + monkeypatch.setenv("TESTRAIL_API_KEY", "secret") + monkeypatch.setenv("TESTRAIL_PROJECT_ID", "73") + + client = Client() + + assert client.settings.username == "qa@example.com" + assert client.settings.project_id == 73 + + +def test_client_uses_httpx(): + assert client_module.httpx is httpx diff --git a/uv.lock b/uv.lock index 58bbae6eeb..3a03cee948 100644 --- a/uv.lock +++ b/uv.lock @@ -31,6 +31,7 @@ members = [ "hackbot-pulse-listener", "hackbot-runtime", "phabricator-client", + "testrail-client", "reviewhelper-api", ] @@ -2725,6 +2726,7 @@ dependencies = [ { name = "phabricator-client" }, { name = "pydantic-settings" }, { name = "requests" }, + { name = "testrail-client" }, { name = "weave" }, ] @@ -2749,6 +2751,7 @@ requires-dist = [ { name = "phabricator-client", editable = "libs/phabricator-client" }, { name = "pydantic-settings", specifier = ">=2.1.0" }, { name = "requests", specifier = ">=2.32.0" }, + { name = "testrail-client", editable = "libs/testrail-client" }, { name = "weave", specifier = ">=0.53.1" }, ] provides-extras = ["claude-sdk", "phabricator"] @@ -4998,6 +5001,30 @@ requires-dist = [ ] provides-extras = ["dev"] +[[package]] +name = "testrail-client" +version = "0.1.0" +source = { editable = "libs/testrail-client" } +dependencies = [ + { name = "httpx" }, + { name = "pydantic-settings" }, +] + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, + { name = "pytest-asyncio" }, +] + +[package.metadata] +requires-dist = [ + { name = "httpx", specifier = ">=0.26.0" }, + { name = "pydantic-settings", specifier = ">=2.1.0" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, + { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" }, +] +provides-extras = ["dev"] + [[package]] name = "pillow" version = "12.2.0"