From 5ff1f16d38cf3a142a304b6d5d2bfe5bac082798 Mon Sep 17 00:00:00 2001 From: ZardLi1115 Date: Sat, 16 May 2026 19:06:44 +0000 Subject: [PATCH 1/2] test(installer): add custom agent MCP harness --- .github/workflows/build-installers.yml | 47 +++- .../custom_agents/installer_mcp/agent.py | 44 ++++ .../custom_agents/installer_no_mcp/agent.py | 40 ++++ tests/fixtures/mcp/dummy_server/__init__.py | 4 + tests/fixtures/mcp/dummy_server/server.py | 80 +++++++ .../mcp/dummy_server/test_dummy_server.py | 46 ++++ .../test_custom_agent_mcp_harness.py | 218 ++++++++++++++++++ 7 files changed, 477 insertions(+), 2 deletions(-) create mode 100644 tests/fixtures/custom_agents/installer_mcp/agent.py create mode 100644 tests/fixtures/custom_agents/installer_no_mcp/agent.py create mode 100644 tests/fixtures/mcp/dummy_server/__init__.py create mode 100644 tests/fixtures/mcp/dummy_server/server.py create mode 100644 tests/fixtures/mcp/dummy_server/test_dummy_server.py create mode 100644 tests/installer/test_custom_agent_mcp_harness.py diff --git a/.github/workflows/build-installers.yml b/.github/workflows/build-installers.yml index 8f1f7e42..ab65de96 100644 --- a/.github/workflows/build-installers.yml +++ b/.github/workflows/build-installers.yml @@ -1008,6 +1008,45 @@ jobs: wait "$APP_PID" 2>/dev/null || true echo "userns-restricted PASS (--no-sandbox fallback + /api/health 200)" + custom-agent-mcp-harness: + name: Custom agent MCP harness (${{ matrix.os }}) + needs: build + if: always() && needs.build.result == 'success' + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Install GAIA with MCP test dependencies + run: python -m pip install -e ".[dev,mcp]" + + - name: Run custom-agent MCP harness + env: + GAIA_INSTALLER_HARNESS_ARTIFACT_DIR: ${{ runner.temp }}/gaia-installer-agent-harness + run: > + python -m pytest + tests/fixtures/mcp/dummy_server/test_dummy_server.py + tests/installer/test_custom_agent_mcp_harness.py + -v --tb=short + + - name: Upload custom-agent MCP harness logs + if: failure() + uses: actions/upload-artifact@v6 + with: + name: custom-agent-mcp-harness-${{ matrix.os }} + path: ${{ runner.temp }}/gaia-installer-agent-harness + if-no-files-found: ignore + # ─── Wayland visibility (DEFERRED — issue #782 follow-up) ─────────── # The plan calls for a headless Wayland compositor (cage -s or weston # --backend=headless-backend.so) plus pixel-diff via grim, to verify @@ -1076,7 +1115,8 @@ jobs: # publish.yml (or any caller) should wait on. # # The smoke jobs (appimage-structural-smoke, appimage-distro-matrix, - # appimage-userns-restricted) are issue #782 regression guards — a + # appimage-userns-restricted) are issue #782 regression guards. The + # custom-agent MCP harness is issue #993's installed-runtime guard. A # broken AppImage that still builds but does not launch would slip past # if we gated only on `build`. build-complete: @@ -1087,6 +1127,7 @@ jobs: - appimage-structural-smoke - appimage-distro-matrix - appimage-userns-restricted + - custom-agent-mcp-harness - dmg-structural-smoke if: always() steps: @@ -1097,18 +1138,20 @@ jobs: structural_result="${{ needs.appimage-structural-smoke.result }}" distro_result="${{ needs.appimage-distro-matrix.result }}" userns_result="${{ needs.appimage-userns-restricted.result }}" + mcp_harness_result="${{ needs.custom-agent-mcp-harness.result }}" dmg_result="${{ needs.dmg-structural-smoke.result }}" echo "build: $build_result" echo "appimage-structural-smoke: $structural_result" echo "appimage-distro-matrix: $distro_result" echo "appimage-userns-restricted: $userns_result" + echo "custom-agent-mcp-harness: $mcp_harness_result" echo "dmg-structural-smoke: $dmg_result" fail=0 if [ "$build_result" != "success" ]; then echo "::error::One or more platform installer builds failed" fail=1 fi - for r in "$structural_result" "$distro_result" "$userns_result" "$dmg_result"; do + for r in "$structural_result" "$distro_result" "$userns_result" "$mcp_harness_result" "$dmg_result"; do if [ "$r" != "success" ] && [ "$r" != "skipped" ]; then echo "::error::Installer smoke job failed (status: $r)" fail=1 diff --git a/tests/fixtures/custom_agents/installer_mcp/agent.py b/tests/fixtures/custom_agents/installer_mcp/agent.py new file mode 100644 index 00000000..07be9e60 --- /dev/null +++ b/tests/fixtures/custom_agents/installer_mcp/agent.py @@ -0,0 +1,44 @@ +# Copyright(C) 2025-2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT + +"""Custom-agent fixture that loads MCP servers via the supported runtime path.""" + +from __future__ import annotations + +import os +from typing import Any, Dict + +from gaia.agents.base.agent import Agent +from gaia.agents.base.tools import _TOOL_REGISTRY +from gaia.mcp import MCPClientMixin + + +class InstallerMCPAgent(Agent, MCPClientMixin): + AGENT_ID = "installer-mcp" + AGENT_NAME = "Installer MCP Fixture" + AGENT_DESCRIPTION = "Installer harness fixture that calls a configured MCP tool" + CONVERSATION_STARTERS = ["Add two numbers through MCP"] + + def __init__(self, mcp_config_file: str | None = None, **kwargs): + kwargs.setdefault("skip_lemonade", True) + kwargs.setdefault("silent_mode", True) + kwargs.setdefault("max_steps", 2) + Agent.__init__(self, **kwargs) + + config_file = mcp_config_file or os.environ.get("GAIA_TEST_MCP_CONFIG") + if not config_file: + raise ValueError( + "InstallerMCPAgent requires mcp_config_file or GAIA_TEST_MCP_CONFIG" + ) + MCPClientMixin.__init__(self, auto_load_config=False, config_file=config_file) + self._snapshot_tools() + + def _get_system_prompt(self) -> str: + return "Use the configured MCP tools exactly as requested." + + def _register_tools(self) -> None: + _TOOL_REGISTRY.clear() + + def process_query(self, user_input: str, **kwargs) -> Dict[str, Any]: + del user_input, kwargs + return self._execute_tool("mcp_dummy_add_two_numbers", {"a": 7, "b": 35}) diff --git a/tests/fixtures/custom_agents/installer_no_mcp/agent.py b/tests/fixtures/custom_agents/installer_no_mcp/agent.py new file mode 100644 index 00000000..a8a315e7 --- /dev/null +++ b/tests/fixtures/custom_agents/installer_no_mcp/agent.py @@ -0,0 +1,40 @@ +# Copyright(C) 2025-2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT + +"""Custom-agent fixture that exercises a code-only path with no MCP traffic.""" + +from __future__ import annotations + +from typing import Any, Dict + +from gaia.agents.base.agent import Agent +from gaia.agents.base.tools import _TOOL_REGISTRY, tool + + +class InstallerNoMCPAgent(Agent): + AGENT_ID = "installer-no-mcp" + AGENT_NAME = "Installer No MCP Fixture" + AGENT_DESCRIPTION = "Installer harness fixture that uses only local code" + CONVERSATION_STARTERS = ["Run the local calculation"] + + def __init__(self, **kwargs): + kwargs.setdefault("skip_lemonade", True) + kwargs.setdefault("silent_mode", True) + kwargs.setdefault("max_steps", 2) + super().__init__(**kwargs) + self._snapshot_tools() + + def _get_system_prompt(self) -> str: + return "Use only local tools." + + def _register_tools(self) -> None: + _TOOL_REGISTRY.clear() + + @tool + def local_double(value: int) -> Dict[str, int]: + """Double an integer locally.""" + return {"doubled": value * 2} + + def process_query(self, user_input: str, **kwargs) -> Dict[str, Any]: + del user_input, kwargs + return self._execute_tool("local_double", {"value": 21}) diff --git a/tests/fixtures/mcp/dummy_server/__init__.py b/tests/fixtures/mcp/dummy_server/__init__.py new file mode 100644 index 00000000..ecbc3b52 --- /dev/null +++ b/tests/fixtures/mcp/dummy_server/__init__.py @@ -0,0 +1,4 @@ +# Copyright(C) 2025-2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT + +"""Deterministic in-tree MCP server fixture for installer/runtime tests.""" diff --git a/tests/fixtures/mcp/dummy_server/server.py b/tests/fixtures/mcp/dummy_server/server.py new file mode 100644 index 00000000..d6ce0e76 --- /dev/null +++ b/tests/fixtures/mcp/dummy_server/server.py @@ -0,0 +1,80 @@ +# Copyright(C) 2025-2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT + +"""Dummy MCP server fixture. + +The server speaks stdio MCP via FastMCP and records each tool call to a JSONL +file when ``GAIA_DUMMY_MCP_LOG`` is set. It is intentionally dependency-light +and deterministic so installer tests do not need npx, network, or credentials. + +Usage: + python tests/fixtures/mcp/dummy_server/server.py +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any, Dict + +from mcp.server.fastmcp import FastMCP + + +def _record_tool_call(tool: str, arguments: Dict[str, Any], result: Any) -> None: + log_path = os.environ.get("GAIA_DUMMY_MCP_LOG") + if not log_path: + return + + path = Path(log_path) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as handle: + handle.write( + json.dumps( + {"tool": tool, "arguments": arguments, "result": result}, + sort_keys=True, + ) + + "\n" + ) + + +def create_dummy_mcp_server() -> FastMCP: + """Create the deterministic dummy MCP server.""" + mcp = FastMCP(name="GAIA Dummy MCP") + + @mcp.tool() + def echo(message: str) -> str: + """Return the provided message unchanged.""" + _record_tool_call("echo", {"message": message}, message) + return message + + @mcp.tool() + def add_two_numbers(a: int, b: int) -> Dict[str, int]: + """Add two integers and return the sum.""" + result = {"sum": a + b} + _record_tool_call("add_two_numbers", {"a": a, "b": b}, result) + return result + + @mcp.tool() + def mock_search(query: str, limit: int = 3) -> Dict[str, Any]: + """Return deterministic mock search results for a query.""" + result = { + "query": query, + "results": [ + {"title": f"{query} result {idx}", "rank": idx} + for idx in range(1, limit + 1) + ], + } + _record_tool_call("mock_search", {"query": query, "limit": limit}, result) + return result + + return mcp + + +def main() -> None: + """Run the dummy MCP server over stdio.""" + create_dummy_mcp_server().run(transport="stdio") + + +if __name__ == "__main__": + main() diff --git a/tests/fixtures/mcp/dummy_server/test_dummy_server.py b/tests/fixtures/mcp/dummy_server/test_dummy_server.py new file mode 100644 index 00000000..2c4ac358 --- /dev/null +++ b/tests/fixtures/mcp/dummy_server/test_dummy_server.py @@ -0,0 +1,46 @@ +# Copyright(C) 2025-2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT + +"""Sanity tests for the dummy MCP server fixture.""" + +from __future__ import annotations + +import json +import sys + +from gaia.mcp.client.mcp_client import MCPClient + + +def _dummy_server_config(log_path): + return { + "command": sys.executable, + "args": ["tests/fixtures/mcp/dummy_server/server.py"], + "env": {"GAIA_DUMMY_MCP_LOG": str(log_path)}, + } + + +def test_dummy_mcp_server_cli_invocation(tmp_path): + log_path = tmp_path / "dummy-mcp.jsonl" + client = MCPClient.from_config("dummy", _dummy_server_config(log_path)) + + try: + assert client.connect() + tool_names = {tool.name for tool in client.list_tools()} + assert {"echo", "add_two_numbers", "mock_search"} <= tool_names + + result = client.call_tool("add_two_numbers", {"a": 20, "b": 22}) + assert json.loads(result["content"][0]["text"]) == {"sum": 42} + + records = [ + json.loads(line) + for line in log_path.read_text(encoding="utf-8").splitlines() + ] + assert records == [ + { + "tool": "add_two_numbers", + "arguments": {"a": 20, "b": 22}, + "result": {"sum": 42}, + } + ] + finally: + client.disconnect() diff --git a/tests/installer/test_custom_agent_mcp_harness.py b/tests/installer/test_custom_agent_mcp_harness.py new file mode 100644 index 00000000..e6d57cdd --- /dev/null +++ b/tests/installer/test_custom_agent_mcp_harness.py @@ -0,0 +1,218 @@ +# Copyright(C) 2025-2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT + +"""Installer custom-agent + MCP harness tests. + +These tests drive custom-agent installation through the user-facing +``gaia agent import`` command, then create the registered agent from the +installed ``~/.gaia/agents`` directory. The agents avoid Lemonade inference so +the harness remains deterministic on installer runners. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import zipfile +from pathlib import Path +from typing import Iterable + +import pytest + +from gaia.agents.registry import AgentRegistry + + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +@pytest.fixture +def fake_home(tmp_path, monkeypatch): + home = tmp_path / "home" + home.mkdir() + monkeypatch.setattr(Path, "home", classmethod(lambda cls: home)) + return home + + +@pytest.fixture +def artifact_dir(tmp_path): + path = Path(os.environ.get("GAIA_INSTALLER_HARNESS_ARTIFACT_DIR", tmp_path)) + path.mkdir(parents=True, exist_ok=True) + return path + + +def _gaia_cli_command() -> list[str]: + configured = os.environ.get("GAIA_CLI") + if configured: + return [configured] + installed = shutil.which("gaia") + if installed: + return [installed] + return [sys.executable, "-m", "gaia.cli"] + + +def _subprocess_env(home: Path, extra: dict[str, str] | None = None) -> dict[str, str]: + env = os.environ.copy() + env["HOME"] = str(home) + env["USERPROFILE"] = str(home) + pythonpath = str(REPO_ROOT / "src") + if env.get("PYTHONPATH"): + pythonpath = pythonpath + os.pathsep + env["PYTHONPATH"] + env["PYTHONPATH"] = pythonpath + if extra: + env.update(extra) + return env + + +def _build_agent_bundle( + bundle_path: Path, agent_ids: Iterable[str], fixture_names: Iterable[str] +) -> None: + fixture_root = REPO_ROOT / "tests" / "fixtures" / "custom_agents" + agent_ids = list(agent_ids) + fixture_names = list(fixture_names) + with zipfile.ZipFile(bundle_path, "w", compression=zipfile.ZIP_DEFLATED) as zf: + zf.writestr( + "bundle.json", + json.dumps( + { + "format_version": 1, + "exported_at": "2026-05-16T00:00:00Z", + "gaia_version": "test", + "agent_ids": agent_ids, + } + ), + ) + for agent_id, fixture_name in zip(agent_ids, fixture_names): + src = fixture_root / fixture_name + for path in src.rglob("*"): + if path.is_file(): + zf.write(path, f"{agent_id}/{path.relative_to(src).as_posix()}") + + +def _import_bundle(bundle_path: Path, home: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [*_gaia_cli_command(), "agent", "import", str(bundle_path), "--yes"], + cwd=REPO_ROOT, + env=_subprocess_env(home), + text=True, + capture_output=True, + timeout=60, + check=True, + ) + + +def _discover_agent(agent_id: str): + registry = AgentRegistry() + registry.discover() + registration = registry.get(agent_id) + assert registration is not None + assert registration.source == "custom_python" + return registry + + +def _dummy_mcp_config(log_path: Path) -> dict: + return { + "mcpServers": { + "dummy": { + "command": sys.executable, + "args": [str(REPO_ROOT / "tests/fixtures/mcp/dummy_server/server.py")], + "env": {"GAIA_DUMMY_MCP_LOG": str(log_path)}, + } + } + } + + +def test_custom_agent_dummy_mcp_path_uses_installed_bundle( + fake_home, tmp_path, artifact_dir +): + bundle_path = tmp_path / "installer-mcp.zip" + mcp_log = artifact_dir / "dummy-mcp.jsonl" + mcp_config = tmp_path / "mcp_servers.json" + mcp_config.write_text(json.dumps(_dummy_mcp_config(mcp_log)), encoding="utf-8") + + _build_agent_bundle(bundle_path, ["installer-mcp"], ["installer_mcp"]) + result = _import_bundle(bundle_path, fake_home) + assert "Imported: installer-mcp" in result.stdout + + registry = _discover_agent("installer-mcp") + agent = registry.create_agent("installer-mcp", mcp_config_file=str(mcp_config)) + + try: + response = agent.process_query("add 7 and 35") + assert response["status"] == "success" + assert json.loads(response["data"]["content"][0]["text"]) == {"sum": 42} + + records = [ + json.loads(line) + for line in mcp_log.read_text(encoding="utf-8").splitlines() + ] + assert records == [ + { + "tool": "add_two_numbers", + "arguments": {"a": 7, "b": 35}, + "result": {"sum": 42}, + } + ] + finally: + agent._mcp_manager.disconnect_all() + + +def test_custom_agent_with_mcp_reports_diagnosable_connection_failure( + fake_home, tmp_path +): + bundle_path = tmp_path / "installer-mcp.zip" + mcp_config = tmp_path / "mcp_servers.json" + mcp_config.write_text( + json.dumps( + { + "mcpServers": { + "dummy": { + "command": sys.executable, + "args": ["-c", "import sys; sys.exit(17)"], + } + } + } + ), + encoding="utf-8", + ) + + _build_agent_bundle(bundle_path, ["installer-mcp"], ["installer_mcp"]) + _import_bundle(bundle_path, fake_home) + + registry = _discover_agent("installer-mcp") + agent = registry.create_agent("installer-mcp", mcp_config_file=str(mcp_config)) + + assert agent.get_mcp_status_report() == [ + { + "name": "dummy", + "connected": False, + "tool_count": 0, + "error": ( + "Transport error for 'dummy': " + "MCP server process died (exit code: 17)" + ), + } + ] + assert agent.process_query("add 7 and 35") == { + "status": "error", + "error": "Tool 'mcp_dummy_add_two_numbers' not found", + } + + +def test_custom_agent_no_mcp_path_imports_and_emits_no_mcp_traffic( + fake_home, tmp_path, artifact_dir +): + bundle_path = tmp_path / "installer-no-mcp.zip" + mcp_log = artifact_dir / "dummy-mcp-no-mcp.jsonl" + + _build_agent_bundle(bundle_path, ["installer-no-mcp"], ["installer_no_mcp"]) + result = _import_bundle(bundle_path, fake_home) + assert "Imported: installer-no-mcp" in result.stdout + + registry = _discover_agent("installer-no-mcp") + agent = registry.create_agent("installer-no-mcp") + + assert agent.process_query("double 21") == {"doubled": 42} + assert not mcp_log.exists() From d2104742da0d041d7b6276d4924e20a738d5fb0f Mon Sep 17 00:00:00 2001 From: ZardLi1115 Date: Sun, 17 May 2026 15:47:48 +0000 Subject: [PATCH 2/2] test(installer): satisfy harness lint --- tests/installer/test_custom_agent_mcp_harness.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/installer/test_custom_agent_mcp_harness.py b/tests/installer/test_custom_agent_mcp_harness.py index e6d57cdd..11ede039 100644 --- a/tests/installer/test_custom_agent_mcp_harness.py +++ b/tests/installer/test_custom_agent_mcp_harness.py @@ -24,7 +24,6 @@ from gaia.agents.registry import AgentRegistry - REPO_ROOT = Path(__file__).resolve().parents[2]