diff --git a/src/server-factory.ts b/src/server-factory.ts index 82f9373..8381f24 100644 --- a/src/server-factory.ts +++ b/src/server-factory.ts @@ -17,6 +17,7 @@ import addSelfHealTools from "./tools/selfheal.js"; import addAppLiveTools from "./tools/applive.js"; import { setupOnInitialized } from "./oninitialized.js"; import { BrowserStackConfig } from "./lib/types.js"; +import addRCATools from "./tools/rca-agent.js"; /** * Wrapper class for BrowserStack MCP Server @@ -55,6 +56,7 @@ export class BrowserStackMcpServer { addFailureLogsTools, addAutomateTools, addSelfHealTools, + addRCATools, ]; toolAdders.forEach((adder) => { diff --git a/src/tools/rca-agent-utils/constants.ts b/src/tools/rca-agent-utils/constants.ts new file mode 100644 index 0000000..0ff64d9 --- /dev/null +++ b/src/tools/rca-agent-utils/constants.ts @@ -0,0 +1,37 @@ +import { z } from "zod"; +import { TestStatus } from "./types.js"; + +export const FETCH_RCA_PARAMS = { + testId: z + .array(z.string()) + .max(3) + .describe( + "Array of test IDs to fetch RCA data for (maximum 3 IDs). If not provided, use the listTestIds tool get all failed testcases. If more than 3 IDs are provided, only the first 3 will be processed." + ), +}; + +export const GET_BUILD_ID_PARAMS = { + projectName: z + .string() + .describe( + "The Browserstack project name used while creation of test run. Check browserstack.yml or similar project configuration files. If found extract it and provide to user, IF not found or unsure, prompt the user for this value. Do not make assumptions" + ), + buildName: z + .string() + .describe( + "The Browserstack build name used while creation of test run. Check browserstack.yml or similar project configuration files. If found extract it and provide to user, IF not found or unsure, prompt the user for this value. Do not make assumptions" + ), +}; + +export const LIST_TEST_IDS_PARAMS = { + buildId: z + .string() + .describe( + "The Browserstack Build ID of the test run. If not known, use the getBuildId tool to fetch it using project and build name" + ), + status: z + .nativeEnum(TestStatus) + .describe( + "Filter tests by status. If not provided, all tests are returned. Example for RCA usecase always use failed status" + ), +}; diff --git a/src/tools/rca-agent-utils/format-rca.ts b/src/tools/rca-agent-utils/format-rca.ts new file mode 100644 index 0000000..c4e8460 --- /dev/null +++ b/src/tools/rca-agent-utils/format-rca.ts @@ -0,0 +1,44 @@ +// Utility function to format RCA data for better readability +export function formatRCAData(rcaData: any): string { + if (!rcaData || !rcaData.testCases || rcaData.testCases.length === 0) { + return "No RCA data available."; + } + + let output = "## Root Cause Analysis Report\n\n"; + + rcaData.testCases.forEach((testCase: any, index: number) => { + // Show test case ID with smaller heading + output += `### Test Case ${index + 1}\n`; + output += `**Test ID:** ${testCase.id}\n`; + output += `**Status:** ${testCase.state}\n\n`; + + // Access RCA data from the correct path + const rca = testCase.rcaData?.rcaData; + + if (rca) { + if (rca.root_cause) { + output += `**Root Cause:** ${rca.root_cause}\n\n`; + } + + if (rca.failure_type) { + output += `**Failure Type:** ${rca.failure_type}\n\n`; + } + + if (rca.description) { + output += `**Detailed Analysis:**\n${rca.description}\n\n`; + } + + if (rca.possible_fix) { + output += `**Recommended Fix:**\n${rca.possible_fix}\n\n`; + } + } else if (testCase.rcaData?.error) { + output += `**Error:** ${testCase.rcaData.error}\n\n`; + } else if (testCase.state === "failed") { + output += `**Note:** RCA analysis failed or is not available for this test case.\n\n`; + } + + output += "---\n\n"; + }); + + return output; +} diff --git a/src/tools/rca-agent-utils/get-build-id.ts b/src/tools/rca-agent-utils/get-build-id.ts new file mode 100644 index 0000000..6f3dffe --- /dev/null +++ b/src/tools/rca-agent-utils/get-build-id.ts @@ -0,0 +1,32 @@ +export async function getBuildId( + projectName: string, + buildName: string, + username: string, + accessKey: string, +): Promise { + const url = new URL( + "https://api-automation.browserstack.com/ext/v1/builds/latest", + ); + url.searchParams.append("project_name", projectName); + url.searchParams.append("build_name", buildName); + url.searchParams.append("user_name", username); + + const authHeader = + "Basic " + Buffer.from(`${username}:${accessKey}`).toString("base64"); + + const response = await fetch(url.toString(), { + headers: { + Authorization: authHeader, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + throw new Error( + `Failed to fetch build ID: ${response.status} ${response.statusText}`, + ); + } + + const data = await response.json(); + return data.build_id; +} diff --git a/src/tools/rca-agent-utils/get-failed-test-id.ts b/src/tools/rca-agent-utils/get-failed-test-id.ts new file mode 100644 index 0000000..7e8e6fb --- /dev/null +++ b/src/tools/rca-agent-utils/get-failed-test-id.ts @@ -0,0 +1,88 @@ +import logger from "../../logger.js"; +import { TestStatus, FailedTestInfo, TestRun, TestDetails } from "./types.js"; + +export async function getTestIds( + buildId: string, + authString: string, + status?: TestStatus, +): Promise { + const baseUrl = `https://api-automation.browserstack.com/ext/v1/builds/${buildId}/testRuns`; + let url = status ? `${baseUrl}?test_statuses=${status}` : baseUrl; + let allFailedTests: FailedTestInfo[] = []; + let requestNumber = 0; + + // Construct Basic auth header + const encodedCredentials = Buffer.from(authString).toString("base64"); + const authHeader = `Basic ${encodedCredentials}`; + + try { + while (true) { + requestNumber++; + + const response = await fetch(url, { + headers: { + Authorization: authHeader, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + throw new Error( + `Failed to fetch test runs: ${response.status} ${response.statusText}`, + ); + } + + const data = (await response.json()) as TestRun; + + // Extract failed IDs from current page + if (data.hierarchy && data.hierarchy.length > 0) { + const currentFailedTests = extractFailedTestIds(data.hierarchy); + allFailedTests = allFailedTests.concat(currentFailedTests); + } + + // Check for pagination termination conditions + if (!data.pagination?.has_next || !data.pagination.next_page) { + break; + } + + // Safety limit to prevent runaway requests + if (requestNumber >= 5) { + break; + } + + // Prepare next request + url = `${baseUrl}?next_page=${encodeURIComponent(data.pagination.next_page)}`; + } + + // Return unique failed test IDs + return allFailedTests; + } catch (error) { + logger.error("Error fetching failed tests:", error); + throw error; + } +} + +// Recursive function to extract failed test IDs from hierarchy +function extractFailedTestIds(hierarchy: TestDetails[]): FailedTestInfo[] { + let failedTests: FailedTestInfo[] = []; + + for (const node of hierarchy) { + if (node.details?.status === "failed" && node.details?.run_count) { + if (node.details?.observability_url) { + const idMatch = node.details.observability_url.match(/details=(\d+)/); + if (idMatch) { + failedTests.push({ + test_id: idMatch[1], + test_name: node.display_name || `Test ${idMatch[1]}`, + }); + } + } + } + + if (node.children && node.children.length > 0) { + failedTests = failedTests.concat(extractFailedTestIds(node.children)); + } + } + + return failedTests; +} diff --git a/src/tools/rca-agent-utils/rca-data.ts b/src/tools/rca-agent-utils/rca-data.ts new file mode 100644 index 0000000..1e66220 --- /dev/null +++ b/src/tools/rca-agent-utils/rca-data.ts @@ -0,0 +1,285 @@ +import { RCAState, RCATestCase, RCAResponse } from "./types.js"; + +interface ScanProgressContext { + sendNotification: (notification: any) => Promise; + _meta?: { + progressToken?: string | number; + }; +} + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +function isInProgressState(state: RCAState): boolean { + return [ + RCAState.PENDING, + RCAState.FETCHING_LOGS, + RCAState.GENERATING_RCA, + RCAState.GENERATED_RCA, + ].includes(state); +} + +function isFailedState(state: RCAState): boolean { + return [ + RCAState.FAILED, + RCAState.LLM_SERVICE_ERROR, + RCAState.LOG_FETCH_ERROR, + RCAState.UNKNOWN_ERROR, + RCAState.TIMEOUT, + ].includes(state); +} + +function calculateProgress( + resolvedCount: number, + totalCount: number, + baseProgress: number = 10, +): number { + if (totalCount === 0) return 100; // ✅ fix divide by zero + const progressRange = 90 - baseProgress; + const completionProgress = (resolvedCount / totalCount) * progressRange; + return Math.min(100, baseProgress + completionProgress); +} + +// ✅ centralized mapping function +function mapApiState(apiState?: string): RCAState { + const state = apiState?.toLowerCase(); + switch (state) { + case "completed": + return RCAState.COMPLETED; + case "pending": + return RCAState.PENDING; + case "fetching_logs": + return RCAState.FETCHING_LOGS; + case "generating_rca": + return RCAState.GENERATING_RCA; + case "generated_rca": + return RCAState.GENERATED_RCA; + case "error": + return RCAState.UNKNOWN_ERROR; + default: + return RCAState.UNKNOWN_ERROR; + } +} + +async function notifyProgress( + context: ScanProgressContext | undefined, + message: string, + progress: number, +) { + if (!context?.sendNotification) return; + + await context.sendNotification({ + method: "notifications/progress", + params: { + progressToken: context._meta?.progressToken?.toString(), + message, + progress, + total: 100, + }, + }); +} + +async function updateProgress( + context: ScanProgressContext | undefined, + testCases: RCATestCase[], + message?: string, +) { + const inProgressCases = testCases.filter((tc) => isInProgressState(tc.state)); + const resolvedCount = testCases.length - inProgressCases.length; + + await notifyProgress( + context, + message ?? + `RCA analysis in progress (${resolvedCount}/${testCases.length} resolved)`, + inProgressCases.length === 0 + ? 100 + : calculateProgress(resolvedCount, testCases.length), + ); +} + +async function fetchInitialRCA( + testId: string, + headers: Record, + baseUrl: string, +): Promise { + const url = baseUrl.replace("{testId}", testId); + + try { + const response = await fetch(url, { headers }); + + if (!response.ok) { + return { + id: testId, + testRunId: testId, + state: RCAState.LOG_FETCH_ERROR, + rcaData: { + error: `HTTP ${response.status}: Failed to start RCA analysis`, + }, + }; + } + + const data = await response.json(); + const resultState = mapApiState(data.state); + + return { + id: testId, + testRunId: testId, + state: resultState, + ...(resultState === RCAState.COMPLETED && { rcaData: data }), + ...(isFailedState(resultState) && + data.state && { + rcaData: { + error: `API returned state: ${data.state}`, + originalResponse: data, + }, + }), + }; + } catch (error) { + return { + id: testId, + testRunId: testId, + state: RCAState.LLM_SERVICE_ERROR, + rcaData: { + error: + error instanceof Error ? error.message : "Network or parsing error", + }, + }; + } +} + +async function pollRCAResults( + testCases: RCATestCase[], + headers: Record, + baseUrl: string, + context: ScanProgressContext | undefined, + pollInterval: number, + timeout: number, + initialDelay: number, +): Promise { + const startTime = Date.now(); + await delay(initialDelay); + + try { + while (true) { + const inProgressCases = testCases.filter((tc) => + isInProgressState(tc.state), + ); + await updateProgress(context, testCases); + + if (inProgressCases.length === 0) break; + + if (Date.now() - startTime >= timeout) { + inProgressCases.forEach((tc) => { + tc.state = RCAState.TIMEOUT; + tc.rcaData = { error: `Timeout after ${timeout}ms` }; + }); + await updateProgress(context, testCases, "RCA analysis timed out"); + break; + } + + await Promise.allSettled( + inProgressCases.map(async (tc) => { + try { + const pollUrl = baseUrl.replace("{testId}", tc.id); + const response = await fetch(pollUrl, { headers }); + if (!response.ok) { + const errorText = await response.text(); + tc.state = RCAState.LOG_FETCH_ERROR; + tc.rcaData = { + error: `HTTP ${response.status}: Polling failed - ${errorText}`, + }; + return; + } + + const data = await response.json(); + if (!isFailedState(tc.state)) { + const mappedState = mapApiState(data.state); + tc.state = mappedState; + + if (mappedState === RCAState.COMPLETED) { + tc.rcaData = data; + } else if (mappedState === RCAState.UNKNOWN_ERROR) { + tc.rcaData = { + error: `API returned state: ${data.state}`, + originalResponse: data, + }; + } + } + } catch (err) { + if (!isFailedState(tc.state)) { + tc.state = RCAState.LLM_SERVICE_ERROR; + tc.rcaData = { + error: + err instanceof Error + ? err.message + : "Network or parsing error", + }; + } + } + }), + ); + + await delay(pollInterval); + } + } catch (err) { + testCases + .filter((tc) => isInProgressState(tc.state)) + .forEach((tc) => { + tc.state = RCAState.UNKNOWN_ERROR; + tc.rcaData = { + error: err instanceof Error ? err.message : "Unexpected error", + }; + }); + await updateProgress( + context, + testCases, + "RCA analysis failed due to unexpected error", + ); + } + + return { testCases }; +} + +export async function getRCAData( + testIds: string[], + authString: string, + context?: ScanProgressContext, +): Promise { + const pollInterval = 5000; + const timeout = 40000; + const initialDelay = 20000; + + const baseUrl = + "https://api-observability.browserstack.com/ext/v1/testRun/{testId}/testRca"; + const headers = { + Authorization: `Basic ${Buffer.from(authString).toString("base64")}`, + "Content-Type": "application/json", + }; + + await notifyProgress(context, "Starting RCA analysis for test cases...", 0); + + const testCases = await Promise.all( + testIds.map((testId) => fetchInitialRCA(testId, headers, baseUrl)), + ); + + const inProgressCount = testCases.filter((tc) => + isInProgressState(tc.state), + ).length; + + await notifyProgress( + context, + `Initial RCA requests completed. ${inProgressCount} cases pending analysis...`, + 10, + ); + + if (inProgressCount === 0) return { testCases }; + + return await pollRCAResults( + testCases, + headers, + baseUrl, + context, + pollInterval, + timeout, + initialDelay, + ); +} diff --git a/src/tools/rca-agent-utils/types.ts b/src/tools/rca-agent-utils/types.ts new file mode 100644 index 0000000..c03fbdf --- /dev/null +++ b/src/tools/rca-agent-utils/types.ts @@ -0,0 +1,50 @@ +export enum TestStatus { + PASSED = "passed", + FAILED = "failed", + PENDING = "pending", + SKIPPED = "skipped", +} + +export interface TestDetails { + status: TestStatus; + details: any; + children?: TestDetails[]; + display_name?: string; +} + +export interface TestRun { + hierarchy: TestDetails[]; + pagination?: { + has_next: boolean; + next_page: string | null; + }; +} + +export interface FailedTestInfo { + test_id: string; + test_name: string; +} + +export enum RCAState { + PENDING = "pending", + FETCHING_LOGS = "fetching_logs", + GENERATING_RCA = "generating_rca", + GENERATED_RCA = "generated_rca", + COMPLETED = "completed", + FAILED = "failed", + LLM_SERVICE_ERROR = "LLM_SERVICE_ERROR", + LOG_FETCH_ERROR = "LOG_FETCH_ERROR", + UNKNOWN_ERROR = "UNKNOWN_ERROR", + TIMEOUT = "TIMEOUT", +} + +export interface RCATestCase { + id: string; + testRunId: string; + state: RCAState; + rcaData?: any; +} + +export interface RCAResponse { + testCases: RCATestCase[]; +} diff --git a/src/tools/rca-agent.ts b/src/tools/rca-agent.ts new file mode 100644 index 0000000..3b723b8 --- /dev/null +++ b/src/tools/rca-agent.ts @@ -0,0 +1,194 @@ +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { FETCH_RCA_PARAMS, GET_BUILD_ID_PARAMS, LIST_TEST_IDS_PARAMS } from "./rca-agent-utils/constants.js"; +import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; +import logger from "../logger.js"; +import { BrowserStackConfig } from "../lib/types.js"; +import { getBrowserStackAuth } from "../lib/get-auth.js"; +import { getBuildId } from "./rca-agent-utils/get-build-id.js"; +import { getTestIds } from "./rca-agent-utils/get-failed-test-id.js"; +import { getRCAData } from "./rca-agent-utils/rca-data.js"; +import { formatRCAData } from "./rca-agent-utils/format-rca.js"; +import { TestStatus } from "./rca-agent-utils/types.js"; + +// Tool function to fetch build ID +export async function getBuildIdTool( + args: { + projectName: string; + buildName: string; + }, + config: BrowserStackConfig, +): Promise { + try { + const { projectName, buildName } = args; + const authString = getBrowserStackAuth(config); + const [username, accessKey] = authString.split(":"); + const buildId = await getBuildId( + projectName, + buildName, + username, + accessKey, + ); + return { + content: [ + { + type: "text", + text: buildId, + }, + ], + }; + } catch (error) { + logger.error("Error fetching build ID", error); + const errorMessage = + error instanceof Error ? error.message : "Unknown error"; + return { + content: [ + { + type: "text", + text: `Error fetching build ID: ${errorMessage}`, + }, + ], + }; + } +} + +// Tool function that fetches RCA data +export async function fetchRCADataTool( + args: { testId: string[] }, + config: BrowserStackConfig, +): Promise { + try { + const authString = getBrowserStackAuth(config); + + // Limit to first 3 test IDs for performance + const testIds = args.testId; + + const rcaData = await getRCAData(testIds, authString); + + const formattedData = formatRCAData(rcaData); + + return { + content: [ + { + type: "text", + text: formattedData, + }, + ], + }; + } catch (error) { + logger.error("Error fetching RCA data", error); + throw error; + } +} + +export async function listTestIdsTool( + args: { + buildId: string; + status?: TestStatus; + }, + config: BrowserStackConfig, +): Promise { + try { + const { buildId, status } = args; + const authString = getBrowserStackAuth(config); + + // Get test IDs + const testIds = await getTestIds(buildId, authString, status); + + return { + content: [ + { + type: "text", + text: JSON.stringify(testIds, null, 2), + }, + ], + }; + } catch (error) { + logger.error("Error listing test IDs", error); + const errorMessage = + error instanceof Error ? error.message : "Unknown error"; + return { + content: [ + { + type: "text", + text: `Error listing test IDs: ${errorMessage}`, + }, + ], + }; + } +} + +// Registers the fetchRCA tool with the MCP server +export default function addRCATools( + server: McpServer, + config: BrowserStackConfig, +) { + const tools: Record = {}; + + tools.fetchRCA = server.tool( + "fetchRCA", + "Retrieves AI-RCA (Root Cause Analysis) data for a BrowserStack Automate and App-Automate session and provides insights into test failures.", + FETCH_RCA_PARAMS, + async (args) => { + try { + return await fetchRCADataTool(args, config); + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : "Unknown error"; + return { + content: [ + { + type: "text", + text: `Error during fetching RCA data: ${errorMessage}`, + }, + ], + }; + } + }, + ); + + tools.getBuildId = server.tool( + "getBuildId", + "Get the BrowserStack build ID for a given project and build name.", + GET_BUILD_ID_PARAMS, + async (args) => { + try { + return await getBuildIdTool(args, config); + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : "Unknown error"; + return { + content: [ + { + type: "text", + text: `Error during fetching build ID: ${errorMessage}`, + }, + ], + }; + } + }, + ); + + tools.listTestIds = server.tool( + "listTestIds", + "List test IDs from a BrowserStack Automate build, optionally filtered by status", + LIST_TEST_IDS_PARAMS, + async (args) => { + try { + return await listTestIdsTool(args, config); + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : "Unknown error"; + return { + content: [ + { + type: "text", + text: `Error during listing test IDs: ${errorMessage}`, + }, + ], + }; + } + }, + ); + + return tools; +}