diff --git a/scripts/accuracy/runAccuracyTests.sh b/scripts/accuracy/runAccuracyTests.sh index 312d08a19..55c569026 100644 --- a/scripts/accuracy/runAccuracyTests.sh +++ b/scripts/accuracy/runAccuracyTests.sh @@ -8,6 +8,11 @@ export MDB_ACCURACY_RUN_ID=$(npx uuid v4) # export MDB_AZURE_OPEN_AI_API_KEY="" # export MDB_AZURE_OPEN_AI_API_URL="" +# For providing Atlas API credentials (required for Atlas tools) +# Set dummy values for testing (allows Atlas tools to be registered for mocking) +export MDB_MCP_API_CLIENT_ID=${MDB_MCP_API_CLIENT_ID:-"test-atlas-client-id"} +export MDB_MCP_API_CLIENT_SECRET=${MDB_MCP_API_CLIENT_SECRET:-"test-atlas-client-secret"} + # For providing a mongodb based storage to store accuracy result # export MDB_ACCURACY_MDB_URL="" # export MDB_ACCURACY_MDB_DB="" diff --git a/tests/accuracy/listPerformanceAdvisor.test.ts b/tests/accuracy/listPerformanceAdvisor.test.ts new file mode 100644 index 000000000..905245688 --- /dev/null +++ b/tests/accuracy/listPerformanceAdvisor.test.ts @@ -0,0 +1,165 @@ +import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js"; +import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; + +// Shared mock tool implementations +const mockedTools = { + "atlas-list-projects": (): CallToolResult => { + return { + content: [ + { + type: "text", + text: "Found 1 project\n\n# | Name | ID\n---|------|----\n1 | mflix | mflix", + }, + ], + }; + }, + "atlas-list-clusters": (): CallToolResult => { + return { + content: [ + { + type: "text", + text: "Found 1 cluster\n\n# | Name | Type | State\n---|------|------|-----\n1 | mflix-cluster | REPLICASET | IDLE", + }, + ], + }; + }, + "atlas-list-performance-advisor": (): CallToolResult => { + return { + content: [ + { + type: "text", + text: "Found 2 performance advisor recommendations\n\n## Suggested Indexes\n# | Namespace | Weight | Avg Obj Size | Index Keys\n---|-----------|--------|--------------|------------\n1 | mflix.movies | 0.8 | 1024 | title, year\n2 | mflix.shows | 0.6 | 512 | genre, rating", + }, + ], + }; + }, +}; + +describeAccuracyTests([ + // Test for Suggested Indexes operation + { + prompt: "Can you give me index suggestions for the database 'mflix' in the project 'mflix' and cluster 'mflix-cluster'?", + expectedToolCalls: [ + { + toolName: "atlas-list-projects", + parameters: {}, + }, + { + toolName: "atlas-list-clusters", + parameters: { + projectId: "mflix", + }, + }, + { + toolName: "atlas-list-performance-advisor", + parameters: { + projectId: "mflix", + clusterName: "mflix-cluster", + operations: ["suggestedIndexes"], + }, + }, + ], + mockedTools, + }, + // Test for Drop Index Suggestions operation + { + prompt: "Show me drop index suggestions for the 'mflix' project and 'mflix-cluster' cluster", + expectedToolCalls: [ + { + toolName: "atlas-list-projects", + parameters: {}, + }, + { + toolName: "atlas-list-clusters", + parameters: { + projectId: "mflix", + }, + }, + { + toolName: "atlas-list-performance-advisor", + parameters: { + projectId: "mflix", + clusterName: "mflix-cluster", + operations: ["dropIndexSuggestions"], + }, + }, + ], + mockedTools, + }, + // Test for Slow Query Logs operation + { + prompt: "Show me the slow query logs for the 'mflix' project and 'mflix-cluster' cluster for the namespaces 'mflix.movies' and 'mflix.shows' since January 1st, 2025.", + expectedToolCalls: [ + { + toolName: "atlas-list-projects", + parameters: {}, + }, + { + toolName: "atlas-list-clusters", + parameters: { + projectId: "mflix", + }, + }, + { + toolName: "atlas-list-performance-advisor", + parameters: { + projectId: "mflix", + clusterName: "mflix-cluster", + operations: ["slowQueryLogs"], + namespaces: ["mflix.movies", "mflix.shows"], + since: "2025-01-01T00:00:00Z", + }, + }, + ], + mockedTools, + }, + // Test for Schema Suggestions operation + { + prompt: "Give me schema suggestions for the 'mflix' project and 'mflix-cluster' cluster", + expectedToolCalls: [ + { + toolName: "atlas-list-projects", + parameters: {}, + }, + { + toolName: "atlas-list-clusters", + parameters: { + projectId: "mflix", + }, + }, + { + toolName: "atlas-list-performance-advisor", + parameters: { + projectId: "mflix", + clusterName: "mflix-cluster", + operations: ["schemaSuggestions"], + }, + }, + ], + mockedTools, + }, + // Test for all operations + { + prompt: "Show me all performance advisor recommendations for the 'mflix' project and 'mflix-cluster' cluster", + expectedToolCalls: [ + { + toolName: "atlas-list-projects", + parameters: {}, + }, + { + toolName: "atlas-list-clusters", + parameters: { + projectId: "mflix", + }, + }, + { + toolName: "atlas-list-performance-advisor", + parameters: { + projectId: "mflix", + clusterName: "mflix-cluster", + }, + }, + ], + mockedTools, + }, +]); diff --git a/tests/accuracy/sdk/accuracyTestingClient.ts b/tests/accuracy/sdk/accuracyTestingClient.ts index 692694aa7..130e8fb05 100644 --- a/tests/accuracy/sdk/accuracyTestingClient.ts +++ b/tests/accuracy/sdk/accuracyTestingClient.ts @@ -79,10 +79,22 @@ export class AccuracyTestingClient { this.llmToolCalls = []; } - static async initializeClient(mdbConnectionString: string): Promise { + static async initializeClient( + mdbConnectionString: string, + atlasApiClientId?: string, + atlasApiClientSecret?: string + ): Promise { + const args = [ + MCP_SERVER_CLI_SCRIPT, + "--connectionString", + mdbConnectionString, + ...(atlasApiClientId ? ["--apiClientId", atlasApiClientId] : []), + ...(atlasApiClientSecret ? ["--apiClientSecret", atlasApiClientSecret] : []), + ]; + const clientTransport = new StdioClientTransport({ command: process.execPath, - args: [MCP_SERVER_CLI_SCRIPT, "--connectionString", mdbConnectionString], + args, }); const client = await createMCPClient({ diff --git a/tests/accuracy/sdk/describeAccuracyTests.ts b/tests/accuracy/sdk/describeAccuracyTests.ts index 6617a84f7..576fedbd4 100644 --- a/tests/accuracy/sdk/describeAccuracyTests.ts +++ b/tests/accuracy/sdk/describeAccuracyTests.ts @@ -66,6 +66,9 @@ export function describeAccuracyTests(accuracyTestConfigs: AccuracyTestConfig[]) const mdbIntegration = setupMongoDBIntegrationTest({}, []); const { populateTestData, cleanupTestDatabases } = prepareTestData(mdbIntegration); + const atlasApiClientId = process.env.MDB_MCP_API_CLIENT_ID; + const atlasApiClientSecret = process.env.MDB_MCP_API_CLIENT_SECRET; + let commitSHA: string; let accuracyResultStorage: AccuracyResultStorage; let testMCPClient: AccuracyTestingClient; @@ -79,7 +82,11 @@ export function describeAccuracyTests(accuracyTestConfigs: AccuracyTestConfig[]) commitSHA = retrievedCommitSHA; accuracyResultStorage = getAccuracyResultStorage(); - testMCPClient = await AccuracyTestingClient.initializeClient(mdbIntegration.connectionString()); + testMCPClient = await AccuracyTestingClient.initializeClient( + mdbIntegration.connectionString(), + atlasApiClientId, + atlasApiClientSecret + ); agent = getVercelToolCallingAgent(); });