feat: Atlas performance advisor tool accuracy tests

scripts/accuracy/runAccuracyTests.sh

@@ -8,6 +8,11 @@ export MDB_ACCURACY_RUN_ID=$(npx uuid v4)
 # export MDB_AZURE_OPEN_AI_API_KEY=""
 # export MDB_AZURE_OPEN_AI_API_URL=""
 
+# For providing Atlas API credentials (required for Atlas tools)
+# Set dummy values for testing (allows Atlas tools to be registered for mocking)
+export MDB_MCP_API_CLIENT_ID=${MDB_MCP_API_CLIENT_ID:-"test-atlas-client-id"}
+export MDB_MCP_API_CLIENT_SECRET=${MDB_MCP_API_CLIENT_SECRET:-"test-atlas-client-secret"}
+
 # For providing a mongodb based storage to store accuracy result
 # export MDB_ACCURACY_MDB_URL=""
 # export MDB_ACCURACY_MDB_DB=""

tests/accuracy/listPerformanceAdvisor.test.ts

@@ -0,0 +1,165 @@
+import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js";
+import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
+
+// Shared mock tool implementations
+const mockedTools = {
+    "atlas-list-projects": (): CallToolResult => {
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: "Found 1 project\n\n# | Name | ID\n---|------|----\n1 | mflix | mflix",
+                },
+            ],
+        };
+    },
+    "atlas-list-clusters": (): CallToolResult => {
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: "Found 1 cluster\n\n# | Name | Type | State\n---|------|------|-----\n1 | mflix-cluster | REPLICASET | IDLE",
+                },
+            ],
+        };
+    },
+    "atlas-list-performance-advisor": (): CallToolResult => {
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: "Found 2 performance advisor recommendations\n\n## Suggested Indexes\n# | Namespace | Weight | Avg Obj Size | Index Keys\n---|-----------|--------|--------------|------------\n1 | mflix.movies | 0.8 | 1024 | title, year\n2 | mflix.shows | 0.6 | 512 | genre, rating",
+                },
+            ],
+        };
+    },
+};
+
+describeAccuracyTests([
+    // Test for Suggested Indexes operation
+    {
+        prompt: "Can you give me index suggestions for the database 'mflix' in the project 'mflix' and cluster 'mflix-cluster'?",
+        expectedToolCalls: [
+            {
+                toolName: "atlas-list-projects",
+                parameters: {},
+            },
+            {
+                toolName: "atlas-list-clusters",
+                parameters: {
+                    projectId: "mflix",
+                },
+            },
+            {
+                toolName: "atlas-list-performance-advisor",
+                parameters: {
+                    projectId: "mflix",
+                    clusterName: "mflix-cluster",
+                    operations: ["suggestedIndexes"],
+                },
+            },
+        ],
+        mockedTools,
+    },
+    // Test for Drop Index Suggestions operation
+    {
+        prompt: "Show me drop index suggestions for the 'mflix' project and 'mflix-cluster' cluster",
+        expectedToolCalls: [
+            {
+                toolName: "atlas-list-projects",
+                parameters: {},
+            },
+            {
+                toolName: "atlas-list-clusters",
+                parameters: {
+                    projectId: "mflix",
+                },
+            },
+            {
+                toolName: "atlas-list-performance-advisor",
+                parameters: {
+                    projectId: "mflix",
+                    clusterName: "mflix-cluster",
+                    operations: ["dropIndexSuggestions"],
+                },
+            },
+        ],
+        mockedTools,
+    },
+    // Test for Slow Query Logs operation
+    {
+        prompt: "Show me the slow query logs for the 'mflix' project and 'mflix-cluster' cluster for the namespaces 'mflix.movies' and 'mflix.shows' since January 1st, 2025.",
+        expectedToolCalls: [
+            {
+                toolName: "atlas-list-projects",
+                parameters: {},
+            },
+            {
+                toolName: "atlas-list-clusters",
+                parameters: {
+                    projectId: "mflix",
+                },
+            },
+            {
+                toolName: "atlas-list-performance-advisor",
+                parameters: {
+                    projectId: "mflix",
+                    clusterName: "mflix-cluster",
+                    operations: ["slowQueryLogs"],
+                    namespaces: ["mflix.movies", "mflix.shows"],
+                    since: "2025-01-01T00:00:00Z",
+                },
+            },
+        ],
+        mockedTools,
+    },
+    // Test for Schema Suggestions operation
+    {
+        prompt: "Give me schema suggestions for the 'mflix' project and 'mflix-cluster' cluster",
+        expectedToolCalls: [
+            {
+                toolName: "atlas-list-projects",
+                parameters: {},
+            },
+            {
+                toolName: "atlas-list-clusters",
+                parameters: {
+                    projectId: "mflix",
+                },
+            },
+            {
+                toolName: "atlas-list-performance-advisor",
+                parameters: {
+                    projectId: "mflix",
+                    clusterName: "mflix-cluster",
+                    operations: ["schemaSuggestions"],
+                },
+            },
+        ],
+        mockedTools,
+    },
+    // Test for all operations
+    {
+        prompt: "Show me all performance advisor recommendations for the 'mflix' project and 'mflix-cluster' cluster",
+        expectedToolCalls: [
+            {
+                toolName: "atlas-list-projects",
+                parameters: {},
+            },
+            {
+                toolName: "atlas-list-clusters",
+                parameters: {
+                    projectId: "mflix",
+                },
+            },
+            {
+                toolName: "atlas-list-performance-advisor",
+                parameters: {
+                    projectId: "mflix",
+                    clusterName: "mflix-cluster",
+                },
+            },
+        ],
+        mockedTools,
+    },
+]);

tests/accuracy/sdk/accuracyTestingClient.ts

@@ -79,10 +79,22 @@ export class AccuracyTestingClient {
         this.llmToolCalls = [];
     }
 
-    static async initializeClient(mdbConnectionString: string): Promise<AccuracyTestingClient> {
+    static async initializeClient(
+        mdbConnectionString: string,
+        atlasApiClientId?: string,
+        atlasApiClientSecret?: string
+    ): Promise<AccuracyTestingClient> {
+        const args = [
+            MCP_SERVER_CLI_SCRIPT,
+            "--connectionString",
+            mdbConnectionString,
+            ...(atlasApiClientId ? ["--apiClientId", atlasApiClientId] : []),
+            ...(atlasApiClientSecret ? ["--apiClientSecret", atlasApiClientSecret] : []),
+        ];
+
         const clientTransport = new StdioClientTransport({
             command: process.execPath,
-            args: [MCP_SERVER_CLI_SCRIPT, "--connectionString", mdbConnectionString],
+            args,
         });
 
         const client = await createMCPClient({

tests/accuracy/sdk/describeAccuracyTests.ts

@@ -66,6 +66,9 @@ export function describeAccuracyTests(accuracyTestConfigs: AccuracyTestConfig[])
         const mdbIntegration = setupMongoDBIntegrationTest({}, []);
         const { populateTestData, cleanupTestDatabases } = prepareTestData(mdbIntegration);
 
+        const atlasApiClientId = process.env.MDB_MCP_API_CLIENT_ID;
+        const atlasApiClientSecret = process.env.MDB_MCP_API_CLIENT_SECRET;
+
         let commitSHA: string;
         let accuracyResultStorage: AccuracyResultStorage;
         let testMCPClient: AccuracyTestingClient;
@@ -79,7 +82,11 @@ export function describeAccuracyTests(accuracyTestConfigs: AccuracyTestConfig[])
             commitSHA = retrievedCommitSHA;
 
             accuracyResultStorage = getAccuracyResultStorage();
-            testMCPClient = await AccuracyTestingClient.initializeClient(mdbIntegration.connectionString());
+            testMCPClient = await AccuracyTestingClient.initializeClient(
+                mdbIntegration.connectionString(),
+                atlasApiClientId,
+                atlasApiClientSecret
+            );
             agent = getVercelToolCallingAgent();
         });