Skip to content

Commit 6ee1274

Browse files
kylelai1kylelai1
authored
feat: Atlas performance advisor tool accuracy tests (#577)
1 parent 173c587 commit 6ee1274

File tree

4 files changed

+192
-3
lines changed

4 files changed

+192
-3
lines changed

‎scripts/accuracy/runAccuracyTests.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@ export MDB_ACCURACY_RUN_ID=$(npx uuid v4)
88
# export MDB_AZURE_OPEN_AI_API_KEY=""
99
# export MDB_AZURE_OPEN_AI_API_URL=""
1010

11+
# For providing Atlas API credentials (required for Atlas tools)
12+
# Set dummy values for testing (allows Atlas tools to be registered for mocking)
13+
export MDB_MCP_API_CLIENT_ID=${MDB_MCP_API_CLIENT_ID:-"test-atlas-client-id"}
14+
export MDB_MCP_API_CLIENT_SECRET=${MDB_MCP_API_CLIENT_SECRET:-"test-atlas-client-secret"}
15+
1116
# For providing a mongodb based storage to store accuracy result
1217
# export MDB_ACCURACY_MDB_URL=""
1318
# export MDB_ACCURACY_MDB_DB=""

‎tests/accuracy/listPerformanceAdvisor.test.ts‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js";
2+
import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
3+
4+
// Shared mock tool implementations
5+
const mockedTools = {
6+
"atlas-list-projects": (): CallToolResult => {
7+
return {
8+
content: [
9+
{
10+
type: "text",
11+
text: "Found 1 project\n\n# | Name | ID\n---|------|----\n1 | mflix | mflix",
12+
},
13+
],
14+
};
15+
},
16+
"atlas-list-clusters": (): CallToolResult => {
17+
return {
18+
content: [
19+
{
20+
type: "text",
21+
text: "Found 1 cluster\n\n# | Name | Type | State\n---|------|------|-----\n1 | mflix-cluster | REPLICASET | IDLE",
22+
},
23+
],
24+
};
25+
},
26+
"atlas-list-performance-advisor": (): CallToolResult => {
27+
return {
28+
content: [
29+
{
30+
type: "text",
31+
text: "Found 2 performance advisor recommendations\n\n## Suggested Indexes\n# | Namespace | Weight | Avg Obj Size | Index Keys\n---|-----------|--------|--------------|------------\n1 | mflix.movies | 0.8 | 1024 | title, year\n2 | mflix.shows | 0.6 | 512 | genre, rating",
32+
},
33+
],
34+
};
35+
},
36+
};
37+
38+
describeAccuracyTests([
39+
// Test for Suggested Indexes operation
40+
{
41+
prompt: "Can you give me index suggestions for the database 'mflix' in the project 'mflix' and cluster 'mflix-cluster'?",
42+
expectedToolCalls: [
43+
{
44+
toolName: "atlas-list-projects",
45+
parameters: {},
46+
},
47+
{
48+
toolName: "atlas-list-clusters",
49+
parameters: {
50+
projectId: "mflix",
51+
},
52+
},
53+
{
54+
toolName: "atlas-list-performance-advisor",
55+
parameters: {
56+
projectId: "mflix",
57+
clusterName: "mflix-cluster",
58+
operations: ["suggestedIndexes"],
59+
},
60+
},
61+
],
62+
mockedTools,
63+
},
64+
// Test for Drop Index Suggestions operation
65+
{
66+
prompt: "Show me drop index suggestions for the 'mflix' project and 'mflix-cluster' cluster",
67+
expectedToolCalls: [
68+
{
69+
toolName: "atlas-list-projects",
70+
parameters: {},
71+
},
72+
{
73+
toolName: "atlas-list-clusters",
74+
parameters: {
75+
projectId: "mflix",
76+
},
77+
},
78+
{
79+
toolName: "atlas-list-performance-advisor",
80+
parameters: {
81+
projectId: "mflix",
82+
clusterName: "mflix-cluster",
83+
operations: ["dropIndexSuggestions"],
84+
},
85+
},
86+
],
87+
mockedTools,
88+
},
89+
// Test for Slow Query Logs operation
90+
{
91+
prompt: "Show me the slow query logs for the 'mflix' project and 'mflix-cluster' cluster for the namespaces 'mflix.movies' and 'mflix.shows' since January 1st, 2025.",
92+
expectedToolCalls: [
93+
{
94+
toolName: "atlas-list-projects",
95+
parameters: {},
96+
},
97+
{
98+
toolName: "atlas-list-clusters",
99+
parameters: {
100+
projectId: "mflix",
101+
},
102+
},
103+
{
104+
toolName: "atlas-list-performance-advisor",
105+
parameters: {
106+
projectId: "mflix",
107+
clusterName: "mflix-cluster",
108+
operations: ["slowQueryLogs"],
109+
namespaces: ["mflix.movies", "mflix.shows"],
110+
since: "2025-01-01T00:00:00Z",
111+
},
112+
},
113+
],
114+
mockedTools,
115+
},
116+
// Test for Schema Suggestions operation
117+
{
118+
prompt: "Give me schema suggestions for the 'mflix' project and 'mflix-cluster' cluster",
119+
expectedToolCalls: [
120+
{
121+
toolName: "atlas-list-projects",
122+
parameters: {},
123+
},
124+
{
125+
toolName: "atlas-list-clusters",
126+
parameters: {
127+
projectId: "mflix",
128+
},
129+
},
130+
{
131+
toolName: "atlas-list-performance-advisor",
132+
parameters: {
133+
projectId: "mflix",
134+
clusterName: "mflix-cluster",
135+
operations: ["schemaSuggestions"],
136+
},
137+
},
138+
],
139+
mockedTools,
140+
},
141+
// Test for all operations
142+
{
143+
prompt: "Show me all performance advisor recommendations for the 'mflix' project and 'mflix-cluster' cluster",
144+
expectedToolCalls: [
145+
{
146+
toolName: "atlas-list-projects",
147+
parameters: {},
148+
},
149+
{
150+
toolName: "atlas-list-clusters",
151+
parameters: {
152+
projectId: "mflix",
153+
},
154+
},
155+
{
156+
toolName: "atlas-list-performance-advisor",
157+
parameters: {
158+
projectId: "mflix",
159+
clusterName: "mflix-cluster",
160+
},
161+
},
162+
],
163+
mockedTools,
164+
},
165+
]);

‎tests/accuracy/sdk/accuracyTestingClient.ts‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,10 +79,22 @@ export class AccuracyTestingClient {
7979
this.llmToolCalls = [];
8080
}
8181

82-
static async initializeClient(mdbConnectionString: string): Promise<AccuracyTestingClient> {
82+
static async initializeClient(
83+
mdbConnectionString: string,
84+
atlasApiClientId?: string,
85+
atlasApiClientSecret?: string
86+
): Promise<AccuracyTestingClient> {
87+
const args = [
88+
MCP_SERVER_CLI_SCRIPT,
89+
"--connectionString",
90+
mdbConnectionString,
91+
...(atlasApiClientId ? ["--apiClientId", atlasApiClientId] : []),
92+
...(atlasApiClientSecret ? ["--apiClientSecret", atlasApiClientSecret] : []),
93+
];
94+
8395
const clientTransport = new StdioClientTransport({
8496
command: process.execPath,
85-
args: [MCP_SERVER_CLI_SCRIPT, "--connectionString", mdbConnectionString],
97+
args,
8698
});
8799

88100
const client = await createMCPClient({

‎tests/accuracy/sdk/describeAccuracyTests.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,9 @@ export function describeAccuracyTests(accuracyTestConfigs: AccuracyTestConfig[])
6666
const mdbIntegration = setupMongoDBIntegrationTest({}, []);
6767
const { populateTestData, cleanupTestDatabases } = prepareTestData(mdbIntegration);
6868

69+
const atlasApiClientId = process.env.MDB_MCP_API_CLIENT_ID;
70+
const atlasApiClientSecret = process.env.MDB_MCP_API_CLIENT_SECRET;
71+
6972
let commitSHA: string;
7073
let accuracyResultStorage: AccuracyResultStorage;
7174
let testMCPClient: AccuracyTestingClient;
@@ -79,7 +82,11 @@ export function describeAccuracyTests(accuracyTestConfigs: AccuracyTestConfig[])
7982
commitSHA = retrievedCommitSHA;
8083

8184
accuracyResultStorage = getAccuracyResultStorage();
82-
testMCPClient = await AccuracyTestingClient.initializeClient(mdbIntegration.connectionString());
85+
testMCPClient = await AccuracyTestingClient.initializeClient(
86+
mdbIntegration.connectionString(),
87+
atlasApiClientId,
88+
atlasApiClientSecret
89+
);
8390
agent = getVercelToolCallingAgent();
8491
});
8592

0 commit comments

Comments
 (0)